go mod tidy

* Fix typos

- Add a quantifier to make the description unambiguous
- Add a missing verb

* Fix href of FAQs

.github/workflows/build-test.yml

@@ -66,3 +66,7 @@ jobs:
       - name: Example SDK Advanced
         run: go run .
         working-directory: examples/advanced/
+
+      - name: Example SDK with speed control
+        run: go run .
+        working-directory: examples/with_speed_control/

.gitignore

@@ -36,5 +36,8 @@ pkg/protocols/headless/engine/.cache
 /fuzzplayground
 integration_tests/fuzzplayground
 /dsl.md
+/nuclei-stats
+/nuclei-stats-*
+/scan-charts
 
 

Makefile

@@ -11,9 +11,18 @@ ifneq ($(shell go env GOOS),darwin)
 LDFLAGS := -extldflags "-static"
 endif
     
+.PHONY: all build build-stats scan-charts docs test integration functional tidy devtools jsupdate ts fuzzplayground memogen dsl-docs
+
 all: build
 build:
+	rm -f nuclei 2>/dev/null
 	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "nuclei" cmd/nuclei/main.go
+build-stats:
+	rm -f nuclei-stats 2>/dev/null
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -tags=stats -o "nuclei-stats" cmd/nuclei/main.go
+scan-charts:
+	rm -f scan-charts 2>/dev/null
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scan-charts" cmd/scan-charts/main.go
 docs:
 	if ! which dstdocgen > /dev/null; then
 		echo -e "Command not found! Install? (y/n) \c"

README.md

@@ -22,7 +22,7 @@
   <a href="#install-nuclei">Install</a> •
   <a href="https://docs.projectdiscovery.io/tools/nuclei/">Documentation</a> •
   <a href="#credits">Credits</a> •
-  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">FAQs</a> •
+  <a href="https://docs.projectdiscovery.io/tools/nuclei/faq">FAQs</a> •
   <a href="https://discord.gg/projectdiscovery">Join Discord</a>
 </p>
 

README_CN.md

@@ -23,8 +23,8 @@
   <a href="#对于安全工程师">对于安全工程师</a> •
   <a href="#对于开发和组织">对于开发者</a> •
   <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">文档</a> •
-  <a href="#c致谢">致谢</a> •
-  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">常见问题</a> •
+  <a href="#致谢">致谢</a> •
+  <a href="https://docs.projectdiscovery.io/tools/nuclei/faq">常见问题</a> •
   <a href="https://discord.gg/projectdiscovery">加入Discord</a>
 </p>
 
@@ -39,7 +39,7 @@
 
 Nuclei使用零误报的定制模板向目标发送请求，同时可以对主机进行批量快速扫描。Nuclei提供TCP、DNS、HTTP、FILE等各类协议的扫描，通过强大且灵活的模板，可以使用Nuclei模拟各种安全检查。
 
-我们的[模板仓库](https://github.com/projectdiscovery/nuclei-templates)包含**超过300**安全研究员和工程师提供的模板。
+我们的[模板仓库](https://github.com/projectdiscovery/nuclei-templates)包含**超过300名**安全研究员和工程师提供的模板。
 
 
 
@@ -439,7 +439,7 @@ Nuclei构建很简单，通过数百名安全研究员的社区模板，Nuclei
 </a>
 </p>
 
-另外您可以其他类似的开源项目：
+另外您可以了解其他类似的开源项目：
 
 [FFuF](https://github.com/ffuf/ffuf), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [Jaeles](https://github.com/jaeles-project/jaeles), [ChopChop](https://github.com/michelin/ChopChop)
 

README_ID.md

@@ -24,7 +24,7 @@
   <a href="#untuk-pengembang-dan-organisasi">Untuk Pengembang</a> •
   <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">Dokumentasi</a> •
   <a href="#kredit">Kredit</a> •
-  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">Tanya Jawab</a> •
+  <a href="https://docs.projectdiscovery.io/tools/nuclei/faq">Tanya Jawab</a> •
   <a href="https://discord.gg/projectdiscovery">Gabung Discord</a>
 </p>
 

README_KR.md

@@ -23,7 +23,7 @@
   <a href="#개발자를-위한">개발자를 위한</a> •
   <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">문서</a> •
   <a href="#credits">Credits</a> •
-  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">FAQs</a> •
+  <a href="https://docs.projectdiscovery.io/tools/nuclei/faq">FAQs</a> •
   <a href="https://discord.gg/projectdiscovery"> Discord 참가</a>
 </p>
 

SYNTAX-REFERENCE.md

@@ -2750,6 +2750,8 @@ Enum Values:
   - <code>TLSA</code>
 
   - <code>ANY</code>
+
+  - <code>SRV</code>
 </div>
 
 <hr />

cmd/docgen/docgen.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"log"
 	"os"
+	"reflect"
 	"regexp"
 	"strings"
 
@@ -33,6 +34,12 @@ func main() {
 
 	// Generate jsonschema
 	r := &jsonschema.Reflector{}
+	r.Namer = func(r reflect.Type) string {
+		if r.Kind() == reflect.Slice {
+			return ""
+		}
+		return r.String()
+	}
 	jsonschemaData := r.Reflect(&templates.Template{})
 
 	var buf bytes.Buffer

cmd/integration-test/dns.go

@@ -5,7 +5,12 @@ import (
 )
 
 var dnsTestCases = []TestCaseInfo{
-	{Path: "protocols/dns/basic.yaml", TestCase: &dnsBasic{}},
+	{Path: "protocols/dns/a.yaml", TestCase: &dnsBasic{}},
+	{Path: "protocols/dns/aaaa.yaml", TestCase: &dnsBasic{}},
+	{Path: "protocols/dns/cname.yaml", TestCase: &dnsBasic{}},
+	{Path: "protocols/dns/srv.yaml", TestCase: &dnsBasic{}},
+	{Path: "protocols/dns/ns.yaml", TestCase: &dnsBasic{}},
+	{Path: "protocols/dns/txt.yaml", TestCase: &dnsBasic{}},
 	{Path: "protocols/dns/ptr.yaml", TestCase: &dnsPtr{}},
 	{Path: "protocols/dns/caa.yaml", TestCase: &dnsCAA{}},
 	{Path: "protocols/dns/tlsa.yaml", TestCase: &dnsTLSA{}},

cmd/integration-test/workflow.go

@@ -3,11 +3,15 @@ package main
 import (
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 
 	"github.com/julienschmidt/httprouter"
 
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
 )
 
@@ -16,11 +20,40 @@ var workflowTestcases = []TestCaseInfo{
 	{Path: "workflow/condition-matched.yaml", TestCase: &workflowConditionMatched{}},
 	{Path: "workflow/condition-unmatched.yaml", TestCase: &workflowConditionUnmatch{}},
 	{Path: "workflow/matcher-name.yaml", TestCase: &workflowMatcherName{}},
+	{Path: "workflow/complex-conditions.yaml", TestCase: &workflowComplexConditions{}},
 	{Path: "workflow/http-value-share-workflow.yaml", TestCase: &workflowHttpKeyValueShare{}},
 	{Path: "workflow/dns-value-share-workflow.yaml", TestCase: &workflowDnsKeyValueShare{}},
+	{Path: "workflow/code-value-share-workflow.yaml", TestCase: &workflowCodeKeyValueShare{}, DisableOn: isCodeDisabled}, // isCodeDisabled declared in code.go
+	{Path: "workflow/multiprotocol-value-share-workflow.yaml", TestCase: &workflowMultiProtocolKeyValueShare{}},
 	{Path: "workflow/shared-cookie.yaml", TestCase: &workflowSharedCookies{}},
 }
 
+func init() {
+	// sign code templates (unless they are disabled)
+	if !isCodeDisabled() {
+		// allow local file access to load content of file references in template
+		// in order to sign them for testing purposes
+		templates.TemplateSignerLFA()
+
+		// testCertFile and testKeyFile are declared in code.go
+		tsigner, err := signer.NewTemplateSignerFromFiles(testCertFile, testKeyFile)
+		if err != nil {
+			panic(err)
+		}
+
+		// only the code templates are necessary to be signed
+		var templatesToSign = []string{
+			"workflow/code-template-1.yaml",
+			"workflow/code-template-2.yaml",
+		}
+		for _, templatePath := range templatesToSign {
+			if err := templates.SignTemplate(tsigner, templatePath); err != nil {
+				log.Fatalf("Could not sign template %v got: %s\n", templatePath, err)
+			}
+		}
+	}
+}
+
 type workflowBasic struct{}
 
 // Execute executes a test case and returns an error if occurred
@@ -97,6 +130,30 @@ func (h *workflowMatcherName) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+type workflowComplexConditions struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *workflowComplexConditions) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "This is test matcher text")
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	for _, result := range results {
+		if !strings.Contains(result, "test-matcher-3") {
+			return fmt.Errorf("incorrect result: the \"basic-get-third:test-matcher-3\" and only that should be matched!\nResults:\n\t%s", strings.Join(results, "\n\t"))
+		}
+	}
+	return expectResultsCount(results, 2)
+}
+
 type workflowHttpKeyValueShare struct{}
 
 // Execute executes a test case and returns an error if occurred
@@ -133,6 +190,45 @@ func (h *workflowDnsKeyValueShare) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+type workflowCodeKeyValueShare struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *workflowCodeKeyValueShare) Execute(filePath string) error {
+	// provide the Certificate File that the code templates are signed with
+	certEnvVar := signer.CertEnvVarName + "=" + testCertFile
+
+	results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, []string{certEnvVar}, "-workflows", filePath, "-target", "input", "-code")
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}
+
+type workflowMultiProtocolKeyValueShare struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *workflowMultiProtocolKeyValueShare) Execute(filePath string) error {
+	router := httprouter.New()
+	// the response of path1 contains a domain that will be extracted and shared with the second template
+	router.GET("/path1", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "href=\"blog.projectdiscovery.io\"")
+	})
+	// path2 responds with the value of the "extracted" query parameter, e.g.: /path2?extracted=blog.projectdiscovery.io => blog.projectdiscovery.io
+	router.GET("/path2", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "%s", r.URL.Query().Get("extracted"))
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 2)
+}
+
 type workflowSharedCookies struct{}
 
 // Execute executes a test case and returns an error if occurred

cmd/nuclei/main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/projectdiscovery/utils/auth/pdcp"
 	"github.com/projectdiscovery/utils/env"
 	_ "github.com/projectdiscovery/utils/pprof"
+	stringsutil "github.com/projectdiscovery/utils/strings"
 
 	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
@@ -329,13 +330,15 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	flagSet.CreateGroup("rate-limit", "Rate-Limit",
 		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
-		flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"),
+		flagSet.DurationVarP(&options.RateLimitDuration, "rate-limit-duration", "rld", time.Second, "maximum number of requests to send per second"),
+		flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute (DEPRECATED)"),
 		flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
 		flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"),
 		flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"),
 		flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "headc", 10, "maximum number of headless templates to be executed in parallel"),
 		flagSet.IntVarP(&options.JsConcurrency, "js-concurrency", "jsc", 120, "maximum number of javascript runtimes to be executed in parallel"),
 		flagSet.IntVarP(&options.PayloadConcurrency, "payload-concurrency", "pc", 25, "max payload concurrency for each template"),
+		flagSet.IntVarP(&options.ProbeConcurrency, "probe-concurrency", "prc", 50, "http probe concurrency with httpx"),
 	)
 	flagSet.CreateGroup("optimization", "Optimizations",
 		flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait in seconds before timeout"),
@@ -412,6 +415,11 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.PreFetchSecrets, "prefetch-secrets", "ps", false, "prefetch secrets from the secrets file"),
 	)
 
+	flagSet.CreateGroup("tests", "Tests",
+		flagSet.BoolVarP(&options.AutogenerateTests, "autogenerate-tests", "agt", false, "enable autogeneration of nuclei test-cases (http only now)"),
+		flagSet.BoolVarP(&options.RunNucleiTests, "run-tests", "rts", false, "run generated nuclei test-cases"),
+	)
+
 	flagSet.SetCustomHelpText(`EXAMPLES:
 Run nuclei on single host:
 	$ nuclei -target example.com
@@ -597,10 +605,10 @@ Note: Make sure you have backup of your custom nuclei-templates before proceedin
 			gologger.Fatal().Msgf("could not read response: %s", err)
 		}
 		resp = strings.TrimSpace(resp)
-		if strings.EqualFold(resp, "y") || strings.EqualFold(resp, "yes") {
+		if stringsutil.EqualFoldAny(resp, "y", "yes") {
 			break
 		}
-		if strings.EqualFold(resp, "n") || strings.EqualFold(resp, "no") || resp == "" {
+		if stringsutil.EqualFoldAny(resp, "n", "no", "") {
 			fmt.Println("Exiting...")
 			os.Exit(0)
 		}

cmd/nuclei/srv.yaml

@@ -0,0 +1,18 @@
+id: basic-dns-a-example
+
+info:
+  name: Test DNS A Query Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: SRV
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        part: all
+        words:
+          - "SRV"

cmd/scan-charts/main.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/scan/charts"
+)
+
+var (
+	dir     string
+	address string
+	output  string
+)
+
+func main() {
+	flag.StringVar(&dir, "dir", "", "directory to scan")
+	flag.StringVar(&address, "address", ":9000", "address to run the server on")
+	flag.StringVar(&output, "output", "", "output filename of generated html file")
+	flag.Parse()
+
+	if dir == "" {
+		flag.Usage()
+		return
+	}
+
+	server, err := charts.NewScanEventsCharts(dir)
+	if err != nil {
+		panic(err)
+	}
+	server.PrintInfo()
+
+	if output != "" {
+		if err = server.GenerateHTML(output); err != nil {
+			panic(err)
+		}
+		return
+	}
+
+	server.Start(address)
+}

cmd/tmc/main.go

@@ -231,7 +231,7 @@ func logErrMsg(path string, err error, debug bool, errFile *os.File) string {
 	return msg
 }
 
-// enhanceTemplateData enhances template data using templateman
+// enhanceTemplate enhances template data using templateman
 // ref: https://github.com/projectdiscovery/templateman/blob/main/templateman-rest-api/README.md#enhance-api
 func enhanceTemplate(data string) (string, bool, error) {
 	resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/enhance", tmBaseUrl), "application/x-yaml", strings.NewReader(data))
@@ -266,7 +266,7 @@ func enhanceTemplate(data string) (string, bool, error) {
 	return data, false, errorutil.New("template enhance failed")
 }
 
-// formatTemplateData formats template data using templateman format api
+// formatTemplate formats template data using templateman format api
 func formatTemplate(data string) (string, bool, error) {
 	resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/format", tmBaseUrl), "application/x-yaml", strings.NewReader(data))
 	if err != nil {

examples/advanced/advanced.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 func main() {
@@ -12,7 +12,10 @@ func main() {
 		panic(err)
 	}
 	// setup sizedWaitgroup to handle concurrency
-	sg := sizedwaitgroup.New(10)
+	sg, err := syncutil.New(syncutil.WithSize(10))
+	if err != nil {
+		panic(err)
+	}
 
 	// scan 1 = run dns templates on scanme.sh
 	sg.Add()

examples/with_speed_control/main.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"log"
+	"sync"
+	"time"
+
+	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
+)
+
+func main() {
+	ne, err := initializeNucleiEngine()
+	if err != nil {
+		panic(err)
+	}
+	defer ne.Close()
+
+	ne.LoadTargets([]string{"http://honey.scanme.sh"}, false)
+
+	var wg sync.WaitGroup
+	wg.Add(3)
+
+	go testRateLimit(&wg, ne)
+	go testThreadsAndBulkSize(&wg, ne)
+	go testPayloadConcurrency(&wg, ne)
+
+	err = ne.ExecuteWithCallback(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	wg.Wait()
+}
+
+func initializeNucleiEngine() (*nuclei.NucleiEngine, error) {
+	return nuclei.NewNucleiEngine(
+		nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"oast"}}),
+		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}),
+		nuclei.WithGlobalRateLimit(1, time.Second),
+		nuclei.WithConcurrency(nuclei.Concurrency{
+			TemplateConcurrency:           1,
+			HostConcurrency:               1,
+			HeadlessHostConcurrency:       1,
+			HeadlessTemplateConcurrency:   1,
+			JavascriptTemplateConcurrency: 1,
+			TemplatePayloadConcurrency:    1,
+			ProbeConcurrency:              1,
+		}),
+	)
+}
+
+func testRateLimit(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
+	defer wg.Done()
+	verifyRateLimit(ne, 1, 5000)
+}
+
+func testThreadsAndBulkSize(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
+	defer wg.Done()
+	initialTemplateThreads, initialBulkSize := 1, 1
+	verifyThreadsAndBulkSize(ne, initialTemplateThreads, initialBulkSize, 25, 25)
+}
+
+func testPayloadConcurrency(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
+	defer wg.Done()
+	verifyPayloadConcurrency(ne, 1, 500)
+}
+
+func verifyRateLimit(ne *nuclei.NucleiEngine, initialRate, finalRate int) {
+	if ne.GetExecuterOptions().RateLimiter.GetLimit() != uint(initialRate) {
+		panic("wrong initial rate limit")
+	}
+	time.Sleep(5 * time.Second)
+	ne.Options().RateLimit = finalRate
+	time.Sleep(20 * time.Second)
+	if ne.GetExecuterOptions().RateLimiter.GetLimit() != uint(finalRate) {
+		panic("wrong final rate limit")
+	}
+}
+
+func verifyThreadsAndBulkSize(ne *nuclei.NucleiEngine, initialThreads, initialBulk, finalThreads, finalBulk int) {
+	if ne.Options().TemplateThreads != initialThreads || ne.Options().BulkSize != initialBulk {
+		panic("wrong initial standard concurrency")
+	}
+	time.Sleep(5 * time.Second)
+	ne.Options().TemplateThreads = finalThreads
+	ne.Options().BulkSize = finalBulk
+	time.Sleep(20 * time.Second)
+	if ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size != finalBulk || ne.Engine().WorkPool().Default.Size != finalThreads {
+		log.Fatal("wrong final concurrency", ne.Engine().WorkPool().Default.Size, finalThreads, ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size, finalBulk)
+	}
+}
+
+func verifyPayloadConcurrency(ne *nuclei.NucleiEngine, initialPayloadConcurrency, finalPayloadConcurrency int) {
+	if ne.Options().PayloadConcurrency != initialPayloadConcurrency {
+		panic("wrong initial payload concurrency")
+	}
+	time.Sleep(5 * time.Second)
+	ne.Options().PayloadConcurrency = finalPayloadConcurrency
+	time.Sleep(20 * time.Second)
+	if ne.GetExecuterOptions().GetThreadsForNPayloadRequests(100, 0) != finalPayloadConcurrency {
+		panic("wrong final payload concurrency")
+	}
+}

go.mod

@@ -20,12 +20,12 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/fastdialer v0.0.64
+	github.com/projectdiscovery/fastdialer v0.0.67
 	github.com/projectdiscovery/hmap v0.0.41
 	github.com/projectdiscovery/interactsh v1.1.9
-	github.com/projectdiscovery/rawhttp v0.1.41
+	github.com/projectdiscovery/rawhttp v0.1.45
 	github.com/projectdiscovery/retryabledns v1.0.58
-	github.com/projectdiscovery/retryablehttp-go v1.0.54
+	github.com/projectdiscovery/retryablehttp-go v1.0.55
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -38,7 +38,7 @@ require (
 	github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222
 	github.com/xanzy/go-gitlab v0.84.0
 	go.uber.org/multierr v1.11.0
-	golang.org/x/net v0.21.0
+	golang.org/x/net v0.24.0
 	golang.org/x/oauth2 v0.11.0
 	golang.org/x/text v0.14.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -70,6 +70,7 @@ require (
 	github.com/go-ldap/ldap/v3 v3.4.5
 	github.com/go-pg/pg v8.0.7+incompatible
 	github.com/go-sql-driver/mysql v1.7.1
+	github.com/goccy/go-yaml v1.11.3
 	github.com/h2non/filetype v1.1.3
 	github.com/labstack/echo/v4 v4.10.2
 	github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa
@@ -78,29 +79,32 @@ require (
 	github.com/mholt/archiver v3.1.1+incompatible
 	github.com/ory/dockertest/v3 v3.10.0
 	github.com/praetorian-inc/fingerprintx v1.1.9
-	github.com/projectdiscovery/dsl v0.0.50
+	github.com/projectdiscovery/dsl v0.0.52
 	github.com/projectdiscovery/fasttemplate v0.0.2
+	github.com/projectdiscovery/freeport v0.0.5
 	github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
-	github.com/projectdiscovery/goflags v0.1.46
+	github.com/projectdiscovery/goflags v0.1.48
 	github.com/projectdiscovery/gologger v1.1.12
 	github.com/projectdiscovery/gostruct v0.0.2
 	github.com/projectdiscovery/gozero v0.0.2
 	github.com/projectdiscovery/httpx v1.6.0
 	github.com/projectdiscovery/mapcidr v1.1.16
+	github.com/projectdiscovery/martian/v3 v3.0.0-20240219194442-fed3b744f477
 	github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
+	github.com/projectdiscovery/proxify v0.0.13-0.20240407134611-de309e8af9ca
 	github.com/projectdiscovery/ratelimit v0.0.35
 	github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
 	github.com/projectdiscovery/useragent v0.0.40
-	github.com/projectdiscovery/utils v0.0.87
+	github.com/projectdiscovery/utils v0.0.88
 	github.com/projectdiscovery/wappalyzergo v0.0.116
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
 	github.com/stretchr/testify v1.9.0
 	github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
-	golang.org/x/term v0.17.0
+	golang.org/x/term v0.19.0
 	gopkg.in/yaml.v3 v3.0.1
 	moul.io/http2curl v1.0.0
 )
@@ -114,8 +118,10 @@ require (
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
+	github.com/Shopify/sarama v1.38.1 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
@@ -142,7 +148,12 @@ require (
 	github.com/docker/cli v24.0.5+incompatible // indirect
 	github.com/docker/docker v24.0.9+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/fatih/color v1.15.0 // indirect
+	github.com/eapache/channels v1.1.0 // indirect
+	github.com/eapache/go-resiliency v1.3.0 // indirect
+	github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
+	github.com/eapache/queue v1.1.0 // indirect
+	github.com/elastic/go-elasticsearch/v7 v7.17.10 // indirect
+	github.com/fatih/color v1.16.0 // indirect
 	github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/geoffgarside/ber v1.1.0 // indirect
@@ -161,9 +172,12 @@ require (
 	github.com/google/go-github/v30 v30.1.0 // indirect
 	github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.6 // indirect
+	github.com/haxii/fastproxy v0.5.37 // indirect
 	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect
 	github.com/invopop/yaml v0.2.0 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
@@ -198,11 +212,13 @@ require (
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/projectdiscovery/asnmap v1.1.0 // indirect
 	github.com/projectdiscovery/cdncheck v1.0.9 // indirect
-	github.com/projectdiscovery/freeport v0.0.5 // indirect
 	github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
 	github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
+	github.com/projectdiscovery/roundrobin v0.0.6 // indirect
 	github.com/projectdiscovery/stringsutil v0.0.2 // indirect
+	github.com/projectdiscovery/tinydns v0.0.31 // indirect
 	github.com/quic-go/quic-go v0.42.0 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/refraction-networking/utls v1.6.1 // indirect
 	github.com/sashabaranov/go-openai v1.15.3 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
@@ -232,6 +248,7 @@ require (
 	github.com/zeebo/blake3 v0.2.3 // indirect
 	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
 	mellium.im/sasl v0.3.1 // indirect
 )
@@ -300,10 +317,10 @@ require (
 	go.etcd.io/bbolt v1.3.8 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	goftp.io/server/v2 v2.0.1 // indirect
-	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
 	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.17.0
 	google.golang.org/appengine v1.6.7 // indirect
@@ -327,6 +344,7 @@ require (
 	github.com/aws/smithy-go v1.13.5 // indirect
 	github.com/dop251/goja_nodejs v0.0.0-20230821135201-94e508132562
 	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/go-echarts/go-echarts/v2 v2.3.3
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
@@ -346,3 +364,5 @@ require (
 
 // https://go.dev/ref/mod#go-mod-file-retract
 retract v3.2.0 // retract due to broken js protocol issue
+
+replace github.com/go-echarts/go-echarts/v2 => github.com/tarunKoyalwar/go-echarts/v2 v2.1.1

go.sum

@@ -85,7 +85,12 @@ github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAc
 github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
 github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
+github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A=
+github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g=
+github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc=
+github.com/Shopify/toxiproxy/v2 v2.5.0/go.mod h1:yhM2epWtAmel9CB8r2+L+PCmhH6yH2pITaPAo7jxJl0=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
@@ -296,10 +301,19 @@ github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj6
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
+github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
+github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0=
+github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 h1:8yY/I9ndfrgrXUbOGObLHKBR4Fl3nZXwM2c7OYTT8hM=
+github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
+github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
+github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
@@ -310,10 +324,12 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
-github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
@@ -404,6 +420,8 @@ github.com/gobwas/ws v1.2.1 h1:F2aeBZrm2NDsc7vbovKrWSogd4wvfAxg0FQ89/iqOTk=
 github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I=
+github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
 github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -523,6 +541,8 @@ github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy
 github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
 github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
@@ -531,6 +551,8 @@ github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrj
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
 github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
@@ -554,6 +576,9 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/haxii/fastproxy v0.5.37 h1:grfso8V9sNO8jZjI/vkizFXzz8fE/yrsgl+XxTb3fio=
+github.com/haxii/fastproxy v0.5.37/go.mod h1:VFy3M4EmTbeKu+IccQ6UiJ1W4sPeQ75GV+1JDa8h864=
+github.com/haxii/log v1.0.0/go.mod h1:y9MlOm+u2ny65yQxScWfSGZFOhRVLXz2vJlkiIx2jfI=
 github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk=
 github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA=
 github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A=
@@ -829,18 +854,18 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO
 github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs=
 github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4=
 github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
-github.com/projectdiscovery/dsl v0.0.50 h1:4SuAwTS9l6o1tqlIC/79+EcUwTM6CjaU7MpY/nDlFaM=
-github.com/projectdiscovery/dsl v0.0.50/go.mod h1:6g740l4tH4d2j9UYtIchtxudb0Dphkq4o+VatpR4M6g=
-github.com/projectdiscovery/fastdialer v0.0.64 h1:xivkA4g14nwQElOVsxPkGMWsdcYPcp7DPhVjvI6yQkw=
-github.com/projectdiscovery/fastdialer v0.0.64/go.mod h1:S/7PAQRmVDYRaU7u4xXD0qA5a48NAZq2JcpcVoEVrlo=
+github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE=
+github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI=
+github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E=
+github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
 github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
-github.com/projectdiscovery/goflags v0.1.46 h1:JlYvFxJcimKJGWYbygiFBN052MWrbls/kKiwOKpLzEE=
-github.com/projectdiscovery/goflags v0.1.46/go.mod h1:X7A6ELNgczyOyEy2gyNC/tJTuhtwQk6ZLyzsnDVlZkw=
+github.com/projectdiscovery/goflags v0.1.48 h1:iJgq+YuQOEm0V7BqlV/0wx7GaI2rmGiFzPkvL/4sCj0=
+github.com/projectdiscovery/goflags v0.1.48/go.mod h1:C9Qqyivehk5iYCf3VAC3+85y60Qbde1ZBMqe5wHhcVw=
 github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A=
 github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
 github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
@@ -859,32 +884,40 @@ github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScL
 github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
 github.com/projectdiscovery/mapcidr v1.1.16 h1:rjj1w5D6hbTsUQXYClLcGdfBEy9bryclgi70t0vBggo=
 github.com/projectdiscovery/mapcidr v1.1.16/go.mod h1:rGqpBhStdwOQ2uS62QM9qPsybwMwIhT7CTd2bxoHs8Q=
+github.com/projectdiscovery/martian/v3 v3.0.0-20240219194442-fed3b744f477 h1:VJaBELAC5Hw+kc+ylrFF5nSf7Wasnb9mZxMlF/kR3gg=
+github.com/projectdiscovery/martian/v3 v3.0.0-20240219194442-fed3b744f477/go.mod h1:wPvVUl2C/XOFacugXwsUp65GN0upmKfwKMyimA/AAaM=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
 github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8=
 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
+github.com/projectdiscovery/proxify v0.0.13-0.20240407134611-de309e8af9ca h1:Adduz+Lli837Ga2K1NqxVFOFxRYsWjK4zRWzHQWZtV4=
+github.com/projectdiscovery/proxify v0.0.13-0.20240407134611-de309e8af9ca/go.mod h1:9x1db4kybvc6BpSvpvbMzAuWjNFrt6Foy1t1VI7CUTc=
 github.com/projectdiscovery/ratelimit v0.0.35 h1:epEzFATOcXZ4tssV4Hax5Op9lrbUnQMEGMV5PoUpTKc=
 github.com/projectdiscovery/ratelimit v0.0.35/go.mod h1:mPqa8UpV5I7eAN5/ZcsjLiXMhjtVvZRrHtpBRsTPuyA=
-github.com/projectdiscovery/rawhttp v0.1.41 h1:0n6CohOf0Aq7dsXv+ozznhlYr4ANDKLwvPmdzTet3qU=
-github.com/projectdiscovery/rawhttp v0.1.41/go.mod h1:TyVfCwNbAsQSwrMOKu8o1g80AO3t1OnlJx+flgSV/CQ=
+github.com/projectdiscovery/rawhttp v0.1.45 h1:5jssUybhRz2ljI3bW4kUO9MTvZ37ArG8pISYtgqzwBg=
+github.com/projectdiscovery/rawhttp v0.1.45/go.mod h1:oLWuZkyKi4rddPoLEN3TeAqyLvFNfr3dxmzV2kwpPvs=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
 github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE=
 github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY=
-github.com/projectdiscovery/retryablehttp-go v1.0.54 h1:lUmQA3obq3Ya3xU1vouKf+hVjbLFKzJCK6FcNKPZ8vQ=
-github.com/projectdiscovery/retryablehttp-go v1.0.54/go.mod h1:J+pg00bYLEgWOZJISi16icHUDbsnkjnA1PmSa2kSMYs=
+github.com/projectdiscovery/retryablehttp-go v1.0.55 h1:ADgugnl9jKkNXn5m/Zd8TGPq1P7GplYlqUNKm/qTmls=
+github.com/projectdiscovery/retryablehttp-go v1.0.55/go.mod h1:Kpvh4ruFPOEPYaYxgbFmlvBJr4lJKqpcbGvx1j0r/Ng=
+github.com/projectdiscovery/roundrobin v0.0.6 h1:zoJAFRgP9XK7B+iKSjR+djRAuDYxnc57+Fx+qpoPvds=
+github.com/projectdiscovery/roundrobin v0.0.6/go.mod h1:vTxcWqNLyMH6VE2Q/hsNNvDHFLiIzHozC1rLLT/vocQ=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
 github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
 github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
+github.com/projectdiscovery/tinydns v0.0.31 h1:iQL1ze2/4PHWH+Nzsxg35kVIWkvEIatAZCDcP0c1/v4=
+github.com/projectdiscovery/tinydns v0.0.31/go.mod h1:6rj/dEzXZPn+nLl4O1wUMJdfkdUQ9eCV7NsDWPakYv8=
 github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A=
 github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
 github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A=
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
 github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo=
 github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4=
-github.com/projectdiscovery/utils v0.0.87 h1:9+RiTEhpUB/vk6XJUVpysNWJ2aCTD7WuyoyAcNnbIzk=
-github.com/projectdiscovery/utils v0.0.87/go.mod h1:jGK450sL9AVDTjaPwEs9za8NVeEC9xE97IWNoK138kI=
+github.com/projectdiscovery/utils v0.0.88 h1:oYfCXM+8VHNLyH/H6cOibkuDUwHUAOBAMRNPFX6NPrs=
+github.com/projectdiscovery/utils v0.0.88/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA=
 github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8=
 github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@@ -924,6 +957,8 @@ github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB
 github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM=
 github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
+github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY=
 github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c=
 github.com/refraction-networking/utls v1.6.1 h1:n1JG5karzdGWsI6iZmGrOv3SNzR4c+4M8J6KWGsk3lA=
@@ -941,6 +976,7 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.11.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
@@ -1010,6 +1046,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/tarunKoyalwar/go-echarts/v2 v2.1.1 h1:5fsXGPmK+i18J8cDgxy7AJkiXWBARpVTb0Gbv+bAzPo=
+github.com/tarunKoyalwar/go-echarts/v2 v2.1.1/go.mod h1:VEeyPT5Odx/UHeuxtIAHGu2+87MWGA5OBaZ120NFi/w=
 github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
 github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
 github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
@@ -1183,8 +1221,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
 golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1277,8 +1315,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1379,8 +1417,9 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1392,8 +1431,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
-golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1473,6 +1512,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
@@ -1529,6 +1570,8 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20220706185917-7780775163c4 h1:7YDGQC/0sigNGzsEWyb9s72jTxlFdwVEYNJHbfQ+Dtg=
+google.golang.org/genproto v0.0.0-20220706185917-7780775163c4/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
@@ -1546,6 +1589,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w=
+google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

integration_tests/protocols/code/pre-condition.yaml

@@ -23,4 +23,4 @@ code:
       - type: dsl
         dsl:
           - true
-# digest: 4a0a00473045022100c7215ce9f11e6a51c193bb54643a05cdd1cde18a3abb6c9983c5c7524d3ff03002203d93581c81d3ad5db463570cbbd2bdee529328d32a5b00e037610c211e448cef:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 490a004630440220192fb8f704b078c2885047b85ac1a0491be86485c033a976d201599683a35aab0220604b1c3781e9d97079d0e5c23c18e6a2d87493c8e2b930536e692ee7d06e9247:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-env-var.yaml

@@ -20,4 +20,4 @@ code:
       - type: word
         words:
           - "hello from input baz"
-# digest: 4a0a0047304502207e3a5eda5f3207c3c01c820562243281926c1215224a7c80ed7528559b9f52cb022100f6ef99bb45843f481705778630f2cfd8f4d1cc3acb96392ff016f22e06aa91af:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a00473045022033f72f1b9d5143f58a2dc79c2597000f34080251ac3702c36c3fad00917dfeeb0221009ba05c715c9e2e36dba471be6c0106a09ae3822d8a3e9e4bcf377e9f4a395a01:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-file.yaml

@@ -18,4 +18,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4a0a004730450220069673af9bd6d6677f9529d06f5d8bd46d543089a4731ed18ee806761d75fd60022100913a3e27b0a5809baf710ba9585bf9fe729634c0e19e3e13eef70a6bd100df34:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a004730450220377128cb11d9f6f0fee1f4dbd841e46783de26e90a216fa55a7609ee2bc823c60221009166ee0f85e3a1811588ab19e73ea96ab3d582dc8180dbcbbad0ea9ab7e9025d:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-interactsh.yaml

@@ -26,4 +26,4 @@ code:
         part: interactsh_protocol
         words:
           - "http"
-# digest: 490a00463044022003b8d069e3c84412729c43e33013a52ee04eabcf096d511979691d71d8e905f60220011f4475899abed4f86b4bd5e6c2423750759135206e4729826afe1ed8a44f4d:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4b0a00483046022100d472d50bd83117d334f5217c7a40dcdf34138e90029eaace51697d902296bf37022100a393b49420a96f60d6d89b79b5135ee2233b2468d374851890eea114b08195d1:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-snippet.yaml

@@ -21,4 +21,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4a0a00473045022100c291615cf2a8005450c17a6554e81a9cdab14743b299f0679c644247929198b502206fdacc8ab173bde2b4015340012637916bf2659f66f320fcc06b97ac639072a1:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4b0a004830460221008886054bb5dd6345e434e30f31c8fddce3c484a4f33aa6321b5185675866029d022100d188a83d0fde029f8b586061c65ab72b43755c3fb10fdd59501bb9bbadbb1ff7:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/dns/a.yaml

@@ -1,7 +1,7 @@
-id: basic-dns-example
+id: dns-a-query-example
 
 info:
-  name: Test DNS Template
+  name: Test DNS A Query Template
   author: pdteam
   severity: info
 
@@ -14,4 +14,4 @@ dns:
     matchers:
       - type: word
         words:
-          - "1.1.1.1"
+          - "1.1.1.1"

integration_tests/protocols/dns/aaaa.yaml

@@ -0,0 +1,17 @@
+id: dns-aaaa-query-example
+
+info:
+  name: Test DNS AAAA Query Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: AAAA
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        words:
+          - "2606:4700:4700::1001"

integration_tests/protocols/dns/cname.yaml

@@ -0,0 +1,18 @@
+id: dns-cname-query-example
+
+info:
+  name: Test DNS CNAME Query Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: CNAME
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        part: all
+        words:
+          - "CNAME"

integration_tests/protocols/dns/ns.yaml

@@ -0,0 +1,18 @@
+id: dns-ns-query-example
+
+info:
+  name: Test DNS NS Query Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: NS
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        part: all
+        words:
+          - "NS"

integration_tests/protocols/dns/srv.yaml

@@ -0,0 +1,18 @@
+id: dns-a-query-example
+
+info:
+  name: Test DNS SRV Query Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: SRV
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        part: all
+        words:
+          - "SRV"

integration_tests/protocols/dns/txt.yaml

@@ -0,0 +1,18 @@
+id: dns-txt-query-example
+
+info:
+  name: Test DNS TXT Query Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: TXT
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        part: all
+        words:
+          - "TXT"

integration_tests/workflow/code-template-1.yaml

@@ -0,0 +1,22 @@
+id: code-template-1
+
+info:
+  name: code-template-1
+  author: tovask
+  severity: info
+  tags: code
+
+code:
+  - engine:
+      - py
+      - python3
+      - python
+    source: |
+      print("hello from first")
+    extractors:
+      - type: regex
+        name: extracted
+        regex:
+          - 'hello from (.*)'
+        group: 1
+# digest: 490a0046304402202c63d47bb0acdd40b3b852d95490d492ff5741b84071b2a8a40371be7797c13602202b6b977e157edf2ef70a402a2e57d4eb5a67c5ca91f0a2f9a10a966e8485ebaf:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/workflow/code-template-2.yaml

@@ -0,0 +1,21 @@
+id: code-template-2
+
+info:
+  name: code-template-2
+  author: tovask
+  severity: info
+  tags: code
+
+code:
+  - engine:
+      - py
+      - python3
+      - python
+    source: |
+      import os
+      print("hello from " + os.getenv("extracted"))
+    matchers:
+      - type: word
+        words:
+          - "hello from first"
+# digest: 490a00463044022025661eab353b7f359c0d428a86b6287545d7f759375e8025cc8c9c77b616ca6502200bc2c019059622df3c88e7caa6dd7d1fb9b956010aa0de2ee2b9f7dd0a3c4954:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/workflow/code-value-share-workflow.yaml

@@ -0,0 +1,12 @@
+id: code-value-sharing-workflow
+
+info:
+  name: Code Value Sharing Workflow
+  author: tovask
+  severity: info
+  tags: code
+
+workflows:
+  - template: workflow/code-template-1.yaml
+    subtemplates:
+      - template: workflow/code-template-2.yaml

integration_tests/workflow/complex-conditions.yaml

@@ -0,0 +1,23 @@
+id: complex-conditions-workflow
+
+info:
+  name: Complex Conditions Workflow
+  author: tovask
+  severity: info
+  description: Workflow to test a complex scenario, e.g. race conditions when evaluating the results of the templates
+
+workflows:
+  - template: workflow/match-1.yaml
+    subtemplates:
+      - template: workflow/nomatch-1.yaml
+        subtemplates:
+          - template: workflow/match-2.yaml
+  - template: workflow/match-3.yaml
+  - template: workflow/match-2.yaml
+    matchers: 
+      - name: test-matcher
+        subtemplates:
+          - template: workflow/nomatch-1.yaml
+            subtemplates:
+              - template: workflow/match-1.yaml
+  - template: workflow/match-3.yaml

integration_tests/workflow/match-3.yaml

@@ -0,0 +1,16 @@
+id: basic-get-third
+
+info:
+  name: Basic 3rd GET Request
+  author: tovask
+  severity: info
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        name: test-matcher-3
+        words:
+          - "This is test matcher text"

integration_tests/workflow/multiprotocol-value-share-template.yaml

@@ -0,0 +1,22 @@
+id: multiprotocol-value-sharing-template
+
+info:
+  name: MultiProtocol Value Sharing Template
+  author: tovask
+  severity: info
+
+dns:
+  - name: "{{extracted}}"
+    type: PTR
+    matchers:
+      - type: word
+        words:
+          - "blog.projectdiscovery.io"
+
+http:
+  - path:
+      - "{{BaseURL}}/path2?extracted={{extracted}}"
+    matchers:
+      - type: word
+        words:
+          - "blog.projectdiscovery.io"

integration_tests/workflow/multiprotocol-value-share-workflow.yaml

@@ -0,0 +1,11 @@
+id: multiprotocol-value-sharing-workflow
+
+info:
+  name: MultiProtocol Value Sharing Workflow
+  author: tovask
+  severity: info
+
+workflows:
+  - template: workflow/http-value-share-template-1.yaml
+    subtemplates:
+      - template: workflow/multiprotocol-value-share-template.yaml

internal/runner/inputs.go

@@ -10,17 +10,17 @@ import (
 	"github.com/projectdiscovery/httpx/common/httpx"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	stringsutil "github.com/projectdiscovery/utils/strings"
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
-const probeBulkSize = 50
+var GlobalProbeBulkSize = 50
 
 // initializeTemplatesHTTPInput initializes the http form of input
 // for any loaded http templates if input is in non-standard format.
 func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
-
 	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create temporary input file")
@@ -31,33 +31,43 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	}
 	gologger.Info().Msgf("Running httpx on input host")
 
-	var bulkSize = probeBulkSize
-	if r.options.BulkSize > probeBulkSize {
+	var bulkSize = GlobalProbeBulkSize
+	if r.options.BulkSize > GlobalProbeBulkSize {
 		bulkSize = r.options.BulkSize
 	}
 
 	httpxOptions := httpx.DefaultOptions
 	httpxOptions.RetryMax = r.options.Retries
 	httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
+	httpxOptions.NetworkPolicy = protocolstate.NetworkPolicy
 	httpxClient, err := httpx.New(&httpxOptions)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create httpx client")
 	}
 
+	shouldFollowGlobalProbeBulkSize := bulkSize == GlobalProbeBulkSize
+
 	// Probe the non-standard URLs and store them in cache
-	swg := sizedwaitgroup.New(bulkSize)
-	count := int32(0)
+	swg, err := syncutil.New(syncutil.WithSize(bulkSize))
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create adaptive group")
+	}
+	var count atomic.Int32
 	r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool {
 		if stringsutil.HasPrefixAny(value.Input, "http://", "https://") {
 			return true
 		}
 
+		if shouldFollowGlobalProbeBulkSize && swg.Size != GlobalProbeBulkSize {
+			swg.Resize(GlobalProbeBulkSize)
+		}
+
 		swg.Add()
 		go func(input *contextargs.MetaInput) {
 			defer swg.Done()
 
 			if result := utils.ProbeURL(input.Input, httpxClient); result != "" {
-				atomic.AddInt32(&count, 1)
+				count.Add(1)
 				_ = hm.Set(input.Input, []byte(result))
 			}
 		}(value)
@@ -65,6 +75,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	})
 	swg.Wait()
 
-	gologger.Info().Msgf("Found %d URL from httpx", atomic.LoadInt32(&count))
+	gologger.Info().Msgf("Found %d URL from httpx", count.Load())
 	return hm, nil
 }

internal/runner/lazy.go

@@ -21,7 +21,7 @@ type AuthLazyFetchOptions struct {
 	OnError       func(error)
 }
 
-// GetAuthTemlStore create new loader for loading auth templates
+// GetAuthTmplStore create new loader for loading auth templates
 func GetAuthTmplStore(opts types.Options, catalog catalog.Catalog, execOpts protocols.ExecutorOptions) (*loader.Store, error) {
 	tmpls := []string{}
 	for _, file := range opts.SecretsFile {

internal/runner/options.go

@@ -25,6 +25,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
+	"github.com/projectdiscovery/nuclei/v3/pkg/testing"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
 	fileutil "github.com/projectdiscovery/utils/file"
@@ -50,6 +51,8 @@ func ConfigureOptions() error {
 	return nil
 }
 
+var proxyServer *testing.ProxyServer
+
 // ParseOptions parses the command line flags provided by a user
 func ParseOptions(options *types.Options) {
 	// Check if stdin pipe was given
@@ -83,6 +86,19 @@ func ParseOptions(options *types.Options) {
 	if err := ValidateOptions(options); err != nil {
 		gologger.Fatal().Msgf("Program exiting: %s\n", err)
 	}
+	if options.AutogenerateTests {
+		ps, err := testing.NewProxyServer()
+		if err != nil {
+			gologger.Fatal().Msgf("Could not create proxy server: %s\n", err)
+		}
+		proxyServer = ps
+
+		// Add the proxy server to the options
+		proxyURL := fmt.Sprintf("http://%s", proxyServer.ListenAddr)
+		options.Proxy = goflags.StringSlice([]string{proxyURL})
+		types.ProxyURL = proxyURL
+		gologger.Info().Msgf("Using %s as proxy server for automatic test-generation\n", proxyServer.ListenAddr)
+	}
 
 	// Load the resolvers if user asked for them
 	loadResolvers(options)

internal/runner/runner.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/http/httptest"
 	_ "net/http/pprof"
 	"os"
 	"path/filepath"
@@ -18,6 +19,8 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/loader/parser"
+	"github.com/projectdiscovery/nuclei/v3/pkg/testing"
+	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
 	uncoverlib "github.com/projectdiscovery/uncover"
 	pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
 	"github.com/projectdiscovery/utils/env"
@@ -314,11 +317,17 @@ func New(options *types.Options) (*Runner, error) {
 	}
 
 	if options.RateLimitMinute > 0 {
-		runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimitMinute), time.Minute)
-	} else if options.RateLimit > 0 {
-		runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimit), time.Second)
-	} else {
+		gologger.Print().Msgf("[%v] %v", aurora.BrightYellow("WRN"), "rate limit per minute is deprecated - use rate-limit-duration")
+		options.RateLimit = options.RateLimitMinute
+		options.RateLimitDuration = time.Minute
+	}
+	if options.RateLimit > 0 && options.RateLimitDuration == 0 {
+		options.RateLimitDuration = time.Second
+	}
+	if options.RateLimit == 0 && options.RateLimitDuration == 0 {
 		runner.rateLimiter = ratelimit.NewUnlimited(context.Background())
+	} else {
+		runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimit), options.RateLimitDuration)
 	}
 
 	if tmpDir, err := os.MkdirTemp("", "nuclei-tmp-*"); err == nil {
@@ -338,6 +347,9 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecutorOptions,
 
 // Close releases all the resources and cleans up
 func (r *Runner) Close() {
+	if proxyServer != nil {
+		proxyServer.Close()
+	}
 	if r.output != nil {
 		r.output.Close()
 	}
@@ -501,6 +513,14 @@ func (r *Runner) RunEnumeration() error {
 		return nil // exit
 	}
 	store.Load()
+
+	if len(store.Templates()) > 1 && r.options.AutogenerateTests {
+		return errors.New("autogenerate-tests flag can only be used with a single template")
+	}
+	if r.inputProvider.Count() > 1 && r.options.AutogenerateTests {
+		return errors.New("autogenerate-tests flag can only be used with a single input")
+	}
+
 	// TODO: remove below functions after v3 or update warning messages
 	disk.PrintDeprecatedPathsMsgIfApplicable(r.options.Silent)
 	templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
@@ -528,6 +548,17 @@ func (r *Runner) RunEnumeration() error {
 	// display execution info like version , templates used etc
 	r.displayExecutionInfo(store)
 
+	// If prompted, we need to run the tests for all the loaded
+	// templates by first trying to detect test definitions and then
+	// executing them with the relevant inputs.
+	if r.options.RunNucleiTests {
+		gologger.Info().Msgf("Running nuclei-templates mock-generated tests")
+		if err := r.runNucleiTests(store, executorEngine); err != nil {
+			return errors.Wrap(err, "could not run nuclei tests")
+		}
+		return nil
+	}
+
 	// prefetch secrets if enabled
 	if executorOpts.AuthProvider != nil && r.options.PreFetchSecrets {
 		gologger.Info().Msgf("Pre-fetching secrets from authprovider[s]")
@@ -547,6 +578,20 @@ func (r *Runner) RunEnumeration() error {
 		executorOpts.InputHelper.InputsHTTP = inputHelpers
 	}
 
+	// initialize stats worker ( this is no-op unless nuclei is built with stats build tag)
+	// during execution a directory with 2 files will be created in the current directory
+	// config.json - containing below info
+	// events.jsonl - containing all start and end times of all templates
+	events.InitWithConfig(&events.ScanConfig{
+		Name:                "nuclei-stats", // make this configurable
+		TargetCount:         int(r.inputProvider.Count()),
+		TemplatesCount:      len(store.Templates()) + len(store.Workflows()),
+		TemplateConcurrency: r.options.TemplateThreads,
+		PayloadConcurrency:  r.options.PayloadConcurrency,
+		JsConcurrency:       r.options.JsConcurrency,
+		Retries:             r.options.Retries,
+	}, "")
+
 	enumeration := false
 	var results *atomic.Bool
 	results, err = r.runStandardEnumeration(executorOpts, store, executorEngine)
@@ -556,10 +601,17 @@ func (r *Runner) RunEnumeration() error {
 		return err
 	}
 
+	var isInteractMatcher bool
 	if r.interactsh != nil {
 		matched := r.interactsh.Close()
 		if matched {
 			results.CompareAndSwap(false, true)
+			isInteractMatcher = true
+		}
+	}
+	if r.options.AutogenerateTests && proxyServer != nil {
+		if err := r.handleTestAutogenerate(results, isInteractMatcher, store); err != nil {
+			return err
 		}
 	}
 	if executorOpts.InputHelper != nil {
@@ -579,6 +631,108 @@ func (r *Runner) RunEnumeration() error {
 	return err
 }
 
+func (r *Runner) runNucleiTests(store *loader.Store, engine *core.Engine) error {
+	// For each loaded template, check to see if there's
+	// a corresponding test file for that template with extension
+	// .nuclei_test
+	testFiles := make(map[string]string)
+	templatePathToTemplates := make(map[string]*templates.Template)
+
+	for _, template := range store.Templates() {
+		testFile := fmt.Sprintf("%s.nuclei_test", template.Path)
+		if !fileutil.FileExists(testFile) {
+			continue
+		}
+		testFiles[template.Path] = testFile
+		templatePathToTemplates[template.Path] = template
+
+		gologger.Verbose().Msgf("Found test file for template %s: %s", template.ID, filepath.Base(testFile))
+	}
+
+	if len(testFiles) == 0 {
+		gologger.Info().Msgf("No test files found for nuclei-templates")
+		return nil
+	}
+
+	gologger.Info().Msgf("Loaded %d test files for nuclei-templates", len(testFiles))
+
+	for templatePath, testFile := range testFiles {
+		testTemplate, err := testing.ReadNucleiTestTemplate(testFile)
+		if err != nil {
+			return errors.Wrap(err, "could not read nuclei test template")
+		}
+
+		mockHandler, err := testTemplate.MockServer()
+		if err != nil {
+			return errors.Wrap(err, "could not create mock handler for template")
+		}
+
+		testServer := httptest.NewServer(mockHandler)
+		template, ok := templatePathToTemplates[templatePath]
+		if !ok {
+			testServer.Close()
+			return errors.Errorf("no template found for path %s", templatePath)
+		}
+
+		// If we have interactsh matcher, we need to handle it separately
+		// by creating a new interactsh client and closing it after the test
+		// waiting for callbacks.
+		// TODO: Parallelize
+		if testTemplate.IsInteractshMatcher {
+			interactClient, err := interactsh.New(interactsh.DefaultOptions(r.output, r.issuesClient, r.progress))
+			if err != nil {
+				testServer.Close()
+				return errors.Wrap(err, "could not create interactsh client")
+			}
+			template.Options.Interactsh = interactClient
+		}
+
+		results := engine.ExecuteScanWithOpts([]*templates.Template{template}, provider.NewSimpleInputProviderWithUrls(testServer.URL), r.options.DisableClustering)
+		if !results.Load() && !testTemplate.IsInteractshMatcher {
+			testServer.Close()
+			return errors.Errorf("no results found for template %s", template.ID)
+		}
+
+		if testTemplate.IsInteractshMatcher {
+			matched := template.Options.Interactsh.Close()
+			if !matched {
+				testServer.Close()
+				return errors.Errorf("no results found for template %s", template.ID)
+			}
+		}
+	}
+	gologger.Info().Msgf("All tests passed successfully")
+
+	return nil
+}
+
+func (r *Runner) handleTestAutogenerate(results *atomic.Bool, isInteractMatcher bool, store *loader.Store) error {
+	if !results.Load() {
+		return errors.New("no results found to autogenerate tests")
+	}
+
+	intercepted := proxyServer.Intercepted()
+	if len(intercepted) == 0 {
+		return errors.New("no intercepted requests")
+	}
+	gologger.Verbose().Msgf("Intercepted requests and responses (%d)", len(intercepted))
+	for _, pair := range intercepted {
+		if r.options.Debug {
+			fmt.Printf("Pair: %+v\n", pair)
+		}
+	}
+	var gotInput string
+	r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool {
+		gotInput = value.Input
+		return false
+	})
+	err := testing.GenerateTestsFromPair(intercepted, store.Templates()[0], gotInput, isInteractMatcher)
+	if err != nil {
+		return errors.Wrap(err, "could not generate tests from intercepted requests")
+	}
+	return nil
+}
+
 func (r *Runner) isInputNonHTTP() bool {
 	var nonURLInput bool
 	r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool {

lib/config.go

@@ -2,6 +2,7 @@ package nuclei
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	"github.com/projectdiscovery/goflags"
@@ -115,17 +116,48 @@ type Concurrency struct {
 	HeadlessTemplateConcurrency   int // number of templates to run concurrently for headless templates (per host in host-spray mode)
 	JavascriptTemplateConcurrency int // number of templates to run concurrently for javascript templates (per host in host-spray mode)
 	TemplatePayloadConcurrency    int // max concurrent payloads to run for a template (a good default is 25)
+	ProbeConcurrency              int // max concurrent http probes to run (a good default is 50)
 }
 
 // WithConcurrency sets concurrency options
 func WithConcurrency(opts Concurrency) NucleiSDKOptions {
 	return func(e *NucleiEngine) error {
-		e.opts.TemplateThreads = opts.TemplateConcurrency
-		e.opts.BulkSize = opts.HostConcurrency
-		e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency
-		e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency
-		e.opts.JsConcurrency = opts.JavascriptTemplateConcurrency
-		e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency
+		// minimum required is 1
+		if opts.TemplateConcurrency <= 0 {
+			return errors.New("template threads must be at least 1")
+		} else {
+			e.opts.TemplateThreads = opts.TemplateConcurrency
+		}
+		if opts.HostConcurrency <= 0 {
+			return errors.New("host concurrency must be at least 1")
+		} else {
+			e.opts.BulkSize = opts.HostConcurrency
+		}
+		if opts.HeadlessHostConcurrency <= 0 {
+			return errors.New("headless host concurrency must be at least 1")
+		} else {
+			e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency
+		}
+		if opts.HeadlessTemplateConcurrency <= 0 {
+			return errors.New("headless template threads must be at least 1")
+		} else {
+			e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency
+		}
+		if opts.JavascriptTemplateConcurrency <= 0 {
+			return errors.New("js must be at least 1")
+		} else {
+			e.opts.JsConcurrency = opts.JavascriptTemplateConcurrency
+		}
+		if opts.TemplatePayloadConcurrency <= 0 {
+			return errors.New("payload concurrency must be at least 1")
+		} else {
+			e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency
+		}
+		if opts.ProbeConcurrency <= 0 {
+			return errors.New("probe concurrency must be at least 1")
+		} else {
+			e.opts.ProbeConcurrency = opts.ProbeConcurrency
+		}
 		return nil
 	}
 }
@@ -133,7 +165,9 @@ func WithConcurrency(opts Concurrency) NucleiSDKOptions {
 // WithGlobalRateLimit sets global rate (i.e all hosts combined) limit options
 func WithGlobalRateLimit(maxTokens int, duration time.Duration) NucleiSDKOptions {
 	return func(e *NucleiEngine) error {
-		e.rateLimiter = ratelimit.New(context.Background(), uint(maxTokens), duration)
+		e.opts.RateLimit = maxTokens
+		e.opts.RateLimitDuration = duration
+		e.rateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), e.opts.RateLimitDuration)
 		return nil
 	}
 }
@@ -280,7 +314,7 @@ func WithScanStrategy(strategy string) NucleiSDKOptions {
 // OutputWriter
 type OutputWriter output.Writer
 
-// UseWriter allows setting custom output writer
+// UseOutputWriter allows setting custom output writer
 // by default a mock writer is used with user defined callback
 // if outputWriter is used callback will be ignored
 func UseOutputWriter(writer OutputWriter) NucleiSDKOptions {

lib/multi.go

@@ -42,11 +42,16 @@ func createEphemeralObjects(base *NucleiEngine, opts *types.Options) (*unsafeOpt
 		Parser:          base.parser,
 	}
 	if opts.RateLimitMinute > 0 {
-		u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimitMinute), time.Minute)
-	} else if opts.RateLimit > 0 {
-		u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimit), time.Second)
-	} else {
+		opts.RateLimit = opts.RateLimitMinute
+		opts.RateLimitDuration = time.Minute
+	}
+	if opts.RateLimit > 0 && opts.RateLimitDuration == 0 {
+		opts.RateLimitDuration = time.Second
+	}
+	if opts.RateLimit == 0 && opts.RateLimitDuration == 0 {
 		u.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
+	} else {
+		u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimit), opts.RateLimitDuration)
 	}
 	u.engine = core.New(opts)
 	u.engine.SetExecuterOptions(u.executerOpts)

lib/sdk.go

@@ -215,6 +215,14 @@ func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.Result
 	return nil
 }
 
+func (e *NucleiEngine) Options() *types.Options {
+	return e.opts
+}
+
+func (e *NucleiEngine) Engine() *core.Engine {
+	return e.engine
+}
+
 // NewNucleiEngine creates a new nuclei engine instance
 func NewNucleiEngine(options ...NucleiSDKOptions) (*NucleiEngine, error) {
 	// default options

lib/sdk_private.go

@@ -192,11 +192,16 @@ func (e *NucleiEngine) init() error {
 
 	if e.executerOpts.RateLimiter == nil {
 		if e.opts.RateLimitMinute > 0 {
-			e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimitMinute), time.Minute)
-		} else if e.opts.RateLimit > 0 {
-			e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), time.Second)
-		} else {
+			e.opts.RateLimit = e.opts.RateLimitMinute
+			e.opts.RateLimitDuration = time.Minute
+		}
+		if e.opts.RateLimit > 0 && e.opts.RateLimitDuration == 0 {
+			e.opts.RateLimitDuration = time.Second
+		}
+		if e.opts.RateLimit == 0 && e.opts.RateLimitDuration == 0 {
 			e.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
+		} else {
+			e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), e.opts.RateLimitDuration)
 		}
 	}
 

pkg/catalog/config/constants.go

@@ -31,7 +31,7 @@ const (
 	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
-	Version = `v3.2.3`
+	Version = `v3.2.4`
 	// Directory Names of custom templates
 	CustomS3TemplatesDirName     = "s3"
 	CustomGitHubTemplatesDirName = "github"

pkg/core/engine.go

@@ -30,14 +30,19 @@ func New(options *types.Options) *Engine {
 	return engine
 }
 
-// GetWorkPool returns a workpool from options
-func (e *Engine) GetWorkPool() *WorkPool {
-	return NewWorkPool(WorkPoolConfig{
+func (e *Engine) GetWorkPoolConfig() WorkPoolConfig {
+	config := WorkPoolConfig{
 		InputConcurrency:         e.options.BulkSize,
 		TypeConcurrency:          e.options.TemplateThreads,
 		HeadlessInputConcurrency: e.options.HeadlessBulkSize,
 		HeadlessTypeConcurrency:  e.options.HeadlessTemplateThreads,
-	})
+	}
+	return config
+}
+
+// GetWorkPool returns a workpool from options
+func (e *Engine) GetWorkPool() *WorkPool {
+	return NewWorkPool(e.GetWorkPoolConfig())
 }
 
 // SetExecuterOptions sets the executer options for the engine. This is required
@@ -53,5 +58,7 @@ func (e *Engine) ExecuterOptions() protocols.ExecutorOptions {
 
 // WorkPool returns the worker pool for the engine
 func (e *Engine) WorkPool() *WorkPool {
+	// resize check point - nop if there are no changes
+	e.workPool.RefreshWithConfig(e.GetWorkPoolConfig())
 	return e.workPool
 }

pkg/core/execute_options.go

@@ -4,8 +4,6 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/remeh/sizedwaitgroup"
-
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
@@ -14,6 +12,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
 	stringsutil "github.com/projectdiscovery/utils/strings"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 // Execute takes a list of templates/workflows that have been compiled
@@ -109,9 +108,11 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 	wp := e.GetWorkPool()
 
 	for _, template := range templatesList {
-		templateType := template.Type()
+		// resize check point - nop if there are no changes
+		wp.RefreshWithConfig(e.GetWorkPoolConfig())
 
-		var wg *sizedwaitgroup.SizedWaitGroup
+		templateType := template.Type()
+		var wg *syncutil.AdaptiveWaitGroup
 		if templateType == types.HeadlessProtocol {
 			wg = wp.Headless
 		} else {
@@ -134,7 +135,7 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 // executeHostSpray executes scan using host spray strategy where templates are iterated over each target
 func (e *Engine) executeHostSpray(templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
 	results := &atomic.Bool{}
-	wp := sizedwaitgroup.New(e.options.BulkSize + e.options.HeadlessBulkSize)
+	wp, _ := syncutil.New(syncutil.WithSize(e.options.BulkSize + e.options.HeadlessBulkSize))
 
 	target.Iterate(func(value *contextargs.MetaInput) bool {
 		wp.Add()

pkg/core/executors.go

@@ -11,7 +11,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	generalTypes "github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 // Executors are low level executors that deals with template execution on a target
@@ -104,9 +104,9 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 			return true
 		}
 
-		wg.WaitGroup.Add()
+		wg.Add()
 		go func(index uint32, skip bool, value *contextargs.MetaInput) {
-			defer wg.WaitGroup.Done()
+			defer wg.Done()
 			defer cleanupInFlight(index)
 			if skip {
 				return
@@ -140,7 +140,7 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 		index++
 		return true
 	})
-	wg.WaitGroup.Wait()
+	wg.Wait()
 
 	// on completion marks the template as completed
 	currentInfo.Lock()
@@ -158,14 +158,17 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta
 	wp := e.GetWorkPool()
 
 	for _, tpl := range alltemplates {
-		var sg *sizedwaitgroup.SizedWaitGroup
+		// resize check point - nop if there are no changes
+		wp.RefreshWithConfig(e.GetWorkPoolConfig())
+
+		var sg *syncutil.AdaptiveWaitGroup
 		if tpl.Type() == types.HeadlessProtocol {
 			sg = wp.Headless
 		} else {
 			sg = wp.Default
 		}
 		sg.Add()
-		go func(template *templates.Template, value *contextargs.MetaInput, wg *sizedwaitgroup.SizedWaitGroup) {
+		go func(template *templates.Template, value *contextargs.MetaInput, wg *syncutil.AdaptiveWaitGroup) {
 			defer wg.Done()
 
 			var match bool
@@ -213,7 +216,10 @@ func (e *ChildExecuter) Close() *atomic.Bool {
 func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) {
 	templateType := template.Type()
 
-	var wg *sizedwaitgroup.SizedWaitGroup
+	// resize check point - nop if there are no changes
+	e.e.WorkPool().RefreshWithConfig(e.e.GetWorkPoolConfig())
+
+	var wg *syncutil.AdaptiveWaitGroup
 	if templateType == types.HeadlessProtocol {
 		wg = e.e.workPool.Headless
 	} else {

pkg/core/workflow_execute.go

@@ -5,13 +5,12 @@ import (
 	"net/http/cookiejar"
 	"sync/atomic"
 
-	"github.com/remeh/sizedwaitgroup"
-
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
 	"github.com/projectdiscovery/nuclei/v3/pkg/workflows"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 const workflowStepExecutionError = "[%s] Could not execute workflow step: %s\n"
@@ -32,7 +31,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
 	if templateThreads == 1 {
 		templateThreads++
 	}
-	swg := sizedwaitgroup.New(templateThreads)
+	swg, _ := syncutil.New(syncutil.WithSize(templateThreads))
 
 	for _, template := range w.Workflows {
 		swg.Add()
@@ -40,7 +39,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
 		func(template *workflows.WorkflowTemplate) {
 			defer swg.Done()
 
-			if err := e.runWorkflowStep(template, ctx, results, &swg, w); err != nil {
+			if err := e.runWorkflowStep(template, ctx, results, swg, w); err != nil {
 				gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
 			}
 		}(template)
@@ -51,7 +50,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
 
 // runWorkflowStep runs a workflow step for the workflow. It executes the workflow
 // in a recursive manner running all subtemplates and matchers.
-func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error {
+func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *syncutil.AdaptiveWaitGroup, w *workflows.Workflow) error {
 	var firstMatched bool
 	var err error
 	var mainErr error
@@ -139,7 +138,9 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
 						go func(subtemplate *workflows.WorkflowTemplate) {
 							defer swg.Done()
 
-							if err := e.runWorkflowStep(subtemplate, ctx, results, swg, w); err != nil {
+							// create a new context with the same input but with unset callbacks
+							subCtx := scan.NewScanContext(ctx.Input)
+							if err := e.runWorkflowStep(subtemplate, subCtx, results, swg, w); err != nil {
 								gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err)
 							}
 						}(subtemplate)
@@ -163,7 +164,9 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
 			swg.Add()
 
 			go func(template *workflows.WorkflowTemplate) {
-				if err := e.runWorkflowStep(template, ctx, results, swg, w); err != nil {
+				// create a new context with the same input but with unset callbacks
+				subCtx := scan.NewScanContext(ctx.Input)
+				if err := e.runWorkflowStep(template, subCtx, results, swg, w); err != nil {
 					gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
 				}
 				swg.Done()

pkg/core/workpool.go

@@ -1,9 +1,8 @@
 package core
 
 import (
-	"github.com/remeh/sizedwaitgroup"
-
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 // WorkPool implements an execution pool for executing different
@@ -12,8 +11,8 @@ import (
 // It also allows Configuration of such requirements. This is used
 // for per-module like separate headless concurrency etc.
 type WorkPool struct {
-	Headless *sizedwaitgroup.SizedWaitGroup
-	Default  *sizedwaitgroup.SizedWaitGroup
+	Headless *syncutil.AdaptiveWaitGroup
+	Default  *syncutil.AdaptiveWaitGroup
 	config   WorkPoolConfig
 }
 
@@ -31,13 +30,13 @@ type WorkPoolConfig struct {
 
 // NewWorkPool returns a new WorkPool instance
 func NewWorkPool(config WorkPoolConfig) *WorkPool {
-	headlessWg := sizedwaitgroup.New(config.HeadlessTypeConcurrency)
-	defaultWg := sizedwaitgroup.New(config.TypeConcurrency)
+	headlessWg, _ := syncutil.New(syncutil.WithSize(config.HeadlessTypeConcurrency))
+	defaultWg, _ := syncutil.New(syncutil.WithSize(config.TypeConcurrency))
 
 	return &WorkPool{
 		config:   config,
-		Headless: &headlessWg,
-		Default:  &defaultWg,
+		Headless: headlessWg,
+		Default:  defaultWg,
 	}
 }
 
@@ -47,19 +46,39 @@ func (w *WorkPool) Wait() {
 	w.Headless.Wait()
 }
 
-// InputWorkPool is a work pool per-input
-type InputWorkPool struct {
-	WaitGroup *sizedwaitgroup.SizedWaitGroup
-}
-
 // InputPool returns a work pool for an input type
-func (w *WorkPool) InputPool(templateType types.ProtocolType) *InputWorkPool {
+func (w *WorkPool) InputPool(templateType types.ProtocolType) *syncutil.AdaptiveWaitGroup {
 	var count int
 	if templateType == types.HeadlessProtocol {
 		count = w.config.HeadlessInputConcurrency
 	} else {
 		count = w.config.InputConcurrency
 	}
-	swg := sizedwaitgroup.New(count)
-	return &InputWorkPool{WaitGroup: &swg}
+	swg, _ := syncutil.New(syncutil.WithSize(count))
+	return swg
+}
+
+func (w *WorkPool) RefreshWithConfig(config WorkPoolConfig) {
+	if w.config.TypeConcurrency != config.TypeConcurrency {
+		w.config.TypeConcurrency = config.TypeConcurrency
+	}
+	if w.config.HeadlessTypeConcurrency != config.HeadlessTypeConcurrency {
+		w.config.HeadlessTypeConcurrency = config.HeadlessTypeConcurrency
+	}
+	if w.config.InputConcurrency != config.InputConcurrency {
+		w.config.InputConcurrency = config.InputConcurrency
+	}
+	if w.config.HeadlessInputConcurrency != config.HeadlessInputConcurrency {
+		w.config.HeadlessInputConcurrency = config.HeadlessInputConcurrency
+	}
+	w.Refresh()
+}
+
+func (w *WorkPool) Refresh() {
+	if w.Default.Size != w.config.TypeConcurrency {
+		w.Default.Resize(w.config.TypeConcurrency)
+	}
+	if w.Headless.Size != w.config.HeadlessTypeConcurrency {
+		w.Headless.Resize(w.config.HeadlessTypeConcurrency)
+	}
 }

pkg/fuzz/dataformat/form.go

@@ -94,18 +94,33 @@ func (f *Form) Encode(data KV) (string, error) {
 					}
 				}
 			}
-			data := make([]string, maxIndex+1) // Ensure the slice is large enough
-			for key, value := range v {
-				matches := reNormalized.FindStringSubmatch(key)
-				if len(matches) == 2 {
-					dataIdx, _ := strconv.Atoi(matches[1]) // Error already checked above
-					data[dataIdx-1] = value                // Use dataIdx-1 since slice is 0-indexed
+			if maxIndex >= 0 { // Ensure the slice is only created if maxIndex is valid
+				data := make([]string, maxIndex+1) // Ensure the slice is large enough
+				for key, value := range v {
+					matches := reNormalized.FindStringSubmatch(key)
+					if len(matches) == 2 {
+						dataIdx, err := strconv.Atoi(matches[1]) // Error already checked above
+						if err != nil {
+							gologger.Verbose().Msgf("error converting data index to integer: %v", err)
+							continue
+						}
+						// Validate dataIdx to avoid index out of range errors
+						if dataIdx > 0 && dataIdx <= len(data) {
+							data[dataIdx-1] = value // Use dataIdx-1 since slice is 0-indexed
+						} else {
+							gologger.Verbose().Msgf("data index out of range: %d", dataIdx)
+						}
+					}
+				}
+				if len(params.Get(k)) > 0 {
+					data[maxIndex] = fmt.Sprint(params.Get(k)) // Use maxIndex which is the last index
+				}
+				// remove existing
+				params.Del(k)
+				if len(data) > 0 {
+					params.Add(k, data...)
 				}
 			}
-			data[maxIndex] = fmt.Sprint(params.Get(k)) // Use maxIndex which is the last index
-			// remove existing
-			params.Del(k)
-			params.Add(k, data...)
 		}
 	}
 

pkg/fuzz/execute.go

@@ -11,6 +11,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/component"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/retryablehttp-go"
 	errorutil "github.com/projectdiscovery/utils/errors"
@@ -100,8 +101,11 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) {
 	baseValues := input.Values
 	if rule.generator == nil {
 		for _, component := range finalComponentList {
+			// get vars from variables while replacing interactsh urls
 			evaluatedValues, interactURLs := rule.options.Variables.EvaluateWithInteractsh(baseValues, rule.options.Interactsh)
-			input.Values = generators.MergeMaps(evaluatedValues, baseValues, rule.options.Constants)
+			input.Values = generators.MergeMaps(evaluatedValues, baseValues, rule.options.Options.Vars.AsMap(), rule.options.Constants)
+			// evaluate all vars with interactsh
+			input.Values, interactURLs = rule.evaluateVarsWithInteractsh(input.Values, interactURLs)
 			input.InteractURLs = interactURLs
 			err := rule.executeRuleValues(input, component)
 			if err != nil {
@@ -118,9 +122,12 @@ mainLoop:
 			if !next {
 				continue mainLoop
 			}
+			// get vars from variables while replacing interactsh urls
 			evaluatedValues, interactURLs := rule.options.Variables.EvaluateWithInteractsh(generators.MergeMaps(values, baseValues), rule.options.Interactsh)
+			input.Values = generators.MergeMaps(values, evaluatedValues, baseValues, rule.options.Options.Vars.AsMap(), rule.options.Constants)
+			// evaluate all vars with interactsh
+			input.Values, interactURLs = rule.evaluateVarsWithInteractsh(input.Values, interactURLs)
 			input.InteractURLs = interactURLs
-			input.Values = generators.MergeMaps(values, evaluatedValues, baseValues, rule.options.Constants)
 
 			if err := rule.executeRuleValues(input, component); err != nil {
 				if err == io.EOF {
@@ -134,6 +141,33 @@ mainLoop:
 	return nil
 }
 
+// evaluateVarsWithInteractsh evaluates the variables with Interactsh URLs and updates them accordingly.
+func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, interactshUrls []string) (map[string]interface{}, []string) {
+	// Check if Interactsh options are configured
+	if rule.options.Interactsh != nil {
+		// Iterate through the data to replace and evaluate variables with Interactsh URLs
+		for k, v := range data {
+			// Replace variables with Interactsh URLs and collect new URLs
+			got, oastUrls := rule.options.Interactsh.Replace(fmt.Sprint(v), interactshUrls)
+
+			// Append new OAST URLs if any
+			if len(oastUrls) > 0 {
+				interactshUrls = append(interactshUrls, oastUrls...)
+			}
+			// Evaluate the replaced data
+			evaluatedData, err := expressions.Evaluate(got, data)
+			if err == nil {
+				// Update the data if there is a change after evaluation
+				if evaluatedData != got {
+					data[k] = evaluatedData
+				}
+			}
+		}
+	}
+	// Return the updated data and Interactsh URLs without any error
+	return data, interactshUrls
+}
+
 // isInputURLValid returns true if url is valid after parsing it
 func (rule *Rule) isInputURLValid(input *contextargs.Context) bool {
 	if input == nil || input.MetaInput == nil || input.MetaInput.Input == "" {

pkg/fuzz/type.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/invopop/jsonschema"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	"gopkg.in/yaml.v2"
 )
@@ -29,6 +30,44 @@ type SliceOrMapSlice struct {
 	KV    *mapsutil.OrderedMap[string, string]
 }
 
+func (v SliceOrMapSlice) JSONSchemaExtend(schema *jsonschema.Schema) *jsonschema.Schema {
+	schema = &jsonschema.Schema{
+		Title:       schema.Title,
+		Description: schema.Description,
+		Type:        "array",
+		Items: &jsonschema.Schema{
+			OneOf: []*jsonschema.Schema{
+				{
+					Type: "string",
+				},
+				{
+					Type: "object",
+				},
+			},
+		},
+	}
+	return schema
+}
+
+func (v SliceOrMapSlice) JSONSchema() *jsonschema.Schema {
+	gotType := &jsonschema.Schema{
+		Title:       "Payloads of Fuzz Rule",
+		Description: "Payloads to perform fuzzing substitutions with.",
+		Type:        "array",
+		Items: &jsonschema.Schema{
+			OneOf: []*jsonschema.Schema{
+				{
+					Type: "string",
+				},
+				{
+					Type: "object",
+				},
+			},
+		},
+	}
+	return gotType
+}
+
 // UnmarshalJSON implements json.Unmarshaler interface.
 func (v *SliceOrMapSlice) UnmarshalJSON(data []byte) error {
 	// try to unmashal as a string and fallback to map

pkg/js/compiler/non-pool.go

@@ -4,13 +4,13 @@ import (
 	"sync"
 
 	"github.com/dop251/goja"
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var (
-	ephemeraljsc    = sizedwaitgroup.New(NonPoolingVMConcurrency)
+	ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
 	lazyFixedSgInit = sync.OnceFunc(func() {
-		ephemeraljsc = sizedwaitgroup.New(NonPoolingVMConcurrency)
+		ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
 	})
 )
 

pkg/js/compiler/pool.go

@@ -36,7 +36,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/goconsole"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	stringsutil "github.com/projectdiscovery/utils/strings"
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 const (
@@ -51,10 +51,16 @@ var (
 		// autoregister console node module with default printer it uses gologger backend
 		require.RegisterNativeModule(console.ModuleName, console.RequireWithPrinter(goconsole.NewGoConsolePrinter()))
 	})
-	pooljsc    sizedwaitgroup.SizedWaitGroup
+	pooljsc    *syncutil.AdaptiveWaitGroup
 	lazySgInit = sync.OnceFunc(func() {
-		pooljsc = sizedwaitgroup.New(PoolingJsVmConcurrency)
+		pooljsc, _ = syncutil.New(syncutil.WithSize(PoolingJsVmConcurrency))
 	})
+	sgResizeCheck = func() {
+		// resize check point
+		if pooljsc.Size != PoolingJsVmConcurrency {
+			pooljsc.Resize(PoolingJsVmConcurrency)
+		}
+	}
 )
 
 var gojapool = &sync.Pool{
@@ -116,6 +122,8 @@ func executeWithPoolingProgram(p *goja.Program, args *ExecuteArgs, opts *Execute
 	// its unknown (most likely cannot be done) to limit max js runtimes at a moment without making it static
 	// unlike sync.Pool which reacts to GC and its purposes is to reuse objects rather than creating new ones
 	lazySgInit()
+	sgResizeCheck()
+
 	pooljsc.Add()
 	defer pooljsc.Done()
 	runtime := gojapool.Get().(*goja.Runtime)

pkg/js/generated/ts/kerberos.ts

@@ -200,9 +200,9 @@ export interface AuthorizationDataEntry {
  */
 export interface BitString {
     
-    Bytes?: Uint8Array,
-    
     BitLength?: number,
+    
+    Bytes?: Uint8Array,
 }
 
 
@@ -212,9 +212,9 @@ export interface BitString {
  */
 export interface BitString {
     
-    Bytes?: Uint8Array,
-    
     BitLength?: number,
+    
+    Bytes?: Uint8Array,
 }
 
 
@@ -236,17 +236,15 @@ export interface Config {
  */
 export interface EncTicketPart {
     
-    RenewTill?: Date,
-    
-    CRealm?: string,
-    
     AuthTime?: Date,
     
     StartTime?: Date,
     
     EndTime?: Date,
     
-    Transited?: TransitedEncoding,
+    RenewTill?: Date,
+    
+    CRealm?: string,
     
     CAddr?: HostAddress,
     
@@ -257,6 +255,8 @@ export interface EncTicketPart {
     Key?: EncryptionKey,
     
     CName?: PrincipalName,
+    
+    Transited?: TransitedEncoding,
 }
 
 
@@ -266,11 +266,11 @@ export interface EncTicketPart {
  */
 export interface EncryptedData {
     
+    KVNO?: number,
+    
     Cipher?: Uint8Array,
     
     EType?: number,
-    
-    KVNO?: number,
 }
 
 
@@ -318,15 +318,41 @@ export interface HostAddress {
  */
 export interface LibDefaults {
     
+    NoAddresses?: boolean,
+    
+    RealmTryDomains?: number,
+    
+    DNSLookupKDC?: boolean,
+    
+    DefaultRealm?: string,
+    
+    SafeChecksumType?: number,
+    
+    VerifyAPReqNofail?: boolean,
+    
+    AllowWeakCrypto?: boolean,
+    
+    DefaultTGSEnctypes?: string[],
+    
+    DefaultTktEnctypeIDs?: number[],
+    
+    IgnoreAcceptorHostname?: boolean,
+    
+    K5LoginAuthoritative?: boolean,
+    
+    PermittedEnctypes?: string[],
+    
     /**
     * time in nanoseconds
     */
     
     Clockskew?: number,
     
-    KDCTimeSync?: number,
+    DNSCanonicalizeHostname?: boolean,
     
-    SafeChecksumType?: number,
+    Proxiable?: boolean,
+    
+    RDNS?: boolean,
     
     /**
     * time in nanoseconds
@@ -334,51 +360,31 @@ export interface LibDefaults {
     
     TicketLifetime?: number,
     
-    Forwardable?: boolean,
-    
-    K5LoginAuthoritative?: boolean,
-    
-    AllowWeakCrypto?: boolean,
-    
     DefaultClientKeytabName?: string,
     
-    DefaultTktEnctypes?: string[],
-    
-    ExtraAddresses?: Uint8Array,
-    
-    K5LoginDirectory?: string,
-    
-    PreferredPreauthTypes?: number[],
-    
-    RDNS?: boolean,
-    
-    DefaultKeytabName?: string,
-    
-    DefaultRealm?: string,
-    
-    DefaultTGSEnctypeIDs?: number[],
-    
-    DNSCanonicalizeHostname?: boolean,
-    
-    PermittedEnctypes?: string[],
-    
-    VerifyAPReqNofail?: boolean,
-    
-    DNSLookupRealm?: boolean,
+    PermittedEnctypeIDs?: number[],
     
     UDPPreferenceLimit?: number,
     
-    Canonicalize?: boolean,
+    DefaultTGSEnctypeIDs?: number[],
+    
+    DefaultTktEnctypes?: string[],
     
     CCacheType?: number,
     
-    DefaultTGSEnctypes?: string[],
+    DNSLookupRealm?: boolean,
     
-    Proxiable?: boolean,
+    ExtraAddresses?: Uint8Array,
     
-    DNSLookupKDC?: boolean,
+    PreferredPreauthTypes?: number[],
     
-    RealmTryDomains?: number,
+    Canonicalize?: boolean,
+    
+    Forwardable?: boolean,
+    
+    K5LoginDirectory?: string,
+    
+    KDCTimeSync?: number,
     
     /**
     * time in nanoseconds
@@ -386,13 +392,7 @@ export interface LibDefaults {
     
     RenewLifetime?: number,
     
-    DefaultTktEnctypeIDs?: number[],
-    
-    IgnoreAcceptorHostname?: boolean,
-    
-    NoAddresses?: boolean,
-    
-    PermittedEnctypeIDs?: number[],
+    DefaultKeytabName?: string,
     
     KDCDefaultOptions?: BitString,
 }
@@ -404,9 +404,9 @@ export interface LibDefaults {
  */
 export interface PrincipalName {
     
-    NameType?: number,
-    
     NameString?: string[],
+    
+    NameType?: number,
 }
 
 
@@ -416,8 +416,6 @@ export interface PrincipalName {
  */
 export interface Realm {
     
-    Realm?: string,
-    
     AdminServer?: string[],
     
     DefaultDomain?: string,
@@ -427,6 +425,8 @@ export interface Realm {
     KPasswdServer?: string[],
     
     MasterKDC?: string[],
+    
+    Realm?: string,
 }
 
 
@@ -450,10 +450,10 @@ export interface TGS {
  */
 export interface Ticket {
     
-    Realm?: string,
-    
     TktVNO?: number,
     
+    Realm?: string,
+    
     SName?: PrincipalName,
     
     EncPart?: EncryptedData,

pkg/js/generated/ts/ldap.ts

@@ -358,6 +358,22 @@ export class Client {
     }
     
 
+    /**
+    * GetADUserAsRepRoastable returns all AD users that are AsRepRoastable
+    * using FilterIsPerson, and FilterDontRequirePreauth filter query
+    * @example
+    * ```javascript
+    * const ldap = require('nuclei/ldap');
+    * const client = new ldap.Client('ldap://ldap.example.com', 'acme.com');
+    * const AsRepRoastable = client.GetADUserAsRepRoastable();
+    * log(to_json(AsRepRoastable));
+    * ```
+    */
+    public GetADUserAsRepRoastable(): ADObject[] {
+        return [];
+    }
+    
+
     /**
     * GetADDomainSID returns the SID of the AD domain
     * @example

pkg/js/generated/ts/mysql.ts

@@ -209,9 +209,9 @@ export interface MySQLOptions {
  */
 export interface SQLResult {
     
-    Columns?: string[],
-    
     Count?: number,
+    
+    Columns?: string[],
 }
 
 

pkg/js/generated/ts/smb.ts

@@ -137,10 +137,6 @@ export interface NegotiationLog {
  */
 export interface SMBCapabilities {
     
-    DFSSupport?: boolean,
-    
-    Leasing?: boolean,
-    
     LargeMTU?: boolean,
     
     MultiChan?: boolean,
@@ -150,6 +146,10 @@ export interface SMBCapabilities {
     DirLeasing?: boolean,
     
     Encryption?: boolean,
+    
+    DFSSupport?: boolean,
+    
+    Leasing?: boolean,
 }
 
 
@@ -159,6 +159,8 @@ export interface SMBCapabilities {
  */
 export interface SMBLog {
     
+    SupportV1?: boolean,
+    
     NativeOs?: string,
     
     NTLM?: string,
@@ -167,15 +169,13 @@ export interface SMBLog {
     
     HasNTLM?: boolean,
     
-    SupportV1?: boolean,
+    Version?: SMBVersions,
     
     Capabilities?: SMBCapabilities,
     
     NegotiationLog?: NegotiationLog,
     
     SessionSetupLog?: SessionSetupLog,
-    
-    Version?: SMBVersions,
 }
 
 
@@ -185,13 +185,13 @@ export interface SMBLog {
  */
 export interface SMBVersions {
     
+    VerString?: string,
+    
     Major?: number,
     
     Minor?: number,
     
     Revision?: number,
-    
-    VerString?: string,
 }
 
 
@@ -201,10 +201,6 @@ export interface SMBVersions {
  */
 export interface ServiceSMB {
     
-    SigningEnabled?: boolean,
-    
-    SigningRequired?: boolean,
-    
     OSVersion?: string,
     
     NetBIOSComputerName?: string,
@@ -216,6 +212,10 @@ export interface ServiceSMB {
     DNSDomainName?: string,
     
     ForestName?: string,
+    
+    SigningEnabled?: boolean,
+    
+    SigningRequired?: boolean,
 }
 
 
@@ -225,12 +225,12 @@ export interface ServiceSMB {
  */
 export interface SessionSetupLog {
     
+    NegotiateFlags?: number,
+    
     SetupFlags?: number,
     
     TargetName?: string,
     
-    NegotiateFlags?: number,
-    
     HeaderLog?: HeaderLog,
 }
 

pkg/js/generated/ts/ssh.ts

@@ -133,9 +133,9 @@ export interface Algorithms {
     
     HostKey?: string,
     
-    W?: DirectionAlgorithms,
-    
     R?: DirectionAlgorithms,
+    
+    W?: DirectionAlgorithms,
 }
 
 
@@ -197,34 +197,34 @@ export interface HandshakeLog {
  */
 export interface KexInitMsg {
     
-    CiphersServerClient?: string[],
-    
-    MACsClientServer?: string[],
-    
-    MACsServerClient?: string[],
-    
-    LanguagesClientServer?: string[],
-    
-    KexAlgos?: string[],
-    
-    CiphersClientServer?: string[],
-    
     Reserved?: number,
     
-    CompressionClientServer?: string[],
-    
-    CompressionServerClient?: string[],
-    
-    LanguagesServerClient?: string[],
-    
-    FirstKexFollows?: boolean,
-    
     /**
     * fixed size array of length: [16]
     */
     
     Cookie?: Uint8Array,
     
+    CiphersClientServer?: string[],
+    
+    MACsClientServer?: string[],
+    
+    MACsServerClient?: string[],
+    
+    CompressionServerClient?: string[],
+    
+    LanguagesClientServer?: string[],
+    
+    FirstKexFollows?: boolean,
+    
+    KexAlgos?: string[],
+    
+    CiphersServerClient?: string[],
+    
+    CompressionClientServer?: string[],
+    
+    LanguagesServerClient?: string[],
+    
     ServerHostKeyAlgos?: string[],
 }
 

pkg/js/libs/ldap/adenum.go

@@ -253,6 +253,19 @@ func (c *Client) GetADUserKerberoastable() []ADObject {
 	return c.FindADObjects(JoinFilters(FilterIsPerson, FilterAccountEnabled, FilterHasServicePrincipalName))
 }
 
+// GetADUserAsRepRoastable returns all AD users that are AsRepRoastable
+// using FilterIsPerson, and FilterDontRequirePreauth filter query
+// @example
+// ```javascript
+// const ldap = require('nuclei/ldap');
+// const client = new ldap.Client('ldap://ldap.example.com', 'acme.com');
+// const AsRepRoastable = client.GetADUserAsRepRoastable();
+// log(to_json(AsRepRoastable));
+// ```
+func (c *Client) GetADUserAsRepRoastable() []ADObject {
+	return c.FindADObjects(JoinFilters(FilterIsPerson, FilterDontRequirePreauth))
+}
+
 // GetADDomainSID returns the SID of the AD domain
 // @example
 // ```javascript

pkg/model/types/userAgent/user_agent.go

@@ -61,7 +61,7 @@ type UserAgentHolder struct {
 	Value UserAgent `mapping:"true"`
 }
 
-func (userAgentHolder UserAgentHolder) JSONSchemaType() *jsonschema.Schema {
+func (userAgentHolder UserAgentHolder) JSONSchema() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "userAgent for the headless",

pkg/operators/extractors/extractor_types.go

@@ -72,7 +72,7 @@ type ExtractorTypeHolder struct {
 	ExtractorType ExtractorType `mapping:"true"`
 }
 
-func (holder ExtractorTypeHolder) JSONSchemaType() *jsonschema.Schema {
+func (holder ExtractorTypeHolder) JSONSchema() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "type of the extractor",

pkg/operators/matchers/matchers_types.go

@@ -82,7 +82,7 @@ func (t MatcherTypeHolder) String() string {
 	return t.MatcherType.String()
 }
 
-func (holder MatcherTypeHolder) JSONSchemaType() *jsonschema.Schema {
+func (holder MatcherTypeHolder) JSONSchema() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "type of the matcher",

pkg/protocols/code/code.go

@@ -48,25 +48,25 @@ var (
 type Request struct {
 	// Operators for the current request go here.
 	operators.Operators `yaml:",inline,omitempty"`
-	CompiledOperators   *operators.Operators `yaml:"-"`
+	CompiledOperators   *operators.Operators `yaml:"-" json:"-"`
 
 	// ID is the optional id of the request
 	ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the request,description=ID is the optional ID of the Request"`
 	// description: |
 	//   Engine type
-	Engine []string `yaml:"engine,omitempty" jsonschema:"title=engine,description=Engine"`
+	Engine []string `yaml:"engine,omitempty" json:"engine,omitempty" jsonschema:"title=engine,description=Engine"`
 	// description: |
 	//   PreCondition is a condition which is evaluated before sending the request.
 	PreCondition string `yaml:"pre-condition,omitempty" json:"pre-condition,omitempty" jsonschema:"title=pre-condition for the request,description=PreCondition is a condition which is evaluated before sending the request"`
 	// description: |
 	//   Engine Arguments
-	Args []string `yaml:"args,omitempty" jsonschema:"title=args,description=Args"`
+	Args []string `yaml:"args,omitempty" json:"args,omitempty" jsonschema:"title=args,description=Args"`
 	// description: |
 	//   Pattern preferred for file name
-	Pattern string `yaml:"pattern,omitempty" jsonschema:"title=pattern,description=Pattern"`
+	Pattern string `yaml:"pattern,omitempty" json:"pattern,omitempty" jsonschema:"title=pattern,description=Pattern"`
 	// description: |
 	//   Source File/Snippet
-	Source string `yaml:"source,omitempty" jsonschema:"title=source file/snippet,description=Source snippet"`
+	Source string `yaml:"source,omitempty" json:"source,omitempty" jsonschema:"title=source file/snippet,description=Source snippet"`
 
 	options              *protocols.ExecutorOptions `yaml:"-" json:"-"`
 	preConditionCompiled *goja.Program              `yaml:"-" json:"-"`
@@ -160,6 +160,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 	if request.options.HasTemplateCtx(input.MetaInput) {
 		allvars = generators.MergeMaps(allvars, request.options.GetTemplateCtx(input.MetaInput).GetAll())
 	}
+	// add dynamic and previous variables
+	allvars = generators.MergeMaps(allvars, dynamicValues, previous)
 	// optionvars are vars passed from CLI or env variables
 	optionVars := generators.BuildPayloadFromOptions(request.options.Options)
 	variablesMap := request.options.Variables.Evaluate(allvars)

pkg/protocols/common/automaticscan/automaticscan.go

@@ -30,8 +30,8 @@ import (
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
+	syncutil "github.com/projectdiscovery/utils/sync"
 	wappalyzer "github.com/projectdiscovery/wappalyzergo"
-	"github.com/remeh/sizedwaitgroup"
 	"gopkg.in/yaml.v2"
 )
 
@@ -128,7 +128,10 @@ func (s *Service) Close() bool {
 func (s *Service) Execute() error {
 	gologger.Info().Msgf("Executing Automatic scan on %d target[s]", s.target.Count())
 	// setup host concurrency
-	sg := sizedwaitgroup.New(s.opts.Options.BulkSize)
+	sg, err := syncutil.New(syncutil.WithSize(s.opts.Options.BulkSize))
+	if err != nil {
+		return err
+	}
 	s.target.Iterate(func(value *contextargs.MetaInput) bool {
 		sg.Add()
 		go func(input *contextargs.MetaInput) {
@@ -246,7 +249,7 @@ func (s *Service) getTagsUsingDetectionTemplates(input *contextargs.MetaInput) (
 	// execute tech detection templates on target
 	tags := map[string]struct{}{}
 	m := &sync.Mutex{}
-	sg := sizedwaitgroup.New(s.opts.Options.TemplateThreads)
+	sg, _ := syncutil.New(syncutil.WithSize(s.opts.Options.TemplateThreads))
 	counter := atomic.Uint32{}
 
 	for _, t := range s.techTemplates {

pkg/protocols/common/generators/attack_types.go

@@ -61,7 +61,7 @@ type AttackTypeHolder struct {
 	Value AttackType `mapping:"true"`
 }
 
-func (holder AttackTypeHolder) JSONSchemaType() *jsonschema.Schema {
+func (holder AttackTypeHolder) JSONSchema() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "type of the attack",

pkg/protocols/common/protocolstate/state.go

@@ -124,7 +124,7 @@ func Init(options *types.Options) error {
 		opts.ResolversFile = true
 		opts.EnableFallback = true
 	}
-	if options.ResolversFile != "" {
+	if len(options.InternalResolversList) > 0 {
 		opts.BaseResolvers = options.InternalResolversList
 	}
 
@@ -132,6 +132,9 @@ func Init(options *types.Options) error {
 
 	opts.WithDialerHistory = true
 	opts.SNIName = options.SNI
+	// this instance is used in javascript protocol libraries and
+	// dial history is required to get dialed ip of a host
+	opts.WithDialerHistory = true
 
 	// fastdialer now by default fallbacks to ztls when there are tls related errors
 	dialer, err := fastdialer.NewDialer(opts)

pkg/protocols/common/variables/variables.go

@@ -21,7 +21,7 @@ type Variable struct {
 	utils.InsertionOrderedStringMap `yaml:"-" json:"-"`
 }
 
-func (variables Variable) JSONSchemaType() *jsonschema.Schema {
+func (variables Variable) JSONSchema() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:                 "object",
 		Title:                "variables for the request",

pkg/protocols/dns/dns.go

@@ -60,7 +60,7 @@ type Request struct {
 	// examples:
 	//   - name: Use a retry of 100 to 150 generally
 	//     value: 100
-	TraceMaxRecursion int `yaml:"trace-max-recursion,omitempty"  jsonschema:"title=trace-max-recursion level for dns request,description=TraceMaxRecursion is the number of max recursion allowed for trace operations"`
+	TraceMaxRecursion int `yaml:"trace-max-recursion,omitempty" json:"trace-max-recursion,omitempty"  jsonschema:"title=trace-max-recursion level for dns request,description=TraceMaxRecursion is the number of max recursion allowed for trace operations"`
 
 	// description: |
 	//   Attack is the type of payload combinations to perform.
@@ -83,7 +83,7 @@ type Request struct {
 	Threads   int `yaml:"threads,omitempty" json:"threads,omitempty" jsonschema:"title=threads for sending requests,description=Threads specifies number of threads to use sending requests. This enables Connection Pooling"`
 	generator *generators.PayloadGenerator
 
-	CompiledOperators *operators.Operators `yaml:"-"`
+	CompiledOperators *operators.Operators `yaml:"-" json:"-"`
 	dnsClient         *retryabledns.Client
 	options           *protocols.ExecutorOptions
 
@@ -275,6 +275,8 @@ func questionTypeToInt(questionType string) uint16 {
 		question = dns.TypeTLSA
 	case "ANY":
 		question = dns.TypeANY
+	case "SRV":
+		question = dns.TypeSRV
 	}
 	return question
 }

pkg/protocols/dns/dns_types.go

@@ -37,6 +37,8 @@ const (
 	TLSA
 	// name:ANY
 	ANY
+	// name:SRV
+	SRV
 	limit
 )
 
@@ -54,6 +56,7 @@ var DNSRequestTypeMapping = map[DNSRequestType]string{
 	CAA:   "CAA",
 	TLSA:  "TLSA",
 	ANY:   "ANY",
+	SRV:   "SRV",
 }
 
 // GetSupportedDNSRequestTypes returns list of supported types
@@ -92,7 +95,7 @@ func (holder DNSRequestTypeHolder) String() string {
 	return holder.DNSRequestType.String()
 }
 
-func (holder DNSRequestTypeHolder) JSONSchemaType() *jsonschema.Schema {
+func (holder DNSRequestTypeHolder) JSONSchema() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "type of DNS request to make",

pkg/protocols/dns/dnsclientpool/clientpool.go

@@ -34,7 +34,7 @@ func Init(options *types.Options) error {
 	clientPool = make(map[string]*retryabledns.Client)
 
 	resolvers := defaultResolvers
-	if options.ResolversFile != "" {
+	if len(options.InternalResolversList) > 0 {
 		resolvers = options.InternalResolversList
 	}
 	var err error
@@ -78,7 +78,7 @@ func Get(options *types.Options, configuration *Configuration) (*retryabledns.Cl
 	poolMutex.RUnlock()
 
 	resolvers := defaultResolvers
-	if options.ResolversFile != "" {
+	if len(options.InternalResolversList) > 0 {
 		resolvers = options.InternalResolversList
 	} else if len(configuration.Resolvers) > 0 {
 		resolvers = configuration.Resolvers

pkg/protocols/dns/request.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/miekg/dns"
 	"github.com/pkg/errors"
-	"github.com/remeh/sizedwaitgroup"
 	"go.uber.org/multierr"
 	"golang.org/x/exp/maps"
 
@@ -27,6 +26,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	"github.com/projectdiscovery/retryabledns"
 	iputil "github.com/projectdiscovery/utils/ip"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var _ protocols.Request = &Request{}
@@ -62,9 +62,15 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 	variablesMap := request.options.Variables.Evaluate(vars)
 	vars = generators.MergeMaps(vars, variablesMap, request.options.Constants)
 
+	// if request threads matches global payload concurrency we follow it
+	shouldFollowGlobal := request.Threads == request.options.Options.PayloadConcurrency
+
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
-		swg := sizedwaitgroup.New(request.Threads)
+		swg, err := syncutil.New(syncutil.WithSize(request.Threads))
+		if err != nil {
+			return err
+		}
 		var multiErr error
 		m := &sync.Mutex{}
 
@@ -73,6 +79,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 			if !ok {
 				break
 			}
+
+			// resize check point - nop if there are no changes
+			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
+				swg.Resize(request.options.Options.PayloadConcurrency)
+			}
+
 			value = generators.MergeMaps(vars, value)
 			swg.Add()
 			go func(newVars map[string]interface{}) {
@@ -140,7 +152,7 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad
 		}
 	}
 
-	request.options.RateLimiter.Take()
+	request.options.RateLimitTake()
 
 	// Send the request to the target servers
 	response, err := dnsClient.Do(compiledRequest)

pkg/protocols/file/request.go

@@ -11,7 +11,6 @@ import (
 	"github.com/docker/go-units"
 	"github.com/mholt/archiver"
 	"github.com/pkg/errors"
-	"github.com/remeh/sizedwaitgroup"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
@@ -24,6 +23,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	sliceutil "github.com/projectdiscovery/utils/slice"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var _ protocols.Request = &Request{}
@@ -47,8 +47,11 @@ var errEmptyResult = errors.New("Empty result")
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
-	wg := sizedwaitgroup.New(request.options.Options.BulkSize)
-	err := request.getInputPaths(input.MetaInput.Input, func(filePath string) {
+	wg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize))
+	if err != nil {
+		return err
+	}
+	err = request.getInputPaths(input.MetaInput.Input, func(filePath string) {
 		wg.Add()
 		func(filePath string) {
 			defer wg.Done()

pkg/protocols/headless/engine/action.go

@@ -1,6 +1,10 @@
 package engine
 
-import "strings"
+import (
+	"strings"
+
+	"github.com/invopop/jsonschema"
+)
 
 // Action is an action taken by the browser to reach a navigation
 //
@@ -29,6 +33,29 @@ type Action struct {
 	ActionType ActionTypeHolder `yaml:"action" json:"action" jsonschema:"title=action to perform,description=Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"`
 }
 
+func (a Action) JSONSchemaExtend(schema *jsonschema.Schema) {
+	argsSchema, ok := schema.Properties.Get("args")
+	if !ok {
+		return
+	}
+	argsSchema.PatternProperties = map[string]*jsonschema.Schema{
+		".*": {
+			OneOf: []*jsonschema.Schema{
+				{
+					Type: "string",
+				},
+				{
+					Type: "integer",
+				},
+				{
+					Type: "boolean",
+				},
+			},
+		},
+	}
+	argsSchema.Ref = ""
+}
+
 // String returns the string representation of an action
 func (a *Action) String() string {
 	builder := &strings.Builder{}

pkg/protocols/headless/engine/action_types.go

@@ -171,7 +171,7 @@ type ActionTypeHolder struct {
 func (holder ActionTypeHolder) String() string {
 	return holder.ActionType.String()
 }
-func (holder ActionTypeHolder) JSONSchemaType() *jsonschema.Schema {
+func (holder ActionTypeHolder) JSONSchema() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "action to perform",

pkg/protocols/headless/engine/page_actions.go

@@ -214,7 +214,7 @@ func geTimeParameter(p *Page, act *Action, parameterName string, defaultValue ti
 }
 
 // ActionAddHeader executes a AddHeader action.
-func (p *Page) ActionAddHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) ActionAddHeader(act *Action, out map[string]string) error {
 	in := p.getActionArgWithDefaultValues(act, "part")
 
 	args := make(map[string]string)
@@ -225,7 +225,7 @@ func (p *Page) ActionAddHeader(act *Action, out map[string]string /*TODO review
 }
 
 // ActionSetHeader executes a SetHeader action.
-func (p *Page) ActionSetHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) ActionSetHeader(act *Action, out map[string]string) error {
 	in := p.getActionArgWithDefaultValues(act, "part")
 
 	args := make(map[string]string)
@@ -236,7 +236,7 @@ func (p *Page) ActionSetHeader(act *Action, out map[string]string /*TODO review
 }
 
 // ActionDeleteHeader executes a DeleteHeader action.
-func (p *Page) ActionDeleteHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) ActionDeleteHeader(act *Action, out map[string]string) error {
 	in := p.getActionArgWithDefaultValues(act, "part")
 
 	args := make(map[string]string)
@@ -343,7 +343,7 @@ func (p *Page) RunScript(action *Action, out map[string]string) error {
 }
 
 // ClickElement executes click actions for an element.
-func (p *Page) ClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) ClickElement(act *Action, out map[string]string) error {
 	element, err := p.pageElementBy(act.Data)
 	if err != nil {
 		return errors.Wrap(err, errCouldNotGetElement)
@@ -358,12 +358,12 @@ func (p *Page) ClickElement(act *Action, out map[string]string /*TODO review unu
 }
 
 // KeyboardAction executes a keyboard action on the page.
-func (p *Page) KeyboardAction(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) KeyboardAction(act *Action, out map[string]string) error {
 	return p.page.Keyboard.Type([]input.Key(p.getActionArgWithDefaultValues(act, "keys"))...)
 }
 
 // RightClickElement executes right click actions for an element.
-func (p *Page) RightClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) RightClickElement(act *Action, out map[string]string) error {
 	element, err := p.pageElementBy(act.Data)
 	if err != nil {
 		return errors.Wrap(err, errCouldNotGetElement)
@@ -441,7 +441,7 @@ func (p *Page) Screenshot(act *Action, out map[string]string) error {
 }
 
 // InputElement executes input element actions for an element.
-func (p *Page) InputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) InputElement(act *Action, out map[string]string) error {
 	value := p.getActionArgWithDefaultValues(act, "value")
 	if value == "" {
 		return errinvalidArguments
@@ -460,7 +460,7 @@ func (p *Page) InputElement(act *Action, out map[string]string /*TODO review unu
 }
 
 // TimeInputElement executes time input on an element
-func (p *Page) TimeInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) TimeInputElement(act *Action, out map[string]string) error {
 	value := p.getActionArgWithDefaultValues(act, "value")
 	if value == "" {
 		return errinvalidArguments
@@ -483,7 +483,7 @@ func (p *Page) TimeInputElement(act *Action, out map[string]string /*TODO review
 }
 
 // SelectInputElement executes select input statement action on a element
-func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) SelectInputElement(act *Action, out map[string]string) error {
 	value := p.getActionArgWithDefaultValues(act, "value")
 	if value == "" {
 		return errinvalidArguments
@@ -508,7 +508,7 @@ func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO revi
 }
 
 // WaitLoad waits for the page to load
-func (p *Page) WaitLoad(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) WaitLoad(act *Action, out map[string]string) error {
 	p.page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)()
 
 	// Wait for the window.onload event and also wait for the network requests
@@ -538,7 +538,7 @@ func (p *Page) GetResource(act *Action, out map[string]string) error {
 }
 
 // FilesInput acts with a file input element on page
-func (p *Page) FilesInput(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) FilesInput(act *Action, out map[string]string) error {
 	element, err := p.pageElementBy(act.Data)
 	if err != nil {
 		return errors.Wrap(err, errCouldNotGetElement)
@@ -589,7 +589,7 @@ func (p *Page) ExtractElement(act *Action, out map[string]string) error {
 }
 
 // WaitEvent waits for an event to happen on the page.
-func (p *Page) WaitEvent(act *Action, out map[string]string /*TODO review unused parameter*/) (func() error, error) {
+func (p *Page) WaitEvent(act *Action, out map[string]string) (func() error, error) {
 	event := p.getActionArgWithDefaultValues(act, "event")
 	if event == "" {
 		return nil, errors.New("event not recognized")
@@ -661,14 +661,14 @@ func (p *Page) pageElementBy(data map[string]string) (*rod.Element, error) {
 }
 
 // DebugAction enables debug action on a page.
-func (p *Page) DebugAction(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) DebugAction(act *Action, out map[string]string) error {
 	p.instance.browser.engine.SlowMotion(5 * time.Second)
 	p.instance.browser.engine.Trace(true)
 	return nil
 }
 
 // SleepAction sleeps on the page for a specified duration
-func (p *Page) SleepAction(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) SleepAction(act *Action, out map[string]string) error {
 	seconds := act.Data["duration"]
 	if seconds == "" {
 		seconds = "5"

pkg/protocols/http/http.go

@@ -5,9 +5,11 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/invopop/jsonschema"
 	json "github.com/json-iterator/go"
 	"github.com/pkg/errors"
 
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/fuzz"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
@@ -219,6 +221,29 @@ type Request struct {
 	fuzzPreConditionOperator matchers.ConditionType `yaml:"-" json:"-"`
 }
 
+func (e Request) JSONSchemaExtend(schema *jsonschema.Schema) {
+	headersSchema, ok := schema.Properties.Get("headers")
+	if !ok {
+		return
+	}
+	headersSchema.PatternProperties = map[string]*jsonschema.Schema{
+		".*": {
+			OneOf: []*jsonschema.Schema{
+				{
+					Type: "string",
+				},
+				{
+					Type: "integer",
+				},
+				{
+					Type: "boolean",
+				},
+			},
+		},
+	}
+	headersSchema.Ref = ""
+}
+
 // Options returns executer options for http request
 func (r *Request) Options() *protocols.ExecutorOptions {
 	return r.options
@@ -411,12 +436,34 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 		}
 	}
 	if len(request.Payloads) > 0 {
-		// specifically for http requests high concurrency and and threads will lead to memory exausthion, hence reduce the maximum parallelism
-		if protocolstate.IsLowOnMemory() {
-			request.Threads = protocolstate.GuardThreadsOrDefault(request.Threads)
+		// Due to a known issue (https://github.com/projectdiscovery/nuclei/issues/5015),
+		// dynamic extractors cannot be used with payloads. To address this,
+		// execution is handled by the standard engine without concurrency,
+		// achieved by setting the thread count to 0.
+
+		// this limitation will be removed once we have a better way to handle dynamic extractors with payloads
+		hasMultipleRequests := false
+		if len(request.Raw)+len(request.Path) > 1 {
+			hasMultipleRequests = true
+		}
+		// look for dynamic extractor ( internal: true with named extractor)
+		hasNamedInternalExtractor := false
+		for _, extractor := range request.Extractors {
+			if extractor.Internal && extractor.Name != "" {
+				hasNamedInternalExtractor = true
+				break
+			}
+		}
+		if hasNamedInternalExtractor && hasMultipleRequests {
+			gologger.Warning().Label(options.TemplateID).Msgf("Setting thread count to 0 because dynamic extractors are not supported with payloads yet")
+			request.Threads = 0
+		} else {
+			// specifically for http requests high concurrency and and threads will lead to memory exausthion, hence reduce the maximum parallelism
+			if protocolstate.IsLowOnMemory() {
+				request.Threads = protocolstate.GuardThreadsOrDefault(request.Threads)
+			}
+			request.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads)
 		}
-		// if we have payloads, adjust threads if none specified
-		request.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads)
 	}
 
 	return nil

pkg/protocols/http/http_method_types.go

@@ -89,7 +89,7 @@ func (holder HTTPMethodTypeHolder) String() string {
 	return holder.MethodType.String()
 }
 
-func (holder HTTPMethodTypeHolder) JSONSchemaType() *jsonschema.Schema {
+func (holder HTTPMethodTypeHolder) JSONSchema() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "method is the HTTP request method",

pkg/protocols/http/httputils/spm.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"sync"
 
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 // WorkPoolType is the type of work pool to use
@@ -26,7 +26,7 @@ type StopAtFirstMatchHandler[T any] struct {
 
 	// work pool and its type
 	poolType WorkPoolType
-	sgPool   sizedwaitgroup.SizedWaitGroup
+	sgPool   *syncutil.AdaptiveWaitGroup
 	wgPool   *sync.WaitGroup
 
 	// internal / unexported
@@ -40,10 +40,13 @@ type StopAtFirstMatchHandler[T any] struct {
 // NewBlockingSPMHandler creates a new stop at first match handler
 func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *StopAtFirstMatchHandler[T] {
 	ctx1, cancel := context.WithCancel(ctx)
+
+	awg, _ := syncutil.New(syncutil.WithSize(size))
+
 	s := &StopAtFirstMatchHandler[T]{
 		ResultChan:  make(chan T, 1),
 		poolType:    Blocking,
-		sgPool:      sizedwaitgroup.New(size),
+		sgPool:      awg,
 		internalWg:  &sync.WaitGroup{},
 		ctx:         ctx1,
 		cancel:      cancel,
@@ -140,6 +143,16 @@ func (h *StopAtFirstMatchHandler[T]) Release() {
 	}
 }
 
+func (h *StopAtFirstMatchHandler[T]) Resize(size int) {
+	if h.sgPool.Size != size {
+		h.sgPool.Resize(size)
+	}
+}
+
+func (h *StopAtFirstMatchHandler[T]) Size() int {
+	return h.sgPool.Size
+}
+
 // Wait waits for all work to be done
 func (h *StopAtFirstMatchHandler[T]) Wait() {
 	switch h.poolType {

pkg/protocols/http/request.go

@@ -165,6 +165,9 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 	// Workers that keeps enqueuing new requests
 	maxWorkers := request.Threads
 
+	// if request threads matches global payload concurrency we follow it
+	shouldFollowGlobal := maxWorkers == request.options.Options.PayloadConcurrency
+
 	if protocolstate.IsLowOnMemory() {
 		maxWorkers = protocolstate.GuardThreadsOrDefault(request.Threads)
 	}
@@ -198,6 +201,12 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		if !ok {
 			break
 		}
+
+		// resize check point - nop if there are no changes
+		if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency {
+			spmHandler.Resize(request.options.Options.PayloadConcurrency)
+		}
+
 		ctx := request.newContext(input)
 		generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
 		if err != nil {
@@ -222,7 +231,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 				return
 			case spmHandler.ResultChan <- func() error {
 				// putting ratelimiter here prevents any unnecessary waiting if any
-				request.options.RateLimiter.Take()
+				request.options.RateLimitTake()
 				previous := make(map[string]interface{})
 				return request.executeRequest(input, httpRequest, previous, false, wrappedCallback, 0)
 			}():
@@ -366,7 +375,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 		executeFunc := func(data string, payloads, dynamicValue map[string]interface{}) (bool, error) {
 			hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
 
-			request.options.RateLimiter.Take()
+			request.options.RateLimitTake()
 
 			ctx := request.newContext(input)
 			ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(request.options.Options.Timeout)*time.Second)

pkg/protocols/http/request_fuzz.go

@@ -145,7 +145,7 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
 	if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) {
 		return false
 	}
-	request.options.RateLimiter.Take()
+	request.options.RateLimitTake()
 	req := &generatedRequest{
 		request:        gr.Request,
 		dynamicValues:  gr.DynamicValues,

pkg/protocols/http/signature.go

@@ -51,7 +51,7 @@ type SignatureTypeHolder struct {
 	Value SignatureType
 }
 
-func (holder SignatureTypeHolder) JSONSchemaType() *jsonschema.Schema {
+func (holder SignatureTypeHolder) JSONSchema() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "type of the signature",

pkg/protocols/javascript/js.go

@@ -27,13 +27,15 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
 	protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	errorutil "github.com/projectdiscovery/utils/errors"
+	iputil "github.com/projectdiscovery/utils/ip"
+	syncutil "github.com/projectdiscovery/utils/sync"
 	urlutil "github.com/projectdiscovery/utils/url"
-	"github.com/remeh/sizedwaitgroup"
 )
 
 // Request is a request for the javascript protocol
@@ -404,7 +406,11 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 	requestOptions := request.options
 	gotmatches := &atomic.Bool{}
 
-	sg := sizedwaitgroup.New(threads)
+	// if request threads matches global payload concurrency we follow it
+	shouldFollowGlobal := threads == request.options.Options.PayloadConcurrency
+
+	sg, _ := syncutil.New(syncutil.WithSize(threads))
+
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
 		for {
@@ -412,6 +418,12 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 			if !ok {
 				break
 			}
+
+			// resize check point - nop if there are no changes
+			if shouldFollowGlobal && sg.Size != request.options.Options.PayloadConcurrency {
+				sg.Resize(request.options.Options.PayloadConcurrency)
+			}
+
 			sg.Add()
 			go func() {
 				defer sg.Done()
@@ -520,6 +532,46 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
 		data["stop-at-first-match"] = true
 	}
 
+	// add ip address to data
+	if input.MetaInput.CustomIP != "" {
+		data["ip"] = input.MetaInput.CustomIP
+	} else {
+		// context: https://github.com/projectdiscovery/nuclei/issues/5021
+		hostname := input.MetaInput.Input
+		if strings.Contains(hostname, ":") {
+			host, _, err := net.SplitHostPort(hostname)
+			if err == nil {
+				hostname = host
+			} else {
+				// naive way
+				if !strings.Contains(hostname, "]") {
+					hostname = hostname[:strings.LastIndex(hostname, ":")]
+				}
+			}
+		}
+		data["ip"] = protocolstate.Dialer.GetDialedIP(hostname)
+		// if input itself was an ip, use it
+		if iputil.IsIP(hostname) {
+			data["ip"] = hostname
+		}
+
+		// if ip is not found,this is because ssh and other protocols do not use fastdialer
+		// although its not perfect due to its use case dial and get ip
+		dnsData, err := protocolstate.Dialer.GetDNSData(hostname)
+		if err == nil {
+			for _, v := range dnsData.A {
+				data["ip"] = v
+				break
+			}
+			if data["ip"] == "" {
+				for _, v := range dnsData.AAAA {
+					data["ip"] = v
+					break
+				}
+			}
+		}
+	}
+
 	// add and get values from templatectx
 	request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), data)
 	data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll())

pkg/protocols/network/network.go

@@ -61,7 +61,7 @@ type Request struct {
 	// description: |
 	//   Port is the port to send network requests to. this acts as default port but is overriden if target/input contains
 	// non-http(s) ports like 80,8080,8081 etc
-	Port string `yaml:"port,omitempty" json:"port,omitempty" jsonschema:"title=port to send requests to,description=Port to send network requests to"`
+	Port string `yaml:"port,omitempty" json:"port,omitempty" jsonschema:"title=port to send requests to,description=Port to send network requests to,oneof_type=string;integer"`
 
 	// description:	|
 	//	ExcludePorts is the list of ports to exclude from being scanned . It is intended to be used with `Port` field and contains a list of ports which are ignored/skipped
@@ -91,7 +91,7 @@ type Request struct {
 
 	// Operators for the current request go here.
 	operators.Operators `yaml:",inline,omitempty"`
-	CompiledOperators   *operators.Operators `yaml:"-"`
+	CompiledOperators   *operators.Operators `yaml:"-" json:"-"`
 
 	generator *generators.PayloadGenerator
 	// cache any variables that may be needed for operation.
@@ -128,7 +128,7 @@ type Input struct {
 	// examples:
 	//   - value: "\"TEST\""
 	//   - value: "\"hex_decode('50494e47')\""
-	Data string `yaml:"data,omitempty" json:"data,omitempty" jsonschema:"title=data to send as input,description=Data is the data to send as the input"`
+	Data string `yaml:"data,omitempty" json:"data,omitempty" jsonschema:"title=data to send as input,description=Data is the data to send as the input,oneof_type=string;integer"`
 	// description: |
 	//   Type is the type of input specified in `data` field.
 	//

pkg/protocols/network/network_input_types.go

@@ -66,7 +66,7 @@ func (holder NetworkInputTypeHolder) String() string {
 	return holder.NetworkInputType.String()
 }
 
-func (holder NetworkInputTypeHolder) JSONSchemaType() *jsonschema.Schema {
+func (holder NetworkInputTypeHolder) JSONSchema() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "type is the type of input data",

pkg/protocols/network/networkclientpool/clientpool.go

@@ -11,7 +11,7 @@ var (
 )
 
 // Init initializes the clientpool implementation
-func Init(options *types.Options /*TODO review unused parameter*/) error {
+func Init(options *types.Options) error {
 	// Don't create clients if already created in the past.
 	if normalClient != nil {
 		return nil

pkg/protocols/network/request.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/remeh/sizedwaitgroup"
 	"go.uber.org/multierr"
 	"golang.org/x/exp/maps"
 
@@ -34,6 +33,7 @@ import (
 	errorutil "github.com/projectdiscovery/utils/errors"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	"github.com/projectdiscovery/utils/reader"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var (
@@ -174,17 +174,29 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 		return err
 	}
 
+	// if request threads matches global payload concurrency we follow it
+	shouldFollowGlobal := request.Threads == request.options.Options.PayloadConcurrency
+
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
 		var multiErr error
 		m := &sync.Mutex{}
-		swg := sizedwaitgroup.New(request.Threads)
+		swg, err := syncutil.New(syncutil.WithSize(request.Threads))
+		if err != nil {
+			return err
+		}
 
 		for {
 			value, ok := iterator.Value()
 			if !ok {
 				break
 			}
+
+			// resize check point - nop if there are no changes
+			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
+				swg.Resize(request.options.Options.PayloadConcurrency)
+			}
+
 			value = generators.MergeMaps(value, payloads)
 			swg.Add()
 			go func(vars map[string]interface{}) {

pkg/protocols/offlinehttp/request.go

@@ -6,7 +6,6 @@ import (
 	"os"
 
 	"github.com/pkg/errors"
-	"github.com/remeh/sizedwaitgroup"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
@@ -17,6 +16,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/utils/conversion"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var _ protocols.Request = &Request{}
@@ -29,10 +29,13 @@ func (request *Request) Type() templateTypes.ProtocolType {
 }
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
-func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
-	wg := sizedwaitgroup.New(request.options.Options.BulkSize)
+func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
+	wg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize))
+	if err != nil {
+		return err
+	}
 
-	err := request.getInputPaths(input.MetaInput.Input, func(data string) {
+	err = request.getInputPaths(input.MetaInput.Input, func(data string) {
 		wg.Add()
 
 		go func(data string) {

pkg/protocols/protocols.go

@@ -34,9 +34,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 )
 
-// Optional Callback to update Thread count in payloads across all requests
-type PayloadThreadSetterCallback func(opts *ExecutorOptions, totalRequests, currentThreads int) int
-
 var (
 	MaxTemplateFileSizeForEncoding = 1024 * 1024
 )
@@ -114,10 +111,6 @@ type ExecutorOptions struct {
 	// JsCompiler is abstracted javascript compiler which adds node modules and provides execution
 	// environment for javascript templates
 	JsCompiler *compiler.Compiler
-	// Optional Callback function to update Thread count in payloads across all protocols
-	// based on given logic. by default nuclei reverts to using value of `-c` when threads count
-	// is not specified or is 0 in template
-	OverrideThreadsCount PayloadThreadSetterCallback
 	// AuthProvider is a provider for auth strategies
 	AuthProvider authprovider.AuthProvider
 	//TemporaryDirectory is the directory to store temporary files
@@ -128,17 +121,25 @@ type ExecutorOptions struct {
 	ExportReqURLPattern bool
 }
 
+// todo: centralizing components is not feasible with current clogged architecture
+// a possible approach could be an internal event bus with pub-subs? This would be less invasive than
+// reworking dep injection from scratch
+func (eo *ExecutorOptions) RateLimitTake() {
+	if eo.RateLimiter.GetLimit() != uint(eo.Options.RateLimit) {
+		eo.RateLimiter.SetLimit(uint(eo.Options.RateLimit))
+		eo.RateLimiter.SetDuration(eo.Options.RateLimitDuration)
+	}
+	eo.RateLimiter.Take()
+}
+
 // GetThreadsForPayloadRequests returns the number of threads to use as default for
 // given max-request of payloads
 func (e *ExecutorOptions) GetThreadsForNPayloadRequests(totalRequests int, currentThreads int) int {
-	if e.OverrideThreadsCount != nil {
-		return e.OverrideThreadsCount(e, totalRequests, currentThreads)
-	}
 	if currentThreads > 0 {
 		return currentThreads
-	} else {
-		return e.Options.PayloadConcurrency
 	}
+
+	return e.Options.PayloadConcurrency
 }
 
 // CreateTemplateCtxStore creates template context store (which contains templateCtx for every scan)

pkg/protocols/ssl/ssl.go

@@ -282,6 +282,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 	} else {
 		data["ip"] = request.dialer.GetDialedIP(hostname)
 	}
+	data["Port"] = port
 	data["template-path"] = requestOptions.TemplatePath
 	data["template-id"] = requestOptions.TemplateID
 	data["template-info"] = requestOptions.TemplateInfo
@@ -405,6 +406,9 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 	if fields.Port == "80" {
 		fields.Port = "443"
 	}
+	if types.ToString(wrapped.InternalEvent["Port"]) != "" {
+		fields.Port = types.ToString(wrapped.InternalEvent["Port"])
+	}
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),
 		TemplatePath:     types.ToString(wrapped.InternalEvent["template-path"]),

pkg/reporting/exporters/es/elasticsearch.go

@@ -131,7 +131,8 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
 	if err != nil {
 		return err
 	}
-
+	defer res.Body.Close() 
+	
 	b, err = io.ReadAll(res.Body)
 	if err != nil {
 		return errors.New(err.Error() + "error thrown by elasticsearch " + string(b))

pkg/scan/charts/charts.go

@@ -0,0 +1,87 @@
+package charts
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/labstack/echo/v4"
+	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
+	fileutil "github.com/projectdiscovery/utils/file"
+)
+
+// ScanEventsCharts is a struct for nuclei event charts
+type ScanEventsCharts struct {
+	eventsDir string
+	config    *events.ScanConfig
+	data      []events.ScanEvent
+}
+
+func (sc *ScanEventsCharts) PrintInfo() {
+	fmt.Printf("[+] Scan Info\n")
+	fmt.Printf("  - Name: %s\n", sc.config.Name)
+	fmt.Printf("  - Target Count: %d\n", sc.config.TargetCount)
+	fmt.Printf("  - Template Count: %d\n", sc.config.TemplatesCount)
+	fmt.Printf("  - Template Concurrency: %d\n", sc.config.TemplateConcurrency)
+	fmt.Printf("  - Payload Concurrency: %d\n", sc.config.PayloadConcurrency)
+	fmt.Printf("  - Retries: %v\n", sc.config.Retries)
+	fmt.Printf("  - Total Events: %d\n", len(sc.data))
+	fmt.Println()
+}
+
+// NewScanEventsCharts creates a new nuclei event charts
+func NewScanEventsCharts(eventsDir string) (*ScanEventsCharts, error) {
+	sc := &ScanEventsCharts{eventsDir: eventsDir}
+	if !fileutil.FolderExists(eventsDir) {
+		return nil, fmt.Errorf("events directory does not exist")
+	}
+	// open two files
+	// config.json
+	bin, err := os.ReadFile(filepath.Join(eventsDir, events.ConfigFile))
+	if err != nil {
+		return nil, err
+	}
+	var config events.ScanConfig
+	err = json.Unmarshal(bin, &config)
+	if err != nil {
+		return nil, err
+	}
+	sc.config = &config
+
+	// events.jsonl
+	f, err := os.Open(filepath.Join(eventsDir, events.EventsFile))
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	data := []events.ScanEvent{}
+	dec := json.NewDecoder(f)
+	for {
+		var event events.ScanEvent
+		if err := dec.Decode(&event); err != nil {
+			break
+		}
+		data = append(data, event)
+	}
+	sc.data = data
+
+	if len(data) == 0 {
+		return nil, fmt.Errorf("no events found in the events file")
+	}
+
+	return sc, nil
+}
+
+// Start starts the nuclei event charts server
+func (sc *ScanEventsCharts) Start(addr string) {
+	e := echo.New()
+	e.HideBanner = true
+	e.GET("/concurrency", sc.ConcurrencyVsTime)
+	e.GET("/requests", sc.TotalRequestsOverTime)
+	e.GET("/slow", sc.TopSlowTemplates)
+	e.GET("/rps", sc.RequestsVSInterval)
+	e.GET("/", sc.AllCharts)
+	e.Logger.Fatal(e.Start(addr))
+}

pkg/scan/charts/echarts.go

@@ -0,0 +1,351 @@
+package charts
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"time"
+
+	"github.com/go-echarts/go-echarts/v2/charts"
+	"github.com/go-echarts/go-echarts/v2/components"
+	"github.com/go-echarts/go-echarts/v2/opts"
+	"github.com/labstack/echo/v4"
+	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
+	sliceutil "github.com/projectdiscovery/utils/slice"
+)
+
+const (
+	TopK         = 50
+	SpacerHeight = "50px"
+)
+
+func (s *ScanEventsCharts) AllCharts(c echo.Context) error {
+	page := s.allCharts(c)
+	return page.Render(c.Response().Writer)
+}
+
+func (s *ScanEventsCharts) GenerateHTML(filePath string) error {
+	page := s.allCharts(nil)
+	output, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+	return page.Render(output)
+}
+
+// AllCharts generates all the charts for the scan events and returns a page component
+func (s *ScanEventsCharts) allCharts(c echo.Context) *components.Page {
+	page := components.NewPage()
+	page.PageTitle = "Nuclei Charts"
+	line1 := s.totalRequestsOverTime(c)
+	line1.SetSpacerHeight(SpacerHeight)
+	kline := s.topSlowTemplates(c)
+	kline.SetSpacerHeight(SpacerHeight)
+	line2 := s.requestsVSInterval(c)
+	line2.SetSpacerHeight(SpacerHeight)
+	line3 := s.concurrencyVsTime(c)
+	line3.SetSpacerHeight(SpacerHeight)
+	page.AddCharts(line1, kline, line2, line3)
+	page.Validate()
+	page.SetLayout(components.PageCenterLayout)
+	return page
+}
+
+func (s *ScanEventsCharts) TotalRequestsOverTime(c echo.Context) error {
+	line := s.totalRequestsOverTime(c)
+	return line.Render(c.Response().Writer)
+}
+
+// totalRequestsOverTime generates a line chart showing total requests count over time
+func (s *ScanEventsCharts) totalRequestsOverTime(c echo.Context) *charts.Line {
+	line := charts.NewLine()
+	line.SetCaption("Chart Shows Total Requests Count Over Time (for each/all Protocols)")
+
+	var startTime time.Time = time.Now()
+	var endTime time.Time
+
+	for _, event := range s.data {
+		if event.Time.Before(startTime) {
+			startTime = event.Time
+		}
+		if event.Time.After(endTime) {
+			endTime = event.Time
+		}
+	}
+	data := getCategoryRequestCount(s.data)
+	max := 0
+	for _, v := range data {
+		if len(v) > max {
+			max = len(v)
+		}
+	}
+	line.SetXAxis(time.Now().Format(time.RFC3339))
+	for k, v := range data {
+		lineData := make([]opts.LineData, 0)
+		temp := 0
+		for _, scanEvent := range v {
+			temp += scanEvent.MaxRequests
+			val := scanEvent.Time.Sub(startTime)
+			lineData = append(lineData, opts.LineData{
+				Value: []interface{}{val.Milliseconds(), temp},
+				Name:  scanEvent.TemplateID,
+			})
+		}
+		line.AddSeries(k, lineData, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"}))
+	}
+
+	line.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{Title: "Nuclei: total-req vs time"}),
+		charts.WithXAxisOpts(opts.XAxis{Name: "Time", Type: "time", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}),
+		charts.WithYAxisOpts(opts.YAxis{Name: "Requests Sent", Type: "value"}),
+		charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}),
+		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
+		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}),
+		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
+			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
+			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
+			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
+		}}),
+	)
+
+	line.Validate()
+	return line
+}
+
+func (s *ScanEventsCharts) TopSlowTemplates(c echo.Context) error {
+	kline := s.topSlowTemplates(c)
+	return kline.Render(c.Response().Writer)
+}
+
+// topSlowTemplates generates a Kline chart showing the top slow templates by time taken
+func (s *ScanEventsCharts) topSlowTemplates(c echo.Context) *charts.Kline {
+	kline := charts.NewKLine()
+	kline.SetCaption(fmt.Sprintf("Chart Shows Top Slow Templates (by time taken) (Top %v)", TopK))
+
+	ids := map[string][]int64{}
+	var startTime time.Time = time.Now()
+	for _, event := range s.data {
+		if event.Time.Before(startTime) {
+			startTime = event.Time
+		}
+	}
+	for _, event := range s.data {
+		ids[event.TemplateID] = append(ids[event.TemplateID], event.Time.Sub(startTime).Milliseconds())
+	}
+
+	type entry struct {
+		ID        string
+		KlineData opts.KlineData
+		start     int64
+		end       int64
+	}
+	data := []entry{}
+
+	for a, b := range ids {
+		if len(b) < 2 {
+			continue // Prevents index out of range error
+		}
+		d := entry{
+			ID:        a,
+			KlineData: opts.KlineData{Value: []int64{b[0], b[len(b)-1], b[0], b[len(b)-1]}}, // Adjusted to prevent index out of range error
+			start:     b[0],
+			end:       b[len(b)-1],
+		}
+		data = append(data, d)
+	}
+
+	sort.Slice(data, func(i, j int) bool {
+		return data[i].end-data[i].start > data[j].end-data[j].start
+	})
+
+	x := make([]string, 0)
+	y := make([]opts.KlineData, 0)
+	for _, event := range data[:TopK] {
+		x = append(x, event.ID)
+		y = append(y, event.KlineData)
+	}
+
+	kline.SetXAxis(x).AddSeries("templates", y)
+	kline.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{Title: fmt.Sprintf("Nuclei: Top %v Slow Templates", TopK)}),
+		charts.WithXAxisOpts(opts.XAxis{
+			Type:      "category",
+			Show:      true,
+			AxisLabel: &opts.AxisLabel{Rotate: 90, Show: true, ShowMinLabel: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (value) { return value; }`)},
+		}),
+		charts.WithYAxisOpts(opts.YAxis{
+			Scale:     true,
+			Type:      "value",
+			Show:      true,
+			AxisLabel: &opts.AxisLabel{Show: true, Formatter: opts.FuncOpts(`function (ms) {  return Math.floor(ms/60000) + 'm' + Math.floor((ms/60000 - Math.floor(ms/60000))*60) + 's'; }`)},
+		}),
+		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
+		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "40%", Top: "10%"}),
+		charts.WithTooltipOpts(opts.Tooltip{Show: true, Trigger: "events.ScanEvent", TriggerOn: "mousemove|click", Enterable: true, Formatter: opts.FuncOpts(`function (params) { return params.name ; }`)}),
+		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
+			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
+			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
+			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
+		}}),
+	)
+
+	return kline
+}
+
+func (s *ScanEventsCharts) RequestsVSInterval(c echo.Context) error {
+	line := s.requestsVSInterval(c)
+	return line.Render(c.Response().Writer)
+}
+
+// requestsVSInterval generates a line chart showing requests per second over time
+func (s *ScanEventsCharts) requestsVSInterval(c echo.Context) *charts.Line {
+	line := charts.NewLine()
+	line.SetCaption("Chart Shows RPS (Requests Per Second) Over Time")
+
+	sort.Slice(s.data, func(i, j int) bool {
+		return s.data[i].Time.Before(s.data[j].Time)
+	})
+
+	var interval time.Duration
+
+	if c != nil {
+		interval, _ = time.ParseDuration(c.QueryParam("interval"))
+	}
+	if interval <= 3 {
+		interval = 5 * time.Second
+	}
+
+	data := []opts.LineData{}
+	temp := 0
+	if len(s.data) > 0 {
+		orig := s.data[0].Time
+		startTime := orig
+		xaxisData := []int64{}
+		for _, v := range s.data {
+			if v.Time.Sub(startTime) > interval {
+				millisec := v.Time.Sub(orig).Milliseconds()
+				xaxisData = append(xaxisData, millisec)
+				data = append(data, opts.LineData{Value: temp, Name: v.Time.Sub(orig).String()})
+				temp = 0
+				startTime = v.Time
+			}
+			temp += 1
+		}
+		// Handle last interval if exists
+		if temp > 0 {
+			millisec := s.data[len(s.data)-1].Time.Sub(orig).Milliseconds()
+			xaxisData = append(xaxisData, millisec)
+			data = append(data, opts.LineData{Value: temp, Name: s.data[len(s.data)-1].Time.Sub(orig).String()})
+		}
+		line.SetXAxis(xaxisData)
+		line.AddSeries("RPS", data, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"}))
+	}
+
+	line.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{Title: "Nuclei: Template Execution", Subtitle: "Time Interval: " + interval.String()}),
+		charts.WithXAxisOpts(opts.XAxis{Name: "Time Intervals", Type: "category", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}),
+		charts.WithYAxisOpts(opts.YAxis{Name: "RPS Value", Type: "value", Show: true}),
+		charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}),
+		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
+		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}),
+		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
+			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
+			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
+			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
+		}}),
+	)
+
+	line.Validate()
+	return line
+}
+
+func (s *ScanEventsCharts) ConcurrencyVsTime(c echo.Context) error {
+	line := s.concurrencyVsTime(c)
+	return line.Render(c.Response().Writer)
+}
+
+// concurrencyVsTime generates a line chart showing concurrency (total workers) over time
+func (s *ScanEventsCharts) concurrencyVsTime(c echo.Context) *charts.Line {
+	line := charts.NewLine()
+	line.SetCaption("Chart Shows Concurrency (Total Workers) Over Time")
+
+	dataset := sliceutil.Clone(s.data)
+
+	sort.Slice(dataset, func(i, j int) bool {
+		return dataset[i].Time.Before(dataset[j].Time)
+	})
+
+	var interval time.Duration
+	if c != nil {
+		interval, _ = time.ParseDuration(c.QueryParam("interval"))
+	}
+	if interval <= 3 {
+		interval = 5 * time.Second
+	}
+
+	// create array with time interval as x-axis and worker count as y-axis
+	// entry is a struct with time and poolsize
+	type entry struct {
+		Time     time.Duration
+		poolsize int
+	}
+	allEntries := []entry{}
+
+	dataIndex := 0
+	maxIndex := len(dataset) - 1
+	currEntry := entry{}
+
+	lastTime := dataset[0].Time
+	for dataIndex <= maxIndex {
+		currTime := dataset[dataIndex].Time
+		if currTime.Sub(lastTime) > interval {
+			// next batch
+			currEntry.Time = interval
+			allEntries = append(allEntries, currEntry)
+			lastTime = dataset[dataIndex-1].Time
+		}
+		if dataset[dataIndex].EventType == events.ScanStarted {
+			currEntry.poolsize += 1
+		} else {
+			currEntry.poolsize -= 1
+		}
+		dataIndex += 1
+	}
+
+	plotData := []opts.LineData{}
+	xaxisData := []int64{}
+	tempTime := time.Duration(0)
+	for _, v := range allEntries {
+		tempTime += v.Time
+		plotData = append(plotData, opts.LineData{Value: v.poolsize, Name: tempTime.String()})
+		xaxisData = append(xaxisData, tempTime.Milliseconds())
+	}
+	line.SetXAxis(xaxisData)
+	line.AddSeries("Concurrency", plotData, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"}))
+
+	line.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{Title: "Nuclei: WorkerPool", Subtitle: "Time Interval: " + interval.String()}),
+		charts.WithXAxisOpts(opts.XAxis{Name: "Time Intervals", Type: "category", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}),
+		charts.WithYAxisOpts(opts.YAxis{Name: "Total Workers", Type: "value", Show: true}),
+		charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}),
+		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
+		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}),
+		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
+			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
+			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
+			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
+		}}),
+	)
+
+	line.Validate()
+	return line
+}
+
+// getCategoryRequestCount returns a map of category and request count
+func getCategoryRequestCount(values []events.ScanEvent) map[string][]events.ScanEvent {
+	mx := make(map[string][]events.ScanEvent)
+	for _, event := range values {
+		mx[event.TemplateType] = append(mx[event.TemplateType], event)
+	}
+	return mx
+}

pkg/scan/events/scan_noop.go

@@ -0,0 +1,11 @@
+//go:build !stats
+// +build !stats
+
+package events
+
+// AddScanEvent is a no-op function
+func AddScanEvent(event ScanEvent) {
+}
+
+func InitWithConfig(config *ScanConfig, statsDirectory string) {
+}

pkg/scan/events/stats_build.go

@@ -0,0 +1,80 @@
+//go:build stats
+// +build stats
+
+package events
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+)
+
+var _ ScanEventWorker = &ScanStatsWorker{}
+
+var defaultWorker = &ScanStatsWorker{}
+
+// ScanStatsWorker is a worker for scanning stats
+// This tracks basic stats in jsonlines format
+// in given directory or a default directory with name stats_{timestamp} in the current directory
+type ScanStatsWorker struct {
+	config    *ScanConfig
+	m         *sync.Mutex
+	directory string
+	enc       *json.Encoder
+}
+
+// Init initializes the scan stats worker
+func InitWithConfig(config *ScanConfig, statsDirectory string) {
+	currentTime := time.Now().Format("20060102150405")
+	dirName := fmt.Sprintf("nuclei-stats-%s", currentTime)
+	err := os.Mkdir(dirName, 0755)
+	if err != nil {
+		panic(err)
+	}
+	// save the config to the directory
+	bin, err := json.MarshalIndent(config, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+	err = os.WriteFile(filepath.Join(dirName, ConfigFile), bin, 0755)
+	if err != nil {
+		panic(err)
+	}
+	defaultWorker = &ScanStatsWorker{config: config, m: &sync.Mutex{}, directory: dirName}
+	err = defaultWorker.initEventsFile()
+	if err != nil {
+		panic(err)
+	}
+}
+
+// initEventsFile initializes the events file for the worker
+func (s *ScanStatsWorker) initEventsFile() error {
+	f, err := os.Create(filepath.Join(s.directory, EventsFile))
+	if err != nil {
+		return err
+	}
+	s.enc = json.NewEncoder(f)
+	return nil
+}
+
+// AddScanEvent adds a scan event to the worker
+func (s *ScanStatsWorker) AddScanEvent(event ScanEvent) {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	err := s.enc.Encode(event)
+	if err != nil {
+		panic(err)
+	}
+}
+
+// AddScanEvent adds a scan event to the worker
+func AddScanEvent(event ScanEvent) {
+	if defaultWorker == nil {
+		return
+	}
+	defaultWorker.AddScanEvent(event)
+}

pkg/scan/events/utils.go

@@ -0,0 +1,45 @@
+package events
+
+import (
+	"time"
+)
+
+type ScanEventWorker interface {
+	// AddScanEvent adds a scan event to the worker
+	AddScanEvent(event ScanEvent)
+}
+
+// Track scan start / finish status
+type ScanStatus string
+
+const (
+	ScanStarted  ScanStatus = "scan_start"
+	ScanFinished ScanStatus = "scan_end"
+)
+
+const (
+	ConfigFile = "config.json"
+	EventsFile = "events.jsonl"
+)
+
+// ScanEvent represents a single scan event with its metadata
+type ScanEvent struct {
+	Target       string     `json:"target" yaml:"target"`
+	TemplateType string     `json:"template_type" yaml:"template_type"`
+	TemplateID   string     `json:"template_id" yaml:"template_id"`
+	TemplatePath string     `json:"template_path" yaml:"template_path"`
+	MaxRequests  int        `json:"max_requests" yaml:"max_requests"`
+	Time         time.Time  `json:"time" yaml:"time"`
+	EventType    ScanStatus `json:"event_type" yaml:"event_type"`
+}
+
+// ScanConfig is only in context of scan event analysis
+type ScanConfig struct {
+	Name                string `json:"name" yaml:"name"`
+	TargetCount         int    `json:"target_count" yaml:"target_count"`
+	TemplatesCount      int    `json:"templates_count" yaml:"templates_count"`
+	TemplateConcurrency int    `json:"template_concurrency" yaml:"template_concurrency"`
+	PayloadConcurrency  int    `json:"payload_concurrency" yaml:"payload_concurrency"`
+	JsConcurrency       int    `json:"js_concurrency" yaml:"js_concurrency"`
+	Retries             int    `json:"retries" yaml:"retries"`
+}

pkg/scan/scan_context.go

@@ -10,6 +10,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 )
 
+
 type ScanContextOption func(*ScanContext)
 
 func WithEvents() ScanContextOption {