Merge pull request #4983 from projectdiscovery/dev

v3.2.3

.github/workflows/build-test.yml

@@ -66,7 +66,3 @@ jobs:
       - name: Example SDK Advanced
         run: go run .
         working-directory: examples/advanced/
-
-      - name: Example SDK with speed control
-        run: go run .
-        working-directory: examples/with_speed_control/

.gitignore

@@ -36,8 +36,5 @@ pkg/protocols/headless/engine/.cache
 /fuzzplayground
 integration_tests/fuzzplayground
 /dsl.md
-/nuclei-stats
-/nuclei-stats-*
-/scan-charts
 
 

Makefile

@@ -11,18 +11,9 @@ ifneq ($(shell go env GOOS),darwin)
 LDFLAGS := -extldflags "-static"
 endif
     
-.PHONY: all build build-stats scan-charts docs test integration functional tidy devtools jsupdate ts fuzzplayground memogen dsl-docs
-
 all: build
 build:
-	rm -f nuclei 2>/dev/null
 	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "nuclei" cmd/nuclei/main.go
-build-stats:
-	rm -f nuclei-stats 2>/dev/null
-	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -tags=stats -o "nuclei-stats" cmd/nuclei/main.go
-scan-charts:
-	rm -f scan-charts 2>/dev/null
-	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scan-charts" cmd/scan-charts/main.go
 docs:
 	if ! which dstdocgen > /dev/null; then
 		echo -e "Command not found! Install? (y/n) \c"

README.md

@@ -22,7 +22,7 @@
   <a href="#install-nuclei">Install</a> •
   <a href="https://docs.projectdiscovery.io/tools/nuclei/">Documentation</a> •
   <a href="#credits">Credits</a> •
-  <a href="https://docs.projectdiscovery.io/tools/nuclei/faq">FAQs</a> •
+  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">FAQs</a> •
   <a href="https://discord.gg/projectdiscovery">Join Discord</a>
 </p>
 

README_CN.md

@@ -23,8 +23,8 @@
   <a href="#对于安全工程师">对于安全工程师</a> •
   <a href="#对于开发和组织">对于开发者</a> •
   <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">文档</a> •
-  <a href="#致谢">致谢</a> •
-  <a href="https://docs.projectdiscovery.io/tools/nuclei/faq">常见问题</a> •
+  <a href="#c致谢">致谢</a> •
+  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">常见问题</a> •
   <a href="https://discord.gg/projectdiscovery">加入Discord</a>
 </p>
 
@@ -39,7 +39,7 @@
 
 Nuclei使用零误报的定制模板向目标发送请求，同时可以对主机进行批量快速扫描。Nuclei提供TCP、DNS、HTTP、FILE等各类协议的扫描，通过强大且灵活的模板，可以使用Nuclei模拟各种安全检查。
 
-我们的[模板仓库](https://github.com/projectdiscovery/nuclei-templates)包含**超过300名**安全研究员和工程师提供的模板。
+我们的[模板仓库](https://github.com/projectdiscovery/nuclei-templates)包含**超过300**安全研究员和工程师提供的模板。
 
 
 
@@ -439,7 +439,7 @@ Nuclei构建很简单，通过数百名安全研究员的社区模板，Nuclei
 </a>
 </p>
 
-另外您可以了解其他类似的开源项目：
+另外您可以其他类似的开源项目：
 
 [FFuF](https://github.com/ffuf/ffuf), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [Jaeles](https://github.com/jaeles-project/jaeles), [ChopChop](https://github.com/michelin/ChopChop)
 

README_ID.md

@@ -24,7 +24,7 @@
   <a href="#untuk-pengembang-dan-organisasi">Untuk Pengembang</a> •
   <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">Dokumentasi</a> •
   <a href="#kredit">Kredit</a> •
-  <a href="https://docs.projectdiscovery.io/tools/nuclei/faq">Tanya Jawab</a> •
+  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">Tanya Jawab</a> •
   <a href="https://discord.gg/projectdiscovery">Gabung Discord</a>
 </p>
 

README_KR.md

@@ -23,7 +23,7 @@
   <a href="#개발자를-위한">개발자를 위한</a> •
   <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">문서</a> •
   <a href="#credits">Credits</a> •
-  <a href="https://docs.projectdiscovery.io/tools/nuclei/faq">FAQs</a> •
+  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">FAQs</a> •
   <a href="https://discord.gg/projectdiscovery"> Discord 참가</a>
 </p>
 

SYNTAX-REFERENCE.md

@@ -2750,8 +2750,6 @@ Enum Values:
   - <code>TLSA</code>
 
   - <code>ANY</code>
-
-  - <code>SRV</code>
 </div>
 
 <hr />

cmd/docgen/docgen.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"log"
 	"os"
-	"reflect"
 	"regexp"
 	"strings"
 
@@ -34,12 +33,6 @@ func main() {
 
 	// Generate jsonschema
 	r := &jsonschema.Reflector{}
-	r.Namer = func(r reflect.Type) string {
-		if r.Kind() == reflect.Slice {
-			return ""
-		}
-		return r.String()
-	}
 	jsonschemaData := r.Reflect(&templates.Template{})
 
 	var buf bytes.Buffer

cmd/integration-test/dns.go

@@ -5,12 +5,7 @@ import (
 )
 
 var dnsTestCases = []TestCaseInfo{
-	{Path: "protocols/dns/a.yaml", TestCase: &dnsBasic{}},
-	{Path: "protocols/dns/aaaa.yaml", TestCase: &dnsBasic{}},
-	{Path: "protocols/dns/cname.yaml", TestCase: &dnsBasic{}},
-	{Path: "protocols/dns/srv.yaml", TestCase: &dnsBasic{}},
-	{Path: "protocols/dns/ns.yaml", TestCase: &dnsBasic{}},
-	{Path: "protocols/dns/txt.yaml", TestCase: &dnsBasic{}},
+	{Path: "protocols/dns/basic.yaml", TestCase: &dnsBasic{}},
 	{Path: "protocols/dns/ptr.yaml", TestCase: &dnsPtr{}},
 	{Path: "protocols/dns/caa.yaml", TestCase: &dnsCAA{}},
 	{Path: "protocols/dns/tlsa.yaml", TestCase: &dnsTLSA{}},

cmd/integration-test/workflow.go

@@ -3,15 +3,11 @@ package main
 import (
 	"fmt"
 	"io"
-	"log"
 	"net/http"
 	"net/http/httptest"
-	"strings"
 
 	"github.com/julienschmidt/httprouter"
 
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
 )
 
@@ -20,40 +16,11 @@ var workflowTestcases = []TestCaseInfo{
 	{Path: "workflow/condition-matched.yaml", TestCase: &workflowConditionMatched{}},
 	{Path: "workflow/condition-unmatched.yaml", TestCase: &workflowConditionUnmatch{}},
 	{Path: "workflow/matcher-name.yaml", TestCase: &workflowMatcherName{}},
-	{Path: "workflow/complex-conditions.yaml", TestCase: &workflowComplexConditions{}},
 	{Path: "workflow/http-value-share-workflow.yaml", TestCase: &workflowHttpKeyValueShare{}},
 	{Path: "workflow/dns-value-share-workflow.yaml", TestCase: &workflowDnsKeyValueShare{}},
-	{Path: "workflow/code-value-share-workflow.yaml", TestCase: &workflowCodeKeyValueShare{}, DisableOn: isCodeDisabled}, // isCodeDisabled declared in code.go
-	{Path: "workflow/multiprotocol-value-share-workflow.yaml", TestCase: &workflowMultiProtocolKeyValueShare{}},
 	{Path: "workflow/shared-cookie.yaml", TestCase: &workflowSharedCookies{}},
 }
 
-func init() {
-	// sign code templates (unless they are disabled)
-	if !isCodeDisabled() {
-		// allow local file access to load content of file references in template
-		// in order to sign them for testing purposes
-		templates.TemplateSignerLFA()
-
-		// testCertFile and testKeyFile are declared in code.go
-		tsigner, err := signer.NewTemplateSignerFromFiles(testCertFile, testKeyFile)
-		if err != nil {
-			panic(err)
-		}
-
-		// only the code templates are necessary to be signed
-		var templatesToSign = []string{
-			"workflow/code-template-1.yaml",
-			"workflow/code-template-2.yaml",
-		}
-		for _, templatePath := range templatesToSign {
-			if err := templates.SignTemplate(tsigner, templatePath); err != nil {
-				log.Fatalf("Could not sign template %v got: %s\n", templatePath, err)
-			}
-		}
-	}
-}
-
 type workflowBasic struct{}
 
 // Execute executes a test case and returns an error if occurred
@@ -130,30 +97,6 @@ func (h *workflowMatcherName) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
-type workflowComplexConditions struct{}
-
-// Execute executes a test case and returns an error if occurred
-func (h *workflowComplexConditions) Execute(filePath string) error {
-	router := httprouter.New()
-	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-		fmt.Fprintf(w, "This is test matcher text")
-	})
-	ts := httptest.NewServer(router)
-	defer ts.Close()
-
-	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
-	if err != nil {
-		return err
-	}
-
-	for _, result := range results {
-		if !strings.Contains(result, "test-matcher-3") {
-			return fmt.Errorf("incorrect result: the \"basic-get-third:test-matcher-3\" and only that should be matched!\nResults:\n\t%s", strings.Join(results, "\n\t"))
-		}
-	}
-	return expectResultsCount(results, 2)
-}
-
 type workflowHttpKeyValueShare struct{}
 
 // Execute executes a test case and returns an error if occurred
@@ -190,45 +133,6 @@ func (h *workflowDnsKeyValueShare) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
-type workflowCodeKeyValueShare struct{}
-
-// Execute executes a test case and returns an error if occurred
-func (h *workflowCodeKeyValueShare) Execute(filePath string) error {
-	// provide the Certificate File that the code templates are signed with
-	certEnvVar := signer.CertEnvVarName + "=" + testCertFile
-
-	results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, []string{certEnvVar}, "-workflows", filePath, "-target", "input", "-code")
-	if err != nil {
-		return err
-	}
-
-	return expectResultsCount(results, 1)
-}
-
-type workflowMultiProtocolKeyValueShare struct{}
-
-// Execute executes a test case and returns an error if occurred
-func (h *workflowMultiProtocolKeyValueShare) Execute(filePath string) error {
-	router := httprouter.New()
-	// the response of path1 contains a domain that will be extracted and shared with the second template
-	router.GET("/path1", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-		fmt.Fprintf(w, "href=\"blog.projectdiscovery.io\"")
-	})
-	// path2 responds with the value of the "extracted" query parameter, e.g.: /path2?extracted=blog.projectdiscovery.io => blog.projectdiscovery.io
-	router.GET("/path2", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-		fmt.Fprintf(w, "%s", r.URL.Query().Get("extracted"))
-	})
-	ts := httptest.NewServer(router)
-	defer ts.Close()
-
-	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
-	if err != nil {
-		return err
-	}
-
-	return expectResultsCount(results, 2)
-}
-
 type workflowSharedCookies struct{}
 
 // Execute executes a test case and returns an error if occurred

cmd/nuclei/main.go

@@ -15,7 +15,6 @@ import (
 	"github.com/projectdiscovery/utils/auth/pdcp"
 	"github.com/projectdiscovery/utils/env"
 	_ "github.com/projectdiscovery/utils/pprof"
-	stringsutil "github.com/projectdiscovery/utils/strings"
 
 	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
@@ -330,15 +329,13 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	flagSet.CreateGroup("rate-limit", "Rate-Limit",
 		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
-		flagSet.DurationVarP(&options.RateLimitDuration, "rate-limit-duration", "rld", time.Second, "maximum number of requests to send per second"),
-		flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute (DEPRECATED)"),
+		flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"),
 		flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
 		flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"),
 		flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"),
 		flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "headc", 10, "maximum number of headless templates to be executed in parallel"),
 		flagSet.IntVarP(&options.JsConcurrency, "js-concurrency", "jsc", 120, "maximum number of javascript runtimes to be executed in parallel"),
 		flagSet.IntVarP(&options.PayloadConcurrency, "payload-concurrency", "pc", 25, "max payload concurrency for each template"),
-		flagSet.IntVarP(&options.ProbeConcurrency, "probe-concurrency", "prc", 50, "http probe concurrency with httpx"),
 	)
 	flagSet.CreateGroup("optimization", "Optimizations",
 		flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait in seconds before timeout"),
@@ -415,11 +412,6 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.PreFetchSecrets, "prefetch-secrets", "ps", false, "prefetch secrets from the secrets file"),
 	)
 
-	flagSet.CreateGroup("tests", "Tests",
-		flagSet.BoolVarP(&options.AutogenerateTests, "autogenerate-tests", "agt", false, "enable autogeneration of nuclei test-cases (http only now)"),
-		flagSet.BoolVarP(&options.RunNucleiTests, "run-tests", "rts", false, "run generated nuclei test-cases"),
-	)
-
 	flagSet.SetCustomHelpText(`EXAMPLES:
 Run nuclei on single host:
 	$ nuclei -target example.com
@@ -605,10 +597,10 @@ Note: Make sure you have backup of your custom nuclei-templates before proceedin
 			gologger.Fatal().Msgf("could not read response: %s", err)
 		}
 		resp = strings.TrimSpace(resp)
-		if stringsutil.EqualFoldAny(resp, "y", "yes") {
+		if strings.EqualFold(resp, "y") || strings.EqualFold(resp, "yes") {
 			break
 		}
-		if stringsutil.EqualFoldAny(resp, "n", "no", "") {
+		if strings.EqualFold(resp, "n") || strings.EqualFold(resp, "no") || resp == "" {
 			fmt.Println("Exiting...")
 			os.Exit(0)
 		}

cmd/nuclei/srv.yaml

@@ -1,18 +0,0 @@
-id: basic-dns-a-example
-
-info:
-  name: Test DNS A Query Template
-  author: pdteam
-  severity: info
-
-dns:
-  - name: "{{FQDN}}"
-    type: SRV
-    class: inet
-    recursion: true
-    retries: 3
-    matchers:
-      - type: word
-        part: all
-        words:
-          - "SRV"

cmd/scan-charts/main.go

@@ -1,40 +0,0 @@
-package main
-
-import (
-	"flag"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/scan/charts"
-)
-
-var (
-	dir     string
-	address string
-	output  string
-)
-
-func main() {
-	flag.StringVar(&dir, "dir", "", "directory to scan")
-	flag.StringVar(&address, "address", ":9000", "address to run the server on")
-	flag.StringVar(&output, "output", "", "output filename of generated html file")
-	flag.Parse()
-
-	if dir == "" {
-		flag.Usage()
-		return
-	}
-
-	server, err := charts.NewScanEventsCharts(dir)
-	if err != nil {
-		panic(err)
-	}
-	server.PrintInfo()
-
-	if output != "" {
-		if err = server.GenerateHTML(output); err != nil {
-			panic(err)
-		}
-		return
-	}
-
-	server.Start(address)
-}

cmd/tmc/main.go

@@ -231,7 +231,7 @@ func logErrMsg(path string, err error, debug bool, errFile *os.File) string {
 	return msg
 }
 
-// enhanceTemplate enhances template data using templateman
+// enhanceTemplateData enhances template data using templateman
 // ref: https://github.com/projectdiscovery/templateman/blob/main/templateman-rest-api/README.md#enhance-api
 func enhanceTemplate(data string) (string, bool, error) {
 	resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/enhance", tmBaseUrl), "application/x-yaml", strings.NewReader(data))
@@ -266,7 +266,7 @@ func enhanceTemplate(data string) (string, bool, error) {
 	return data, false, errorutil.New("template enhance failed")
 }
 
-// formatTemplate formats template data using templateman format api
+// formatTemplateData formats template data using templateman format api
 func formatTemplate(data string) (string, bool, error) {
 	resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/format", tmBaseUrl), "application/x-yaml", strings.NewReader(data))
 	if err != nil {

examples/advanced/advanced.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
-	syncutil "github.com/projectdiscovery/utils/sync"
+	"github.com/remeh/sizedwaitgroup"
 )
 
 func main() {
@@ -12,10 +12,7 @@ func main() {
 		panic(err)
 	}
 	// setup sizedWaitgroup to handle concurrency
-	sg, err := syncutil.New(syncutil.WithSize(10))
-	if err != nil {
-		panic(err)
-	}
+	sg := sizedwaitgroup.New(10)
 
 	// scan 1 = run dns templates on scanme.sh
 	sg.Add()

examples/with_speed_control/main.go

@@ -1,104 +0,0 @@
-package main
-
-import (
-	"log"
-	"sync"
-	"time"
-
-	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
-)
-
-func main() {
-	ne, err := initializeNucleiEngine()
-	if err != nil {
-		panic(err)
-	}
-	defer ne.Close()
-
-	ne.LoadTargets([]string{"http://honey.scanme.sh"}, false)
-
-	var wg sync.WaitGroup
-	wg.Add(3)
-
-	go testRateLimit(&wg, ne)
-	go testThreadsAndBulkSize(&wg, ne)
-	go testPayloadConcurrency(&wg, ne)
-
-	err = ne.ExecuteWithCallback(nil)
-	if err != nil {
-		panic(err)
-	}
-
-	wg.Wait()
-}
-
-func initializeNucleiEngine() (*nuclei.NucleiEngine, error) {
-	return nuclei.NewNucleiEngine(
-		nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"oast"}}),
-		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}),
-		nuclei.WithGlobalRateLimit(1, time.Second),
-		nuclei.WithConcurrency(nuclei.Concurrency{
-			TemplateConcurrency:           1,
-			HostConcurrency:               1,
-			HeadlessHostConcurrency:       1,
-			HeadlessTemplateConcurrency:   1,
-			JavascriptTemplateConcurrency: 1,
-			TemplatePayloadConcurrency:    1,
-			ProbeConcurrency:              1,
-		}),
-	)
-}
-
-func testRateLimit(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
-	defer wg.Done()
-	verifyRateLimit(ne, 1, 5000)
-}
-
-func testThreadsAndBulkSize(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
-	defer wg.Done()
-	initialTemplateThreads, initialBulkSize := 1, 1
-	verifyThreadsAndBulkSize(ne, initialTemplateThreads, initialBulkSize, 25, 25)
-}
-
-func testPayloadConcurrency(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
-	defer wg.Done()
-	verifyPayloadConcurrency(ne, 1, 500)
-}
-
-func verifyRateLimit(ne *nuclei.NucleiEngine, initialRate, finalRate int) {
-	if ne.GetExecuterOptions().RateLimiter.GetLimit() != uint(initialRate) {
-		panic("wrong initial rate limit")
-	}
-	time.Sleep(5 * time.Second)
-	ne.Options().RateLimit = finalRate
-	time.Sleep(20 * time.Second)
-	if ne.GetExecuterOptions().RateLimiter.GetLimit() != uint(finalRate) {
-		panic("wrong final rate limit")
-	}
-}
-
-func verifyThreadsAndBulkSize(ne *nuclei.NucleiEngine, initialThreads, initialBulk, finalThreads, finalBulk int) {
-	if ne.Options().TemplateThreads != initialThreads || ne.Options().BulkSize != initialBulk {
-		panic("wrong initial standard concurrency")
-	}
-	time.Sleep(5 * time.Second)
-	ne.Options().TemplateThreads = finalThreads
-	ne.Options().BulkSize = finalBulk
-	time.Sleep(20 * time.Second)
-	if ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size != finalBulk || ne.Engine().WorkPool().Default.Size != finalThreads {
-		log.Fatal("wrong final concurrency", ne.Engine().WorkPool().Default.Size, finalThreads, ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size, finalBulk)
-	}
-}
-
-func verifyPayloadConcurrency(ne *nuclei.NucleiEngine, initialPayloadConcurrency, finalPayloadConcurrency int) {
-	if ne.Options().PayloadConcurrency != initialPayloadConcurrency {
-		panic("wrong initial payload concurrency")
-	}
-	time.Sleep(5 * time.Second)
-	ne.Options().PayloadConcurrency = finalPayloadConcurrency
-	time.Sleep(20 * time.Second)
-	if ne.GetExecuterOptions().GetThreadsForNPayloadRequests(100, 0) != finalPayloadConcurrency {
-		panic("wrong final payload concurrency")
-	}
-}

go.mod

@@ -20,12 +20,12 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/fastdialer v0.0.67
+	github.com/projectdiscovery/fastdialer v0.0.64
 	github.com/projectdiscovery/hmap v0.0.41
 	github.com/projectdiscovery/interactsh v1.1.9
-	github.com/projectdiscovery/rawhttp v0.1.45
+	github.com/projectdiscovery/rawhttp v0.1.41
 	github.com/projectdiscovery/retryabledns v1.0.58
-	github.com/projectdiscovery/retryablehttp-go v1.0.55
+	github.com/projectdiscovery/retryablehttp-go v1.0.54
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -38,7 +38,7 @@ require (
 	github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222
 	github.com/xanzy/go-gitlab v0.84.0
 	go.uber.org/multierr v1.11.0
-	golang.org/x/net v0.24.0
+	golang.org/x/net v0.21.0
 	golang.org/x/oauth2 v0.11.0
 	golang.org/x/text v0.14.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -70,7 +70,6 @@ require (
 	github.com/go-ldap/ldap/v3 v3.4.5
 	github.com/go-pg/pg v8.0.7+incompatible
 	github.com/go-sql-driver/mysql v1.7.1
-	github.com/goccy/go-yaml v1.11.3
 	github.com/h2non/filetype v1.1.3
 	github.com/labstack/echo/v4 v4.10.2
 	github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa
@@ -79,32 +78,29 @@ require (
 	github.com/mholt/archiver v3.1.1+incompatible
 	github.com/ory/dockertest/v3 v3.10.0
 	github.com/praetorian-inc/fingerprintx v1.1.9
-	github.com/projectdiscovery/dsl v0.0.52
+	github.com/projectdiscovery/dsl v0.0.50
 	github.com/projectdiscovery/fasttemplate v0.0.2
-	github.com/projectdiscovery/freeport v0.0.5
 	github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
-	github.com/projectdiscovery/goflags v0.1.48
+	github.com/projectdiscovery/goflags v0.1.46
 	github.com/projectdiscovery/gologger v1.1.12
 	github.com/projectdiscovery/gostruct v0.0.2
 	github.com/projectdiscovery/gozero v0.0.2
 	github.com/projectdiscovery/httpx v1.6.0
 	github.com/projectdiscovery/mapcidr v1.1.16
-	github.com/projectdiscovery/martian/v3 v3.0.0-20240219194442-fed3b744f477
 	github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
-	github.com/projectdiscovery/proxify v0.0.13-0.20240407134611-de309e8af9ca
 	github.com/projectdiscovery/ratelimit v0.0.35
 	github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
 	github.com/projectdiscovery/useragent v0.0.40
-	github.com/projectdiscovery/utils v0.0.88
+	github.com/projectdiscovery/utils v0.0.87
 	github.com/projectdiscovery/wappalyzergo v0.0.116
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
 	github.com/stretchr/testify v1.9.0
 	github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
-	golang.org/x/term v0.19.0
+	golang.org/x/term v0.17.0
 	gopkg.in/yaml.v3 v3.0.1
 	moul.io/http2curl v1.0.0
 )
@@ -118,10 +114,8 @@ require (
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
-	github.com/Shopify/sarama v1.38.1 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/andybalholm/brotli v1.1.0 // indirect
-	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
@@ -148,12 +142,7 @@ require (
 	github.com/docker/cli v24.0.5+incompatible // indirect
 	github.com/docker/docker v24.0.9+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/eapache/channels v1.1.0 // indirect
-	github.com/eapache/go-resiliency v1.3.0 // indirect
-	github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
-	github.com/eapache/queue v1.1.0 // indirect
-	github.com/elastic/go-elasticsearch/v7 v7.17.10 // indirect
-	github.com/fatih/color v1.16.0 // indirect
+	github.com/fatih/color v1.15.0 // indirect
 	github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/geoffgarside/ber v1.1.0 // indirect
@@ -172,12 +161,9 @@ require (
 	github.com/google/go-github/v30 v30.1.0 // indirect
 	github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.6 // indirect
-	github.com/haxii/fastproxy v0.5.37 // indirect
 	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect
 	github.com/invopop/yaml v0.2.0 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
@@ -212,13 +198,11 @@ require (
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/projectdiscovery/asnmap v1.1.0 // indirect
 	github.com/projectdiscovery/cdncheck v1.0.9 // indirect
+	github.com/projectdiscovery/freeport v0.0.5 // indirect
 	github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
 	github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
-	github.com/projectdiscovery/roundrobin v0.0.6 // indirect
 	github.com/projectdiscovery/stringsutil v0.0.2 // indirect
-	github.com/projectdiscovery/tinydns v0.0.31 // indirect
 	github.com/quic-go/quic-go v0.42.0 // indirect
-	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/refraction-networking/utls v1.6.1 // indirect
 	github.com/sashabaranov/go-openai v1.15.3 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
@@ -248,7 +232,6 @@ require (
 	github.com/zeebo/blake3 v0.2.3 // indirect
 	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
 	mellium.im/sasl v0.3.1 // indirect
 )
@@ -317,10 +300,10 @@ require (
 	go.etcd.io/bbolt v1.3.8 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	goftp.io/server/v2 v2.0.1 // indirect
-	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
 	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.17.0
 	google.golang.org/appengine v1.6.7 // indirect
@@ -344,7 +327,6 @@ require (
 	github.com/aws/smithy-go v1.13.5 // indirect
 	github.com/dop251/goja_nodejs v0.0.0-20230821135201-94e508132562
 	github.com/emirpasic/gods v1.18.1 // indirect
-	github.com/go-echarts/go-echarts/v2 v2.3.3
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
@@ -364,5 +346,3 @@ require (
 
 // https://go.dev/ref/mod#go-mod-file-retract
 retract v3.2.0 // retract due to broken js protocol issue
-
-replace github.com/go-echarts/go-echarts/v2 => github.com/tarunKoyalwar/go-echarts/v2 v2.1.1

go.sum

@@ -85,12 +85,7 @@ github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAc
 github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
 github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
-github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A=
-github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g=
-github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
-github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc=
-github.com/Shopify/toxiproxy/v2 v2.5.0/go.mod h1:yhM2epWtAmel9CB8r2+L+PCmhH6yH2pITaPAo7jxJl0=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
@@ -301,19 +296,10 @@ github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj6
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
-github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
-github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0=
-github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
-github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 h1:8yY/I9ndfrgrXUbOGObLHKBR4Fl3nZXwM2c7OYTT8hM=
-github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
-github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
-github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
-github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
@@ -324,12 +310,10 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
-github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
+github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
-github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
-github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
@@ -420,8 +404,6 @@ github.com/gobwas/ws v1.2.1 h1:F2aeBZrm2NDsc7vbovKrWSogd4wvfAxg0FQ89/iqOTk=
 github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I=
-github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
 github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -541,8 +523,6 @@ github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy
 github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
 github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
-github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
@@ -551,8 +531,6 @@ github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrj
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
-github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
 github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
@@ -576,9 +554,6 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/haxii/fastproxy v0.5.37 h1:grfso8V9sNO8jZjI/vkizFXzz8fE/yrsgl+XxTb3fio=
-github.com/haxii/fastproxy v0.5.37/go.mod h1:VFy3M4EmTbeKu+IccQ6UiJ1W4sPeQ75GV+1JDa8h864=
-github.com/haxii/log v1.0.0/go.mod h1:y9MlOm+u2ny65yQxScWfSGZFOhRVLXz2vJlkiIx2jfI=
 github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk=
 github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA=
 github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A=
@@ -854,18 +829,18 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO
 github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs=
 github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4=
 github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
-github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE=
-github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI=
-github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E=
-github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84=
+github.com/projectdiscovery/dsl v0.0.50 h1:4SuAwTS9l6o1tqlIC/79+EcUwTM6CjaU7MpY/nDlFaM=
+github.com/projectdiscovery/dsl v0.0.50/go.mod h1:6g740l4tH4d2j9UYtIchtxudb0Dphkq4o+VatpR4M6g=
+github.com/projectdiscovery/fastdialer v0.0.64 h1:xivkA4g14nwQElOVsxPkGMWsdcYPcp7DPhVjvI6yQkw=
+github.com/projectdiscovery/fastdialer v0.0.64/go.mod h1:S/7PAQRmVDYRaU7u4xXD0qA5a48NAZq2JcpcVoEVrlo=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
 github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
-github.com/projectdiscovery/goflags v0.1.48 h1:iJgq+YuQOEm0V7BqlV/0wx7GaI2rmGiFzPkvL/4sCj0=
-github.com/projectdiscovery/goflags v0.1.48/go.mod h1:C9Qqyivehk5iYCf3VAC3+85y60Qbde1ZBMqe5wHhcVw=
+github.com/projectdiscovery/goflags v0.1.46 h1:JlYvFxJcimKJGWYbygiFBN052MWrbls/kKiwOKpLzEE=
+github.com/projectdiscovery/goflags v0.1.46/go.mod h1:X7A6ELNgczyOyEy2gyNC/tJTuhtwQk6ZLyzsnDVlZkw=
 github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A=
 github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
 github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
@@ -884,40 +859,32 @@ github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScL
 github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
 github.com/projectdiscovery/mapcidr v1.1.16 h1:rjj1w5D6hbTsUQXYClLcGdfBEy9bryclgi70t0vBggo=
 github.com/projectdiscovery/mapcidr v1.1.16/go.mod h1:rGqpBhStdwOQ2uS62QM9qPsybwMwIhT7CTd2bxoHs8Q=
-github.com/projectdiscovery/martian/v3 v3.0.0-20240219194442-fed3b744f477 h1:VJaBELAC5Hw+kc+ylrFF5nSf7Wasnb9mZxMlF/kR3gg=
-github.com/projectdiscovery/martian/v3 v3.0.0-20240219194442-fed3b744f477/go.mod h1:wPvVUl2C/XOFacugXwsUp65GN0upmKfwKMyimA/AAaM=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
 github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8=
 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
-github.com/projectdiscovery/proxify v0.0.13-0.20240407134611-de309e8af9ca h1:Adduz+Lli837Ga2K1NqxVFOFxRYsWjK4zRWzHQWZtV4=
-github.com/projectdiscovery/proxify v0.0.13-0.20240407134611-de309e8af9ca/go.mod h1:9x1db4kybvc6BpSvpvbMzAuWjNFrt6Foy1t1VI7CUTc=
 github.com/projectdiscovery/ratelimit v0.0.35 h1:epEzFATOcXZ4tssV4Hax5Op9lrbUnQMEGMV5PoUpTKc=
 github.com/projectdiscovery/ratelimit v0.0.35/go.mod h1:mPqa8UpV5I7eAN5/ZcsjLiXMhjtVvZRrHtpBRsTPuyA=
-github.com/projectdiscovery/rawhttp v0.1.45 h1:5jssUybhRz2ljI3bW4kUO9MTvZ37ArG8pISYtgqzwBg=
-github.com/projectdiscovery/rawhttp v0.1.45/go.mod h1:oLWuZkyKi4rddPoLEN3TeAqyLvFNfr3dxmzV2kwpPvs=
+github.com/projectdiscovery/rawhttp v0.1.41 h1:0n6CohOf0Aq7dsXv+ozznhlYr4ANDKLwvPmdzTet3qU=
+github.com/projectdiscovery/rawhttp v0.1.41/go.mod h1:TyVfCwNbAsQSwrMOKu8o1g80AO3t1OnlJx+flgSV/CQ=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
 github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE=
 github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY=
-github.com/projectdiscovery/retryablehttp-go v1.0.55 h1:ADgugnl9jKkNXn5m/Zd8TGPq1P7GplYlqUNKm/qTmls=
-github.com/projectdiscovery/retryablehttp-go v1.0.55/go.mod h1:Kpvh4ruFPOEPYaYxgbFmlvBJr4lJKqpcbGvx1j0r/Ng=
-github.com/projectdiscovery/roundrobin v0.0.6 h1:zoJAFRgP9XK7B+iKSjR+djRAuDYxnc57+Fx+qpoPvds=
-github.com/projectdiscovery/roundrobin v0.0.6/go.mod h1:vTxcWqNLyMH6VE2Q/hsNNvDHFLiIzHozC1rLLT/vocQ=
+github.com/projectdiscovery/retryablehttp-go v1.0.54 h1:lUmQA3obq3Ya3xU1vouKf+hVjbLFKzJCK6FcNKPZ8vQ=
+github.com/projectdiscovery/retryablehttp-go v1.0.54/go.mod h1:J+pg00bYLEgWOZJISi16icHUDbsnkjnA1PmSa2kSMYs=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
 github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
 github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
-github.com/projectdiscovery/tinydns v0.0.31 h1:iQL1ze2/4PHWH+Nzsxg35kVIWkvEIatAZCDcP0c1/v4=
-github.com/projectdiscovery/tinydns v0.0.31/go.mod h1:6rj/dEzXZPn+nLl4O1wUMJdfkdUQ9eCV7NsDWPakYv8=
 github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A=
 github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
 github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A=
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
 github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo=
 github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4=
-github.com/projectdiscovery/utils v0.0.88 h1:oYfCXM+8VHNLyH/H6cOibkuDUwHUAOBAMRNPFX6NPrs=
-github.com/projectdiscovery/utils v0.0.88/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA=
+github.com/projectdiscovery/utils v0.0.87 h1:9+RiTEhpUB/vk6XJUVpysNWJ2aCTD7WuyoyAcNnbIzk=
+github.com/projectdiscovery/utils v0.0.87/go.mod h1:jGK450sL9AVDTjaPwEs9za8NVeEC9xE97IWNoK138kI=
 github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8=
 github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@@ -957,8 +924,6 @@ github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB
 github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM=
 github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
-github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY=
 github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c=
 github.com/refraction-networking/utls v1.6.1 h1:n1JG5karzdGWsI6iZmGrOv3SNzR4c+4M8J6KWGsk3lA=
@@ -976,7 +941,6 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.11.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
@@ -1046,8 +1010,6 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
-github.com/tarunKoyalwar/go-echarts/v2 v2.1.1 h1:5fsXGPmK+i18J8cDgxy7AJkiXWBARpVTb0Gbv+bAzPo=
-github.com/tarunKoyalwar/go-echarts/v2 v2.1.1/go.mod h1:VEeyPT5Odx/UHeuxtIAHGu2+87MWGA5OBaZ120NFi/w=
 github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
 github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
 github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
@@ -1221,8 +1183,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
 golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
-golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
-golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1315,8 +1277,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
-golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1417,9 +1379,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1431,8 +1392,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
-golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
-golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
+golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1512,8 +1473,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
-golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
@@ -1570,8 +1529,6 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20220706185917-7780775163c4 h1:7YDGQC/0sigNGzsEWyb9s72jTxlFdwVEYNJHbfQ+Dtg=
-google.golang.org/genproto v0.0.0-20220706185917-7780775163c4/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
@@ -1589,8 +1546,6 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w=
-google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

integration_tests/protocols/code/pre-condition.yaml

@@ -23,4 +23,4 @@ code:
       - type: dsl
         dsl:
           - true
-# digest: 490a004630440220192fb8f704b078c2885047b85ac1a0491be86485c033a976d201599683a35aab0220604b1c3781e9d97079d0e5c23c18e6a2d87493c8e2b930536e692ee7d06e9247:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a00473045022100c7215ce9f11e6a51c193bb54643a05cdd1cde18a3abb6c9983c5c7524d3ff03002203d93581c81d3ad5db463570cbbd2bdee529328d32a5b00e037610c211e448cef:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-env-var.yaml

@@ -20,4 +20,4 @@ code:
       - type: word
         words:
           - "hello from input baz"
-# digest: 4a0a00473045022033f72f1b9d5143f58a2dc79c2597000f34080251ac3702c36c3fad00917dfeeb0221009ba05c715c9e2e36dba471be6c0106a09ae3822d8a3e9e4bcf377e9f4a395a01:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a0047304502207e3a5eda5f3207c3c01c820562243281926c1215224a7c80ed7528559b9f52cb022100f6ef99bb45843f481705778630f2cfd8f4d1cc3acb96392ff016f22e06aa91af:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-file.yaml

@@ -18,4 +18,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4a0a004730450220377128cb11d9f6f0fee1f4dbd841e46783de26e90a216fa55a7609ee2bc823c60221009166ee0f85e3a1811588ab19e73ea96ab3d582dc8180dbcbbad0ea9ab7e9025d:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a004730450220069673af9bd6d6677f9529d06f5d8bd46d543089a4731ed18ee806761d75fd60022100913a3e27b0a5809baf710ba9585bf9fe729634c0e19e3e13eef70a6bd100df34:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-interactsh.yaml

@@ -26,4 +26,4 @@ code:
         part: interactsh_protocol
         words:
           - "http"
-# digest: 4b0a00483046022100d472d50bd83117d334f5217c7a40dcdf34138e90029eaace51697d902296bf37022100a393b49420a96f60d6d89b79b5135ee2233b2468d374851890eea114b08195d1:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 490a00463044022003b8d069e3c84412729c43e33013a52ee04eabcf096d511979691d71d8e905f60220011f4475899abed4f86b4bd5e6c2423750759135206e4729826afe1ed8a44f4d:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-snippet.yaml

@@ -21,4 +21,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4b0a004830460221008886054bb5dd6345e434e30f31c8fddce3c484a4f33aa6321b5185675866029d022100d188a83d0fde029f8b586061c65ab72b43755c3fb10fdd59501bb9bbadbb1ff7:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a00473045022100c291615cf2a8005450c17a6554e81a9cdab14743b299f0679c644247929198b502206fdacc8ab173bde2b4015340012637916bf2659f66f320fcc06b97ac639072a1:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/dns/aaaa.yaml

@@ -1,17 +0,0 @@
-id: dns-aaaa-query-example
-
-info:
-  name: Test DNS AAAA Query Template
-  author: pdteam
-  severity: info
-
-dns:
-  - name: "{{FQDN}}"
-    type: AAAA
-    class: inet
-    recursion: true
-    retries: 3
-    matchers:
-      - type: word
-        words:
-          - "2606:4700:4700::1001"

integration_tests/protocols/dns/basic.yaml

@@ -1,7 +1,7 @@
-id: dns-a-query-example
+id: basic-dns-example
 
 info:
-  name: Test DNS A Query Template
+  name: Test DNS Template
   author: pdteam
   severity: info
 
@@ -14,4 +14,4 @@ dns:
     matchers:
       - type: word
         words:
-          - "1.1.1.1"
+          - "1.1.1.1"

integration_tests/protocols/dns/cname.yaml

@@ -1,18 +0,0 @@
-id: dns-cname-query-example
-
-info:
-  name: Test DNS CNAME Query Template
-  author: pdteam
-  severity: info
-
-dns:
-  - name: "{{FQDN}}"
-    type: CNAME
-    class: inet
-    recursion: true
-    retries: 3
-    matchers:
-      - type: word
-        part: all
-        words:
-          - "CNAME"

integration_tests/protocols/dns/ns.yaml

@@ -1,18 +0,0 @@
-id: dns-ns-query-example
-
-info:
-  name: Test DNS NS Query Template
-  author: pdteam
-  severity: info
-
-dns:
-  - name: "{{FQDN}}"
-    type: NS
-    class: inet
-    recursion: true
-    retries: 3
-    matchers:
-      - type: word
-        part: all
-        words:
-          - "NS"

integration_tests/protocols/dns/srv.yaml

@@ -1,18 +0,0 @@
-id: dns-a-query-example
-
-info:
-  name: Test DNS SRV Query Template
-  author: pdteam
-  severity: info
-
-dns:
-  - name: "{{FQDN}}"
-    type: SRV
-    class: inet
-    recursion: true
-    retries: 3
-    matchers:
-      - type: word
-        part: all
-        words:
-          - "SRV"

integration_tests/protocols/dns/txt.yaml

@@ -1,18 +0,0 @@
-id: dns-txt-query-example
-
-info:
-  name: Test DNS TXT Query Template
-  author: pdteam
-  severity: info
-
-dns:
-  - name: "{{FQDN}}"
-    type: TXT
-    class: inet
-    recursion: true
-    retries: 3
-    matchers:
-      - type: word
-        part: all
-        words:
-          - "TXT"

integration_tests/workflow/code-template-1.yaml

@@ -1,22 +0,0 @@
-id: code-template-1
-
-info:
-  name: code-template-1
-  author: tovask
-  severity: info
-  tags: code
-
-code:
-  - engine:
-      - py
-      - python3
-      - python
-    source: |
-      print("hello from first")
-    extractors:
-      - type: regex
-        name: extracted
-        regex:
-          - 'hello from (.*)'
-        group: 1
-# digest: 490a0046304402202c63d47bb0acdd40b3b852d95490d492ff5741b84071b2a8a40371be7797c13602202b6b977e157edf2ef70a402a2e57d4eb5a67c5ca91f0a2f9a10a966e8485ebaf:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/workflow/code-template-2.yaml

@@ -1,21 +0,0 @@
-id: code-template-2
-
-info:
-  name: code-template-2
-  author: tovask
-  severity: info
-  tags: code
-
-code:
-  - engine:
-      - py
-      - python3
-      - python
-    source: |
-      import os
-      print("hello from " + os.getenv("extracted"))
-    matchers:
-      - type: word
-        words:
-          - "hello from first"
-# digest: 490a00463044022025661eab353b7f359c0d428a86b6287545d7f759375e8025cc8c9c77b616ca6502200bc2c019059622df3c88e7caa6dd7d1fb9b956010aa0de2ee2b9f7dd0a3c4954:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/workflow/code-value-share-workflow.yaml

@@ -1,12 +0,0 @@
-id: code-value-sharing-workflow
-
-info:
-  name: Code Value Sharing Workflow
-  author: tovask
-  severity: info
-  tags: code
-
-workflows:
-  - template: workflow/code-template-1.yaml
-    subtemplates:
-      - template: workflow/code-template-2.yaml

integration_tests/workflow/complex-conditions.yaml

@@ -1,23 +0,0 @@
-id: complex-conditions-workflow
-
-info:
-  name: Complex Conditions Workflow
-  author: tovask
-  severity: info
-  description: Workflow to test a complex scenario, e.g. race conditions when evaluating the results of the templates
-
-workflows:
-  - template: workflow/match-1.yaml
-    subtemplates:
-      - template: workflow/nomatch-1.yaml
-        subtemplates:
-          - template: workflow/match-2.yaml
-  - template: workflow/match-3.yaml
-  - template: workflow/match-2.yaml
-    matchers: 
-      - name: test-matcher
-        subtemplates:
-          - template: workflow/nomatch-1.yaml
-            subtemplates:
-              - template: workflow/match-1.yaml
-  - template: workflow/match-3.yaml

integration_tests/workflow/match-3.yaml

@@ -1,16 +0,0 @@
-id: basic-get-third
-
-info:
-  name: Basic 3rd GET Request
-  author: tovask
-  severity: info
-
-http:
-  - method: GET
-    path:
-      - "{{BaseURL}}"
-    matchers:
-      - type: word
-        name: test-matcher-3
-        words:
-          - "This is test matcher text"

integration_tests/workflow/multiprotocol-value-share-template.yaml

@@ -1,22 +0,0 @@
-id: multiprotocol-value-sharing-template
-
-info:
-  name: MultiProtocol Value Sharing Template
-  author: tovask
-  severity: info
-
-dns:
-  - name: "{{extracted}}"
-    type: PTR
-    matchers:
-      - type: word
-        words:
-          - "blog.projectdiscovery.io"
-
-http:
-  - path:
-      - "{{BaseURL}}/path2?extracted={{extracted}}"
-    matchers:
-      - type: word
-        words:
-          - "blog.projectdiscovery.io"

integration_tests/workflow/multiprotocol-value-share-workflow.yaml

@@ -1,11 +0,0 @@
-id: multiprotocol-value-sharing-workflow
-
-info:
-  name: MultiProtocol Value Sharing Workflow
-  author: tovask
-  severity: info
-
-workflows:
-  - template: workflow/http-value-share-template-1.yaml
-    subtemplates:
-      - template: workflow/multiprotocol-value-share-template.yaml

internal/runner/inputs.go

@@ -10,17 +10,17 @@ import (
 	"github.com/projectdiscovery/httpx/common/httpx"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	stringsutil "github.com/projectdiscovery/utils/strings"
-	syncutil "github.com/projectdiscovery/utils/sync"
+	"github.com/remeh/sizedwaitgroup"
 )
 
-var GlobalProbeBulkSize = 50
+const probeBulkSize = 50
 
 // initializeTemplatesHTTPInput initializes the http form of input
 // for any loaded http templates if input is in non-standard format.
 func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
+
 	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create temporary input file")
@@ -31,43 +31,33 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	}
 	gologger.Info().Msgf("Running httpx on input host")
 
-	var bulkSize = GlobalProbeBulkSize
-	if r.options.BulkSize > GlobalProbeBulkSize {
+	var bulkSize = probeBulkSize
+	if r.options.BulkSize > probeBulkSize {
 		bulkSize = r.options.BulkSize
 	}
 
 	httpxOptions := httpx.DefaultOptions
 	httpxOptions.RetryMax = r.options.Retries
 	httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
-	httpxOptions.NetworkPolicy = protocolstate.NetworkPolicy
 	httpxClient, err := httpx.New(&httpxOptions)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create httpx client")
 	}
 
-	shouldFollowGlobalProbeBulkSize := bulkSize == GlobalProbeBulkSize
-
 	// Probe the non-standard URLs and store them in cache
-	swg, err := syncutil.New(syncutil.WithSize(bulkSize))
-	if err != nil {
-		return nil, errors.Wrap(err, "could not create adaptive group")
-	}
-	var count atomic.Int32
+	swg := sizedwaitgroup.New(bulkSize)
+	count := int32(0)
 	r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool {
 		if stringsutil.HasPrefixAny(value.Input, "http://", "https://") {
 			return true
 		}
 
-		if shouldFollowGlobalProbeBulkSize && swg.Size != GlobalProbeBulkSize {
-			swg.Resize(GlobalProbeBulkSize)
-		}
-
 		swg.Add()
 		go func(input *contextargs.MetaInput) {
 			defer swg.Done()
 
 			if result := utils.ProbeURL(input.Input, httpxClient); result != "" {
-				count.Add(1)
+				atomic.AddInt32(&count, 1)
 				_ = hm.Set(input.Input, []byte(result))
 			}
 		}(value)
@@ -75,6 +65,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	})
 	swg.Wait()
 
-	gologger.Info().Msgf("Found %d URL from httpx", count.Load())
+	gologger.Info().Msgf("Found %d URL from httpx", atomic.LoadInt32(&count))
 	return hm, nil
 }

internal/runner/lazy.go

@@ -21,7 +21,7 @@ type AuthLazyFetchOptions struct {
 	OnError       func(error)
 }
 
-// GetAuthTmplStore create new loader for loading auth templates
+// GetAuthTemlStore create new loader for loading auth templates
 func GetAuthTmplStore(opts types.Options, catalog catalog.Catalog, execOpts protocols.ExecutorOptions) (*loader.Store, error) {
 	tmpls := []string{}
 	for _, file := range opts.SecretsFile {

internal/runner/options.go

@@ -25,7 +25,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
-	"github.com/projectdiscovery/nuclei/v3/pkg/testing"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
 	fileutil "github.com/projectdiscovery/utils/file"
@@ -51,8 +50,6 @@ func ConfigureOptions() error {
 	return nil
 }
 
-var proxyServer *testing.ProxyServer
-
 // ParseOptions parses the command line flags provided by a user
 func ParseOptions(options *types.Options) {
 	// Check if stdin pipe was given
@@ -86,19 +83,6 @@ func ParseOptions(options *types.Options) {
 	if err := ValidateOptions(options); err != nil {
 		gologger.Fatal().Msgf("Program exiting: %s\n", err)
 	}
-	if options.AutogenerateTests {
-		ps, err := testing.NewProxyServer()
-		if err != nil {
-			gologger.Fatal().Msgf("Could not create proxy server: %s\n", err)
-		}
-		proxyServer = ps
-
-		// Add the proxy server to the options
-		proxyURL := fmt.Sprintf("http://%s", proxyServer.ListenAddr)
-		options.Proxy = goflags.StringSlice([]string{proxyURL})
-		types.ProxyURL = proxyURL
-		gologger.Info().Msgf("Using %s as proxy server for automatic test-generation\n", proxyServer.ListenAddr)
-	}
 
 	// Load the resolvers if user asked for them
 	loadResolvers(options)

internal/runner/runner.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"net/http/httptest"
 	_ "net/http/pprof"
 	"os"
 	"path/filepath"
@@ -19,8 +18,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/loader/parser"
-	"github.com/projectdiscovery/nuclei/v3/pkg/testing"
-	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
 	uncoverlib "github.com/projectdiscovery/uncover"
 	pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
 	"github.com/projectdiscovery/utils/env"
@@ -317,17 +314,11 @@ func New(options *types.Options) (*Runner, error) {
 	}
 
 	if options.RateLimitMinute > 0 {
-		gologger.Print().Msgf("[%v] %v", aurora.BrightYellow("WRN"), "rate limit per minute is deprecated - use rate-limit-duration")
-		options.RateLimit = options.RateLimitMinute
-		options.RateLimitDuration = time.Minute
-	}
-	if options.RateLimit > 0 && options.RateLimitDuration == 0 {
-		options.RateLimitDuration = time.Second
-	}
-	if options.RateLimit == 0 && options.RateLimitDuration == 0 {
-		runner.rateLimiter = ratelimit.NewUnlimited(context.Background())
+		runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimitMinute), time.Minute)
+	} else if options.RateLimit > 0 {
+		runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimit), time.Second)
 	} else {
-		runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimit), options.RateLimitDuration)
+		runner.rateLimiter = ratelimit.NewUnlimited(context.Background())
 	}
 
 	if tmpDir, err := os.MkdirTemp("", "nuclei-tmp-*"); err == nil {
@@ -347,9 +338,6 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecutorOptions,
 
 // Close releases all the resources and cleans up
 func (r *Runner) Close() {
-	if proxyServer != nil {
-		proxyServer.Close()
-	}
 	if r.output != nil {
 		r.output.Close()
 	}
@@ -513,14 +501,6 @@ func (r *Runner) RunEnumeration() error {
 		return nil // exit
 	}
 	store.Load()
-
-	if len(store.Templates()) > 1 && r.options.AutogenerateTests {
-		return errors.New("autogenerate-tests flag can only be used with a single template")
-	}
-	if r.inputProvider.Count() > 1 && r.options.AutogenerateTests {
-		return errors.New("autogenerate-tests flag can only be used with a single input")
-	}
-
 	// TODO: remove below functions after v3 or update warning messages
 	disk.PrintDeprecatedPathsMsgIfApplicable(r.options.Silent)
 	templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
@@ -548,17 +528,6 @@ func (r *Runner) RunEnumeration() error {
 	// display execution info like version , templates used etc
 	r.displayExecutionInfo(store)
 
-	// If prompted, we need to run the tests for all the loaded
-	// templates by first trying to detect test definitions and then
-	// executing them with the relevant inputs.
-	if r.options.RunNucleiTests {
-		gologger.Info().Msgf("Running nuclei-templates mock-generated tests")
-		if err := r.runNucleiTests(store, executorEngine); err != nil {
-			return errors.Wrap(err, "could not run nuclei tests")
-		}
-		return nil
-	}
-
 	// prefetch secrets if enabled
 	if executorOpts.AuthProvider != nil && r.options.PreFetchSecrets {
 		gologger.Info().Msgf("Pre-fetching secrets from authprovider[s]")
@@ -578,20 +547,6 @@ func (r *Runner) RunEnumeration() error {
 		executorOpts.InputHelper.InputsHTTP = inputHelpers
 	}
 
-	// initialize stats worker ( this is no-op unless nuclei is built with stats build tag)
-	// during execution a directory with 2 files will be created in the current directory
-	// config.json - containing below info
-	// events.jsonl - containing all start and end times of all templates
-	events.InitWithConfig(&events.ScanConfig{
-		Name:                "nuclei-stats", // make this configurable
-		TargetCount:         int(r.inputProvider.Count()),
-		TemplatesCount:      len(store.Templates()) + len(store.Workflows()),
-		TemplateConcurrency: r.options.TemplateThreads,
-		PayloadConcurrency:  r.options.PayloadConcurrency,
-		JsConcurrency:       r.options.JsConcurrency,
-		Retries:             r.options.Retries,
-	}, "")
-
 	enumeration := false
 	var results *atomic.Bool
 	results, err = r.runStandardEnumeration(executorOpts, store, executorEngine)
@@ -601,17 +556,10 @@ func (r *Runner) RunEnumeration() error {
 		return err
 	}
 
-	var isInteractMatcher bool
 	if r.interactsh != nil {
 		matched := r.interactsh.Close()
 		if matched {
 			results.CompareAndSwap(false, true)
-			isInteractMatcher = true
-		}
-	}
-	if r.options.AutogenerateTests && proxyServer != nil {
-		if err := r.handleTestAutogenerate(results, isInteractMatcher, store); err != nil {
-			return err
 		}
 	}
 	if executorOpts.InputHelper != nil {
@@ -631,108 +579,6 @@ func (r *Runner) RunEnumeration() error {
 	return err
 }
 
-func (r *Runner) runNucleiTests(store *loader.Store, engine *core.Engine) error {
-	// For each loaded template, check to see if there's
-	// a corresponding test file for that template with extension
-	// .nuclei_test
-	testFiles := make(map[string]string)
-	templatePathToTemplates := make(map[string]*templates.Template)
-
-	for _, template := range store.Templates() {
-		testFile := fmt.Sprintf("%s.nuclei_test", template.Path)
-		if !fileutil.FileExists(testFile) {
-			continue
-		}
-		testFiles[template.Path] = testFile
-		templatePathToTemplates[template.Path] = template
-
-		gologger.Verbose().Msgf("Found test file for template %s: %s", template.ID, filepath.Base(testFile))
-	}
-
-	if len(testFiles) == 0 {
-		gologger.Info().Msgf("No test files found for nuclei-templates")
-		return nil
-	}
-
-	gologger.Info().Msgf("Loaded %d test files for nuclei-templates", len(testFiles))
-
-	for templatePath, testFile := range testFiles {
-		testTemplate, err := testing.ReadNucleiTestTemplate(testFile)
-		if err != nil {
-			return errors.Wrap(err, "could not read nuclei test template")
-		}
-
-		mockHandler, err := testTemplate.MockServer()
-		if err != nil {
-			return errors.Wrap(err, "could not create mock handler for template")
-		}
-
-		testServer := httptest.NewServer(mockHandler)
-		template, ok := templatePathToTemplates[templatePath]
-		if !ok {
-			testServer.Close()
-			return errors.Errorf("no template found for path %s", templatePath)
-		}
-
-		// If we have interactsh matcher, we need to handle it separately
-		// by creating a new interactsh client and closing it after the test
-		// waiting for callbacks.
-		// TODO: Parallelize
-		if testTemplate.IsInteractshMatcher {
-			interactClient, err := interactsh.New(interactsh.DefaultOptions(r.output, r.issuesClient, r.progress))
-			if err != nil {
-				testServer.Close()
-				return errors.Wrap(err, "could not create interactsh client")
-			}
-			template.Options.Interactsh = interactClient
-		}
-
-		results := engine.ExecuteScanWithOpts([]*templates.Template{template}, provider.NewSimpleInputProviderWithUrls(testServer.URL), r.options.DisableClustering)
-		if !results.Load() && !testTemplate.IsInteractshMatcher {
-			testServer.Close()
-			return errors.Errorf("no results found for template %s", template.ID)
-		}
-
-		if testTemplate.IsInteractshMatcher {
-			matched := template.Options.Interactsh.Close()
-			if !matched {
-				testServer.Close()
-				return errors.Errorf("no results found for template %s", template.ID)
-			}
-		}
-	}
-	gologger.Info().Msgf("All tests passed successfully")
-
-	return nil
-}
-
-func (r *Runner) handleTestAutogenerate(results *atomic.Bool, isInteractMatcher bool, store *loader.Store) error {
-	if !results.Load() {
-		return errors.New("no results found to autogenerate tests")
-	}
-
-	intercepted := proxyServer.Intercepted()
-	if len(intercepted) == 0 {
-		return errors.New("no intercepted requests")
-	}
-	gologger.Verbose().Msgf("Intercepted requests and responses (%d)", len(intercepted))
-	for _, pair := range intercepted {
-		if r.options.Debug {
-			fmt.Printf("Pair: %+v\n", pair)
-		}
-	}
-	var gotInput string
-	r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool {
-		gotInput = value.Input
-		return false
-	})
-	err := testing.GenerateTestsFromPair(intercepted, store.Templates()[0], gotInput, isInteractMatcher)
-	if err != nil {
-		return errors.Wrap(err, "could not generate tests from intercepted requests")
-	}
-	return nil
-}
-
 func (r *Runner) isInputNonHTTP() bool {
 	var nonURLInput bool
 	r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool {

lib/config.go

@@ -2,7 +2,6 @@ package nuclei
 
 import (
 	"context"
-	"errors"
 	"time"
 
 	"github.com/projectdiscovery/goflags"
@@ -116,48 +115,17 @@ type Concurrency struct {
 	HeadlessTemplateConcurrency   int // number of templates to run concurrently for headless templates (per host in host-spray mode)
 	JavascriptTemplateConcurrency int // number of templates to run concurrently for javascript templates (per host in host-spray mode)
 	TemplatePayloadConcurrency    int // max concurrent payloads to run for a template (a good default is 25)
-	ProbeConcurrency              int // max concurrent http probes to run (a good default is 50)
 }
 
 // WithConcurrency sets concurrency options
 func WithConcurrency(opts Concurrency) NucleiSDKOptions {
 	return func(e *NucleiEngine) error {
-		// minimum required is 1
-		if opts.TemplateConcurrency <= 0 {
-			return errors.New("template threads must be at least 1")
-		} else {
-			e.opts.TemplateThreads = opts.TemplateConcurrency
-		}
-		if opts.HostConcurrency <= 0 {
-			return errors.New("host concurrency must be at least 1")
-		} else {
-			e.opts.BulkSize = opts.HostConcurrency
-		}
-		if opts.HeadlessHostConcurrency <= 0 {
-			return errors.New("headless host concurrency must be at least 1")
-		} else {
-			e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency
-		}
-		if opts.HeadlessTemplateConcurrency <= 0 {
-			return errors.New("headless template threads must be at least 1")
-		} else {
-			e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency
-		}
-		if opts.JavascriptTemplateConcurrency <= 0 {
-			return errors.New("js must be at least 1")
-		} else {
-			e.opts.JsConcurrency = opts.JavascriptTemplateConcurrency
-		}
-		if opts.TemplatePayloadConcurrency <= 0 {
-			return errors.New("payload concurrency must be at least 1")
-		} else {
-			e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency
-		}
-		if opts.ProbeConcurrency <= 0 {
-			return errors.New("probe concurrency must be at least 1")
-		} else {
-			e.opts.ProbeConcurrency = opts.ProbeConcurrency
-		}
+		e.opts.TemplateThreads = opts.TemplateConcurrency
+		e.opts.BulkSize = opts.HostConcurrency
+		e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency
+		e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency
+		e.opts.JsConcurrency = opts.JavascriptTemplateConcurrency
+		e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency
 		return nil
 	}
 }
@@ -165,9 +133,7 @@ func WithConcurrency(opts Concurrency) NucleiSDKOptions {
 // WithGlobalRateLimit sets global rate (i.e all hosts combined) limit options
 func WithGlobalRateLimit(maxTokens int, duration time.Duration) NucleiSDKOptions {
 	return func(e *NucleiEngine) error {
-		e.opts.RateLimit = maxTokens
-		e.opts.RateLimitDuration = duration
-		e.rateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), e.opts.RateLimitDuration)
+		e.rateLimiter = ratelimit.New(context.Background(), uint(maxTokens), duration)
 		return nil
 	}
 }
@@ -314,7 +280,7 @@ func WithScanStrategy(strategy string) NucleiSDKOptions {
 // OutputWriter
 type OutputWriter output.Writer
 
-// UseOutputWriter allows setting custom output writer
+// UseWriter allows setting custom output writer
 // by default a mock writer is used with user defined callback
 // if outputWriter is used callback will be ignored
 func UseOutputWriter(writer OutputWriter) NucleiSDKOptions {

lib/multi.go

@@ -42,16 +42,11 @@ func createEphemeralObjects(base *NucleiEngine, opts *types.Options) (*unsafeOpt
 		Parser:          base.parser,
 	}
 	if opts.RateLimitMinute > 0 {
-		opts.RateLimit = opts.RateLimitMinute
-		opts.RateLimitDuration = time.Minute
-	}
-	if opts.RateLimit > 0 && opts.RateLimitDuration == 0 {
-		opts.RateLimitDuration = time.Second
-	}
-	if opts.RateLimit == 0 && opts.RateLimitDuration == 0 {
-		u.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
+		u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimitMinute), time.Minute)
+	} else if opts.RateLimit > 0 {
+		u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimit), time.Second)
 	} else {
-		u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimit), opts.RateLimitDuration)
+		u.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
 	}
 	u.engine = core.New(opts)
 	u.engine.SetExecuterOptions(u.executerOpts)

lib/sdk.go

@@ -215,14 +215,6 @@ func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.Result
 	return nil
 }
 
-func (e *NucleiEngine) Options() *types.Options {
-	return e.opts
-}
-
-func (e *NucleiEngine) Engine() *core.Engine {
-	return e.engine
-}
-
 // NewNucleiEngine creates a new nuclei engine instance
 func NewNucleiEngine(options ...NucleiSDKOptions) (*NucleiEngine, error) {
 	// default options

lib/sdk_private.go

@@ -192,16 +192,11 @@ func (e *NucleiEngine) init() error {
 
 	if e.executerOpts.RateLimiter == nil {
 		if e.opts.RateLimitMinute > 0 {
-			e.opts.RateLimit = e.opts.RateLimitMinute
-			e.opts.RateLimitDuration = time.Minute
-		}
-		if e.opts.RateLimit > 0 && e.opts.RateLimitDuration == 0 {
-			e.opts.RateLimitDuration = time.Second
-		}
-		if e.opts.RateLimit == 0 && e.opts.RateLimitDuration == 0 {
-			e.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
+			e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimitMinute), time.Minute)
+		} else if e.opts.RateLimit > 0 {
+			e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), time.Second)
 		} else {
-			e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), e.opts.RateLimitDuration)
+			e.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
 		}
 	}
 

pkg/catalog/config/constants.go

@@ -31,7 +31,7 @@ const (
 	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
-	Version = `v3.2.4`
+	Version = `v3.2.3`
 	// Directory Names of custom templates
 	CustomS3TemplatesDirName     = "s3"
 	CustomGitHubTemplatesDirName = "github"

pkg/core/engine.go

@@ -30,19 +30,14 @@ func New(options *types.Options) *Engine {
 	return engine
 }
 
-func (e *Engine) GetWorkPoolConfig() WorkPoolConfig {
-	config := WorkPoolConfig{
+// GetWorkPool returns a workpool from options
+func (e *Engine) GetWorkPool() *WorkPool {
+	return NewWorkPool(WorkPoolConfig{
 		InputConcurrency:         e.options.BulkSize,
 		TypeConcurrency:          e.options.TemplateThreads,
 		HeadlessInputConcurrency: e.options.HeadlessBulkSize,
 		HeadlessTypeConcurrency:  e.options.HeadlessTemplateThreads,
-	}
-	return config
-}
-
-// GetWorkPool returns a workpool from options
-func (e *Engine) GetWorkPool() *WorkPool {
-	return NewWorkPool(e.GetWorkPoolConfig())
+	})
 }
 
 // SetExecuterOptions sets the executer options for the engine. This is required
@@ -58,7 +53,5 @@ func (e *Engine) ExecuterOptions() protocols.ExecutorOptions {
 
 // WorkPool returns the worker pool for the engine
 func (e *Engine) WorkPool() *WorkPool {
-	// resize check point - nop if there are no changes
-	e.workPool.RefreshWithConfig(e.GetWorkPoolConfig())
 	return e.workPool
 }

pkg/core/execute_options.go

@@ -4,6 +4,8 @@ import (
 	"sync"
 	"sync/atomic"
 
+	"github.com/remeh/sizedwaitgroup"
+
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
@@ -12,7 +14,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
 	stringsutil "github.com/projectdiscovery/utils/strings"
-	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 // Execute takes a list of templates/workflows that have been compiled
@@ -108,11 +109,9 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 	wp := e.GetWorkPool()
 
 	for _, template := range templatesList {
-		// resize check point - nop if there are no changes
-		wp.RefreshWithConfig(e.GetWorkPoolConfig())
-
 		templateType := template.Type()
-		var wg *syncutil.AdaptiveWaitGroup
+
+		var wg *sizedwaitgroup.SizedWaitGroup
 		if templateType == types.HeadlessProtocol {
 			wg = wp.Headless
 		} else {
@@ -135,7 +134,7 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 // executeHostSpray executes scan using host spray strategy where templates are iterated over each target
 func (e *Engine) executeHostSpray(templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
 	results := &atomic.Bool{}
-	wp, _ := syncutil.New(syncutil.WithSize(e.options.BulkSize + e.options.HeadlessBulkSize))
+	wp := sizedwaitgroup.New(e.options.BulkSize + e.options.HeadlessBulkSize)
 
 	target.Iterate(func(value *contextargs.MetaInput) bool {
 		wp.Add()

pkg/core/executors.go

@@ -11,7 +11,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	generalTypes "github.com/projectdiscovery/nuclei/v3/pkg/types"
-	syncutil "github.com/projectdiscovery/utils/sync"
+	"github.com/remeh/sizedwaitgroup"
 )
 
 // Executors are low level executors that deals with template execution on a target
@@ -104,9 +104,9 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 			return true
 		}
 
-		wg.Add()
+		wg.WaitGroup.Add()
 		go func(index uint32, skip bool, value *contextargs.MetaInput) {
-			defer wg.Done()
+			defer wg.WaitGroup.Done()
 			defer cleanupInFlight(index)
 			if skip {
 				return
@@ -140,7 +140,7 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 		index++
 		return true
 	})
-	wg.Wait()
+	wg.WaitGroup.Wait()
 
 	// on completion marks the template as completed
 	currentInfo.Lock()
@@ -158,17 +158,14 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta
 	wp := e.GetWorkPool()
 
 	for _, tpl := range alltemplates {
-		// resize check point - nop if there are no changes
-		wp.RefreshWithConfig(e.GetWorkPoolConfig())
-
-		var sg *syncutil.AdaptiveWaitGroup
+		var sg *sizedwaitgroup.SizedWaitGroup
 		if tpl.Type() == types.HeadlessProtocol {
 			sg = wp.Headless
 		} else {
 			sg = wp.Default
 		}
 		sg.Add()
-		go func(template *templates.Template, value *contextargs.MetaInput, wg *syncutil.AdaptiveWaitGroup) {
+		go func(template *templates.Template, value *contextargs.MetaInput, wg *sizedwaitgroup.SizedWaitGroup) {
 			defer wg.Done()
 
 			var match bool
@@ -216,10 +213,7 @@ func (e *ChildExecuter) Close() *atomic.Bool {
 func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) {
 	templateType := template.Type()
 
-	// resize check point - nop if there are no changes
-	e.e.WorkPool().RefreshWithConfig(e.e.GetWorkPoolConfig())
-
-	var wg *syncutil.AdaptiveWaitGroup
+	var wg *sizedwaitgroup.SizedWaitGroup
 	if templateType == types.HeadlessProtocol {
 		wg = e.e.workPool.Headless
 	} else {

pkg/core/workflow_execute.go

@@ -5,12 +5,13 @@ import (
 	"net/http/cookiejar"
 	"sync/atomic"
 
+	"github.com/remeh/sizedwaitgroup"
+
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
 	"github.com/projectdiscovery/nuclei/v3/pkg/workflows"
-	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 const workflowStepExecutionError = "[%s] Could not execute workflow step: %s\n"
@@ -31,7 +32,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
 	if templateThreads == 1 {
 		templateThreads++
 	}
-	swg, _ := syncutil.New(syncutil.WithSize(templateThreads))
+	swg := sizedwaitgroup.New(templateThreads)
 
 	for _, template := range w.Workflows {
 		swg.Add()
@@ -39,7 +40,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
 		func(template *workflows.WorkflowTemplate) {
 			defer swg.Done()
 
-			if err := e.runWorkflowStep(template, ctx, results, swg, w); err != nil {
+			if err := e.runWorkflowStep(template, ctx, results, &swg, w); err != nil {
 				gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
 			}
 		}(template)
@@ -50,7 +51,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
 
 // runWorkflowStep runs a workflow step for the workflow. It executes the workflow
 // in a recursive manner running all subtemplates and matchers.
-func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *syncutil.AdaptiveWaitGroup, w *workflows.Workflow) error {
+func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error {
 	var firstMatched bool
 	var err error
 	var mainErr error
@@ -138,9 +139,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
 						go func(subtemplate *workflows.WorkflowTemplate) {
 							defer swg.Done()
 
-							// create a new context with the same input but with unset callbacks
-							subCtx := scan.NewScanContext(ctx.Input)
-							if err := e.runWorkflowStep(subtemplate, subCtx, results, swg, w); err != nil {
+							if err := e.runWorkflowStep(subtemplate, ctx, results, swg, w); err != nil {
 								gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err)
 							}
 						}(subtemplate)
@@ -164,9 +163,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
 			swg.Add()
 
 			go func(template *workflows.WorkflowTemplate) {
-				// create a new context with the same input but with unset callbacks
-				subCtx := scan.NewScanContext(ctx.Input)
-				if err := e.runWorkflowStep(template, subCtx, results, swg, w); err != nil {
+				if err := e.runWorkflowStep(template, ctx, results, swg, w); err != nil {
 					gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
 				}
 				swg.Done()

pkg/core/workpool.go

@@ -1,8 +1,9 @@
 package core
 
 import (
+	"github.com/remeh/sizedwaitgroup"
+
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
-	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 // WorkPool implements an execution pool for executing different
@@ -11,8 +12,8 @@ import (
 // It also allows Configuration of such requirements. This is used
 // for per-module like separate headless concurrency etc.
 type WorkPool struct {
-	Headless *syncutil.AdaptiveWaitGroup
-	Default  *syncutil.AdaptiveWaitGroup
+	Headless *sizedwaitgroup.SizedWaitGroup
+	Default  *sizedwaitgroup.SizedWaitGroup
 	config   WorkPoolConfig
 }
 
@@ -30,13 +31,13 @@ type WorkPoolConfig struct {
 
 // NewWorkPool returns a new WorkPool instance
 func NewWorkPool(config WorkPoolConfig) *WorkPool {
-	headlessWg, _ := syncutil.New(syncutil.WithSize(config.HeadlessTypeConcurrency))
-	defaultWg, _ := syncutil.New(syncutil.WithSize(config.TypeConcurrency))
+	headlessWg := sizedwaitgroup.New(config.HeadlessTypeConcurrency)
+	defaultWg := sizedwaitgroup.New(config.TypeConcurrency)
 
 	return &WorkPool{
 		config:   config,
-		Headless: headlessWg,
-		Default:  defaultWg,
+		Headless: &headlessWg,
+		Default:  &defaultWg,
 	}
 }
 
@@ -46,39 +47,19 @@ func (w *WorkPool) Wait() {
 	w.Headless.Wait()
 }
 
+// InputWorkPool is a work pool per-input
+type InputWorkPool struct {
+	WaitGroup *sizedwaitgroup.SizedWaitGroup
+}
+
 // InputPool returns a work pool for an input type
-func (w *WorkPool) InputPool(templateType types.ProtocolType) *syncutil.AdaptiveWaitGroup {
+func (w *WorkPool) InputPool(templateType types.ProtocolType) *InputWorkPool {
 	var count int
 	if templateType == types.HeadlessProtocol {
 		count = w.config.HeadlessInputConcurrency
 	} else {
 		count = w.config.InputConcurrency
 	}
-	swg, _ := syncutil.New(syncutil.WithSize(count))
-	return swg
-}
-
-func (w *WorkPool) RefreshWithConfig(config WorkPoolConfig) {
-	if w.config.TypeConcurrency != config.TypeConcurrency {
-		w.config.TypeConcurrency = config.TypeConcurrency
-	}
-	if w.config.HeadlessTypeConcurrency != config.HeadlessTypeConcurrency {
-		w.config.HeadlessTypeConcurrency = config.HeadlessTypeConcurrency
-	}
-	if w.config.InputConcurrency != config.InputConcurrency {
-		w.config.InputConcurrency = config.InputConcurrency
-	}
-	if w.config.HeadlessInputConcurrency != config.HeadlessInputConcurrency {
-		w.config.HeadlessInputConcurrency = config.HeadlessInputConcurrency
-	}
-	w.Refresh()
-}
-
-func (w *WorkPool) Refresh() {
-	if w.Default.Size != w.config.TypeConcurrency {
-		w.Default.Resize(w.config.TypeConcurrency)
-	}
-	if w.Headless.Size != w.config.HeadlessTypeConcurrency {
-		w.Headless.Resize(w.config.HeadlessTypeConcurrency)
-	}
+	swg := sizedwaitgroup.New(count)
+	return &InputWorkPool{WaitGroup: &swg}
 }

pkg/fuzz/dataformat/form.go

@@ -94,33 +94,18 @@ func (f *Form) Encode(data KV) (string, error) {
 					}
 				}
 			}
-			if maxIndex >= 0 { // Ensure the slice is only created if maxIndex is valid
-				data := make([]string, maxIndex+1) // Ensure the slice is large enough
-				for key, value := range v {
-					matches := reNormalized.FindStringSubmatch(key)
-					if len(matches) == 2 {
-						dataIdx, err := strconv.Atoi(matches[1]) // Error already checked above
-						if err != nil {
-							gologger.Verbose().Msgf("error converting data index to integer: %v", err)
-							continue
-						}
-						// Validate dataIdx to avoid index out of range errors
-						if dataIdx > 0 && dataIdx <= len(data) {
-							data[dataIdx-1] = value // Use dataIdx-1 since slice is 0-indexed
-						} else {
-							gologger.Verbose().Msgf("data index out of range: %d", dataIdx)
-						}
-					}
-				}
-				if len(params.Get(k)) > 0 {
-					data[maxIndex] = fmt.Sprint(params.Get(k)) // Use maxIndex which is the last index
-				}
-				// remove existing
-				params.Del(k)
-				if len(data) > 0 {
-					params.Add(k, data...)
+			data := make([]string, maxIndex+1) // Ensure the slice is large enough
+			for key, value := range v {
+				matches := reNormalized.FindStringSubmatch(key)
+				if len(matches) == 2 {
+					dataIdx, _ := strconv.Atoi(matches[1]) // Error already checked above
+					data[dataIdx-1] = value                // Use dataIdx-1 since slice is 0-indexed
 				}
 			}
+			data[maxIndex] = fmt.Sprint(params.Get(k)) // Use maxIndex which is the last index
+			// remove existing
+			params.Del(k)
+			params.Add(k, data...)
 		}
 	}
 

pkg/fuzz/execute.go

@@ -11,7 +11,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/component"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/retryablehttp-go"
 	errorutil "github.com/projectdiscovery/utils/errors"
@@ -101,11 +100,8 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) {
 	baseValues := input.Values
 	if rule.generator == nil {
 		for _, component := range finalComponentList {
-			// get vars from variables while replacing interactsh urls
 			evaluatedValues, interactURLs := rule.options.Variables.EvaluateWithInteractsh(baseValues, rule.options.Interactsh)
-			input.Values = generators.MergeMaps(evaluatedValues, baseValues, rule.options.Options.Vars.AsMap(), rule.options.Constants)
-			// evaluate all vars with interactsh
-			input.Values, interactURLs = rule.evaluateVarsWithInteractsh(input.Values, interactURLs)
+			input.Values = generators.MergeMaps(evaluatedValues, baseValues, rule.options.Constants)
 			input.InteractURLs = interactURLs
 			err := rule.executeRuleValues(input, component)
 			if err != nil {
@@ -122,12 +118,9 @@ mainLoop:
 			if !next {
 				continue mainLoop
 			}
-			// get vars from variables while replacing interactsh urls
 			evaluatedValues, interactURLs := rule.options.Variables.EvaluateWithInteractsh(generators.MergeMaps(values, baseValues), rule.options.Interactsh)
-			input.Values = generators.MergeMaps(values, evaluatedValues, baseValues, rule.options.Options.Vars.AsMap(), rule.options.Constants)
-			// evaluate all vars with interactsh
-			input.Values, interactURLs = rule.evaluateVarsWithInteractsh(input.Values, interactURLs)
 			input.InteractURLs = interactURLs
+			input.Values = generators.MergeMaps(values, evaluatedValues, baseValues, rule.options.Constants)
 
 			if err := rule.executeRuleValues(input, component); err != nil {
 				if err == io.EOF {
@@ -141,33 +134,6 @@ mainLoop:
 	return nil
 }
 
-// evaluateVarsWithInteractsh evaluates the variables with Interactsh URLs and updates them accordingly.
-func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, interactshUrls []string) (map[string]interface{}, []string) {
-	// Check if Interactsh options are configured
-	if rule.options.Interactsh != nil {
-		// Iterate through the data to replace and evaluate variables with Interactsh URLs
-		for k, v := range data {
-			// Replace variables with Interactsh URLs and collect new URLs
-			got, oastUrls := rule.options.Interactsh.Replace(fmt.Sprint(v), interactshUrls)
-
-			// Append new OAST URLs if any
-			if len(oastUrls) > 0 {
-				interactshUrls = append(interactshUrls, oastUrls...)
-			}
-			// Evaluate the replaced data
-			evaluatedData, err := expressions.Evaluate(got, data)
-			if err == nil {
-				// Update the data if there is a change after evaluation
-				if evaluatedData != got {
-					data[k] = evaluatedData
-				}
-			}
-		}
-	}
-	// Return the updated data and Interactsh URLs without any error
-	return data, interactshUrls
-}
-
 // isInputURLValid returns true if url is valid after parsing it
 func (rule *Rule) isInputURLValid(input *contextargs.Context) bool {
 	if input == nil || input.MetaInput == nil || input.MetaInput.Input == "" {

pkg/fuzz/type.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/invopop/jsonschema"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	"gopkg.in/yaml.v2"
 )
@@ -30,44 +29,6 @@ type SliceOrMapSlice struct {
 	KV    *mapsutil.OrderedMap[string, string]
 }
 
-func (v SliceOrMapSlice) JSONSchemaExtend(schema *jsonschema.Schema) *jsonschema.Schema {
-	schema = &jsonschema.Schema{
-		Title:       schema.Title,
-		Description: schema.Description,
-		Type:        "array",
-		Items: &jsonschema.Schema{
-			OneOf: []*jsonschema.Schema{
-				{
-					Type: "string",
-				},
-				{
-					Type: "object",
-				},
-			},
-		},
-	}
-	return schema
-}
-
-func (v SliceOrMapSlice) JSONSchema() *jsonschema.Schema {
-	gotType := &jsonschema.Schema{
-		Title:       "Payloads of Fuzz Rule",
-		Description: "Payloads to perform fuzzing substitutions with.",
-		Type:        "array",
-		Items: &jsonschema.Schema{
-			OneOf: []*jsonschema.Schema{
-				{
-					Type: "string",
-				},
-				{
-					Type: "object",
-				},
-			},
-		},
-	}
-	return gotType
-}
-
 // UnmarshalJSON implements json.Unmarshaler interface.
 func (v *SliceOrMapSlice) UnmarshalJSON(data []byte) error {
 	// try to unmashal as a string and fallback to map

pkg/js/compiler/non-pool.go

@@ -4,13 +4,13 @@ import (
 	"sync"
 
 	"github.com/dop251/goja"
-	syncutil "github.com/projectdiscovery/utils/sync"
+	"github.com/remeh/sizedwaitgroup"
 )
 
 var (
-	ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
+	ephemeraljsc    = sizedwaitgroup.New(NonPoolingVMConcurrency)
 	lazyFixedSgInit = sync.OnceFunc(func() {
-		ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
+		ephemeraljsc = sizedwaitgroup.New(NonPoolingVMConcurrency)
 	})
 )
 

pkg/js/compiler/pool.go

@@ -36,7 +36,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/goconsole"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	stringsutil "github.com/projectdiscovery/utils/strings"
-	syncutil "github.com/projectdiscovery/utils/sync"
+	"github.com/remeh/sizedwaitgroup"
 )
 
 const (
@@ -51,16 +51,10 @@ var (
 		// autoregister console node module with default printer it uses gologger backend
 		require.RegisterNativeModule(console.ModuleName, console.RequireWithPrinter(goconsole.NewGoConsolePrinter()))
 	})
-	pooljsc    *syncutil.AdaptiveWaitGroup
+	pooljsc    sizedwaitgroup.SizedWaitGroup
 	lazySgInit = sync.OnceFunc(func() {
-		pooljsc, _ = syncutil.New(syncutil.WithSize(PoolingJsVmConcurrency))
+		pooljsc = sizedwaitgroup.New(PoolingJsVmConcurrency)
 	})
-	sgResizeCheck = func() {
-		// resize check point
-		if pooljsc.Size != PoolingJsVmConcurrency {
-			pooljsc.Resize(PoolingJsVmConcurrency)
-		}
-	}
 )
 
 var gojapool = &sync.Pool{
@@ -122,8 +116,6 @@ func executeWithPoolingProgram(p *goja.Program, args *ExecuteArgs, opts *Execute
 	// its unknown (most likely cannot be done) to limit max js runtimes at a moment without making it static
 	// unlike sync.Pool which reacts to GC and its purposes is to reuse objects rather than creating new ones
 	lazySgInit()
-	sgResizeCheck()
-
 	pooljsc.Add()
 	defer pooljsc.Done()
 	runtime := gojapool.Get().(*goja.Runtime)

pkg/js/generated/ts/kerberos.ts

@@ -200,9 +200,9 @@ export interface AuthorizationDataEntry {
  */
 export interface BitString {
     
-    BitLength?: number,
-    
     Bytes?: Uint8Array,
+    
+    BitLength?: number,
 }
 
 
@@ -212,9 +212,9 @@ export interface BitString {
  */
 export interface BitString {
     
-    BitLength?: number,
-    
     Bytes?: Uint8Array,
+    
+    BitLength?: number,
 }
 
 
@@ -236,15 +236,17 @@ export interface Config {
  */
 export interface EncTicketPart {
     
+    RenewTill?: Date,
+    
+    CRealm?: string,
+    
     AuthTime?: Date,
     
     StartTime?: Date,
     
     EndTime?: Date,
     
-    RenewTill?: Date,
-    
-    CRealm?: string,
+    Transited?: TransitedEncoding,
     
     CAddr?: HostAddress,
     
@@ -255,8 +257,6 @@ export interface EncTicketPart {
     Key?: EncryptionKey,
     
     CName?: PrincipalName,
-    
-    Transited?: TransitedEncoding,
 }
 
 
@@ -266,11 +266,11 @@ export interface EncTicketPart {
  */
 export interface EncryptedData {
     
-    KVNO?: number,
-    
     Cipher?: Uint8Array,
     
     EType?: number,
+    
+    KVNO?: number,
 }
 
 
@@ -318,41 +318,15 @@ export interface HostAddress {
  */
 export interface LibDefaults {
     
-    NoAddresses?: boolean,
-    
-    RealmTryDomains?: number,
-    
-    DNSLookupKDC?: boolean,
-    
-    DefaultRealm?: string,
-    
-    SafeChecksumType?: number,
-    
-    VerifyAPReqNofail?: boolean,
-    
-    AllowWeakCrypto?: boolean,
-    
-    DefaultTGSEnctypes?: string[],
-    
-    DefaultTktEnctypeIDs?: number[],
-    
-    IgnoreAcceptorHostname?: boolean,
-    
-    K5LoginAuthoritative?: boolean,
-    
-    PermittedEnctypes?: string[],
-    
     /**
     * time in nanoseconds
     */
     
     Clockskew?: number,
     
-    DNSCanonicalizeHostname?: boolean,
+    KDCTimeSync?: number,
     
-    Proxiable?: boolean,
-    
-    RDNS?: boolean,
+    SafeChecksumType?: number,
     
     /**
     * time in nanoseconds
@@ -360,31 +334,51 @@ export interface LibDefaults {
     
     TicketLifetime?: number,
     
+    Forwardable?: boolean,
+    
+    K5LoginAuthoritative?: boolean,
+    
+    AllowWeakCrypto?: boolean,
+    
     DefaultClientKeytabName?: string,
     
-    PermittedEnctypeIDs?: number[],
-    
-    UDPPreferenceLimit?: number,
-    
-    DefaultTGSEnctypeIDs?: number[],
-    
     DefaultTktEnctypes?: string[],
     
-    CCacheType?: number,
-    
-    DNSLookupRealm?: boolean,
-    
     ExtraAddresses?: Uint8Array,
     
-    PreferredPreauthTypes?: number[],
-    
-    Canonicalize?: boolean,
-    
-    Forwardable?: boolean,
-    
     K5LoginDirectory?: string,
     
-    KDCTimeSync?: number,
+    PreferredPreauthTypes?: number[],
+    
+    RDNS?: boolean,
+    
+    DefaultKeytabName?: string,
+    
+    DefaultRealm?: string,
+    
+    DefaultTGSEnctypeIDs?: number[],
+    
+    DNSCanonicalizeHostname?: boolean,
+    
+    PermittedEnctypes?: string[],
+    
+    VerifyAPReqNofail?: boolean,
+    
+    DNSLookupRealm?: boolean,
+    
+    UDPPreferenceLimit?: number,
+    
+    Canonicalize?: boolean,
+    
+    CCacheType?: number,
+    
+    DefaultTGSEnctypes?: string[],
+    
+    Proxiable?: boolean,
+    
+    DNSLookupKDC?: boolean,
+    
+    RealmTryDomains?: number,
     
     /**
     * time in nanoseconds
@@ -392,7 +386,13 @@ export interface LibDefaults {
     
     RenewLifetime?: number,
     
-    DefaultKeytabName?: string,
+    DefaultTktEnctypeIDs?: number[],
+    
+    IgnoreAcceptorHostname?: boolean,
+    
+    NoAddresses?: boolean,
+    
+    PermittedEnctypeIDs?: number[],
     
     KDCDefaultOptions?: BitString,
 }
@@ -404,9 +404,9 @@ export interface LibDefaults {
  */
 export interface PrincipalName {
     
-    NameString?: string[],
-    
     NameType?: number,
+    
+    NameString?: string[],
 }
 
 
@@ -416,6 +416,8 @@ export interface PrincipalName {
  */
 export interface Realm {
     
+    Realm?: string,
+    
     AdminServer?: string[],
     
     DefaultDomain?: string,
@@ -425,8 +427,6 @@ export interface Realm {
     KPasswdServer?: string[],
     
     MasterKDC?: string[],
-    
-    Realm?: string,
 }
 
 
@@ -450,10 +450,10 @@ export interface TGS {
  */
 export interface Ticket {
     
-    TktVNO?: number,
-    
     Realm?: string,
     
+    TktVNO?: number,
+    
     SName?: PrincipalName,
     
     EncPart?: EncryptedData,

pkg/js/generated/ts/ldap.ts

@@ -358,22 +358,6 @@ export class Client {
     }
     
 
-    /**
-    * GetADUserAsRepRoastable returns all AD users that are AsRepRoastable
-    * using FilterIsPerson, and FilterDontRequirePreauth filter query
-    * @example
-    * ```javascript
-    * const ldap = require('nuclei/ldap');
-    * const client = new ldap.Client('ldap://ldap.example.com', 'acme.com');
-    * const AsRepRoastable = client.GetADUserAsRepRoastable();
-    * log(to_json(AsRepRoastable));
-    * ```
-    */
-    public GetADUserAsRepRoastable(): ADObject[] {
-        return [];
-    }
-    
-
     /**
     * GetADDomainSID returns the SID of the AD domain
     * @example

pkg/js/generated/ts/mysql.ts

@@ -209,9 +209,9 @@ export interface MySQLOptions {
  */
 export interface SQLResult {
     
-    Count?: number,
-    
     Columns?: string[],
+    
+    Count?: number,
 }
 
 

pkg/js/generated/ts/smb.ts

@@ -137,6 +137,10 @@ export interface NegotiationLog {
  */
 export interface SMBCapabilities {
     
+    DFSSupport?: boolean,
+    
+    Leasing?: boolean,
+    
     LargeMTU?: boolean,
     
     MultiChan?: boolean,
@@ -146,10 +150,6 @@ export interface SMBCapabilities {
     DirLeasing?: boolean,
     
     Encryption?: boolean,
-    
-    DFSSupport?: boolean,
-    
-    Leasing?: boolean,
 }
 
 
@@ -159,8 +159,6 @@ export interface SMBCapabilities {
  */
 export interface SMBLog {
     
-    SupportV1?: boolean,
-    
     NativeOs?: string,
     
     NTLM?: string,
@@ -169,13 +167,15 @@ export interface SMBLog {
     
     HasNTLM?: boolean,
     
-    Version?: SMBVersions,
+    SupportV1?: boolean,
     
     Capabilities?: SMBCapabilities,
     
     NegotiationLog?: NegotiationLog,
     
     SessionSetupLog?: SessionSetupLog,
+    
+    Version?: SMBVersions,
 }
 
 
@@ -185,13 +185,13 @@ export interface SMBLog {
  */
 export interface SMBVersions {
     
-    VerString?: string,
-    
     Major?: number,
     
     Minor?: number,
     
     Revision?: number,
+    
+    VerString?: string,
 }
 
 
@@ -201,6 +201,10 @@ export interface SMBVersions {
  */
 export interface ServiceSMB {
     
+    SigningEnabled?: boolean,
+    
+    SigningRequired?: boolean,
+    
     OSVersion?: string,
     
     NetBIOSComputerName?: string,
@@ -212,10 +216,6 @@ export interface ServiceSMB {
     DNSDomainName?: string,
     
     ForestName?: string,
-    
-    SigningEnabled?: boolean,
-    
-    SigningRequired?: boolean,
 }
 
 
@@ -225,12 +225,12 @@ export interface ServiceSMB {
  */
 export interface SessionSetupLog {
     
-    NegotiateFlags?: number,
-    
     SetupFlags?: number,
     
     TargetName?: string,
     
+    NegotiateFlags?: number,
+    
     HeaderLog?: HeaderLog,
 }
 

pkg/js/generated/ts/ssh.ts

@@ -133,9 +133,9 @@ export interface Algorithms {
     
     HostKey?: string,
     
-    R?: DirectionAlgorithms,
-    
     W?: DirectionAlgorithms,
+    
+    R?: DirectionAlgorithms,
 }
 
 
@@ -197,34 +197,34 @@ export interface HandshakeLog {
  */
 export interface KexInitMsg {
     
+    CiphersServerClient?: string[],
+    
+    MACsClientServer?: string[],
+    
+    MACsServerClient?: string[],
+    
+    LanguagesClientServer?: string[],
+    
+    KexAlgos?: string[],
+    
+    CiphersClientServer?: string[],
+    
     Reserved?: number,
     
+    CompressionClientServer?: string[],
+    
+    CompressionServerClient?: string[],
+    
+    LanguagesServerClient?: string[],
+    
+    FirstKexFollows?: boolean,
+    
     /**
     * fixed size array of length: [16]
     */
     
     Cookie?: Uint8Array,
     
-    CiphersClientServer?: string[],
-    
-    MACsClientServer?: string[],
-    
-    MACsServerClient?: string[],
-    
-    CompressionServerClient?: string[],
-    
-    LanguagesClientServer?: string[],
-    
-    FirstKexFollows?: boolean,
-    
-    KexAlgos?: string[],
-    
-    CiphersServerClient?: string[],
-    
-    CompressionClientServer?: string[],
-    
-    LanguagesServerClient?: string[],
-    
     ServerHostKeyAlgos?: string[],
 }
 

pkg/js/libs/ldap/adenum.go

@@ -253,19 +253,6 @@ func (c *Client) GetADUserKerberoastable() []ADObject {
 	return c.FindADObjects(JoinFilters(FilterIsPerson, FilterAccountEnabled, FilterHasServicePrincipalName))
 }
 
-// GetADUserAsRepRoastable returns all AD users that are AsRepRoastable
-// using FilterIsPerson, and FilterDontRequirePreauth filter query
-// @example
-// ```javascript
-// const ldap = require('nuclei/ldap');
-// const client = new ldap.Client('ldap://ldap.example.com', 'acme.com');
-// const AsRepRoastable = client.GetADUserAsRepRoastable();
-// log(to_json(AsRepRoastable));
-// ```
-func (c *Client) GetADUserAsRepRoastable() []ADObject {
-	return c.FindADObjects(JoinFilters(FilterIsPerson, FilterDontRequirePreauth))
-}
-
 // GetADDomainSID returns the SID of the AD domain
 // @example
 // ```javascript

pkg/model/types/userAgent/user_agent.go

@@ -61,7 +61,7 @@ type UserAgentHolder struct {
 	Value UserAgent `mapping:"true"`
 }
 
-func (userAgentHolder UserAgentHolder) JSONSchema() *jsonschema.Schema {
+func (userAgentHolder UserAgentHolder) JSONSchemaType() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "userAgent for the headless",

pkg/operators/extractors/extractor_types.go

@@ -72,7 +72,7 @@ type ExtractorTypeHolder struct {
 	ExtractorType ExtractorType `mapping:"true"`
 }
 
-func (holder ExtractorTypeHolder) JSONSchema() *jsonschema.Schema {
+func (holder ExtractorTypeHolder) JSONSchemaType() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "type of the extractor",

pkg/operators/matchers/matchers_types.go

@@ -82,7 +82,7 @@ func (t MatcherTypeHolder) String() string {
 	return t.MatcherType.String()
 }
 
-func (holder MatcherTypeHolder) JSONSchema() *jsonschema.Schema {
+func (holder MatcherTypeHolder) JSONSchemaType() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "type of the matcher",

pkg/protocols/code/code.go

@@ -48,25 +48,25 @@ var (
 type Request struct {
 	// Operators for the current request go here.
 	operators.Operators `yaml:",inline,omitempty"`
-	CompiledOperators   *operators.Operators `yaml:"-" json:"-"`
+	CompiledOperators   *operators.Operators `yaml:"-"`
 
 	// ID is the optional id of the request
 	ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the request,description=ID is the optional ID of the Request"`
 	// description: |
 	//   Engine type
-	Engine []string `yaml:"engine,omitempty" json:"engine,omitempty" jsonschema:"title=engine,description=Engine"`
+	Engine []string `yaml:"engine,omitempty" jsonschema:"title=engine,description=Engine"`
 	// description: |
 	//   PreCondition is a condition which is evaluated before sending the request.
 	PreCondition string `yaml:"pre-condition,omitempty" json:"pre-condition,omitempty" jsonschema:"title=pre-condition for the request,description=PreCondition is a condition which is evaluated before sending the request"`
 	// description: |
 	//   Engine Arguments
-	Args []string `yaml:"args,omitempty" json:"args,omitempty" jsonschema:"title=args,description=Args"`
+	Args []string `yaml:"args,omitempty" jsonschema:"title=args,description=Args"`
 	// description: |
 	//   Pattern preferred for file name
-	Pattern string `yaml:"pattern,omitempty" json:"pattern,omitempty" jsonschema:"title=pattern,description=Pattern"`
+	Pattern string `yaml:"pattern,omitempty" jsonschema:"title=pattern,description=Pattern"`
 	// description: |
 	//   Source File/Snippet
-	Source string `yaml:"source,omitempty" json:"source,omitempty" jsonschema:"title=source file/snippet,description=Source snippet"`
+	Source string `yaml:"source,omitempty" jsonschema:"title=source file/snippet,description=Source snippet"`
 
 	options              *protocols.ExecutorOptions `yaml:"-" json:"-"`
 	preConditionCompiled *goja.Program              `yaml:"-" json:"-"`
@@ -160,8 +160,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 	if request.options.HasTemplateCtx(input.MetaInput) {
 		allvars = generators.MergeMaps(allvars, request.options.GetTemplateCtx(input.MetaInput).GetAll())
 	}
-	// add dynamic and previous variables
-	allvars = generators.MergeMaps(allvars, dynamicValues, previous)
 	// optionvars are vars passed from CLI or env variables
 	optionVars := generators.BuildPayloadFromOptions(request.options.Options)
 	variablesMap := request.options.Variables.Evaluate(allvars)

pkg/protocols/common/automaticscan/automaticscan.go

@@ -30,8 +30,8 @@ import (
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
-	syncutil "github.com/projectdiscovery/utils/sync"
 	wappalyzer "github.com/projectdiscovery/wappalyzergo"
+	"github.com/remeh/sizedwaitgroup"
 	"gopkg.in/yaml.v2"
 )
 
@@ -128,10 +128,7 @@ func (s *Service) Close() bool {
 func (s *Service) Execute() error {
 	gologger.Info().Msgf("Executing Automatic scan on %d target[s]", s.target.Count())
 	// setup host concurrency
-	sg, err := syncutil.New(syncutil.WithSize(s.opts.Options.BulkSize))
-	if err != nil {
-		return err
-	}
+	sg := sizedwaitgroup.New(s.opts.Options.BulkSize)
 	s.target.Iterate(func(value *contextargs.MetaInput) bool {
 		sg.Add()
 		go func(input *contextargs.MetaInput) {
@@ -249,7 +246,7 @@ func (s *Service) getTagsUsingDetectionTemplates(input *contextargs.MetaInput) (
 	// execute tech detection templates on target
 	tags := map[string]struct{}{}
 	m := &sync.Mutex{}
-	sg, _ := syncutil.New(syncutil.WithSize(s.opts.Options.TemplateThreads))
+	sg := sizedwaitgroup.New(s.opts.Options.TemplateThreads)
 	counter := atomic.Uint32{}
 
 	for _, t := range s.techTemplates {

pkg/protocols/common/generators/attack_types.go

@@ -61,7 +61,7 @@ type AttackTypeHolder struct {
 	Value AttackType `mapping:"true"`
 }
 
-func (holder AttackTypeHolder) JSONSchema() *jsonschema.Schema {
+func (holder AttackTypeHolder) JSONSchemaType() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "type of the attack",

pkg/protocols/common/protocolstate/state.go

@@ -124,7 +124,7 @@ func Init(options *types.Options) error {
 		opts.ResolversFile = true
 		opts.EnableFallback = true
 	}
-	if len(options.InternalResolversList) > 0 {
+	if options.ResolversFile != "" {
 		opts.BaseResolvers = options.InternalResolversList
 	}
 
@@ -132,9 +132,6 @@ func Init(options *types.Options) error {
 
 	opts.WithDialerHistory = true
 	opts.SNIName = options.SNI
-	// this instance is used in javascript protocol libraries and
-	// dial history is required to get dialed ip of a host
-	opts.WithDialerHistory = true
 
 	// fastdialer now by default fallbacks to ztls when there are tls related errors
 	dialer, err := fastdialer.NewDialer(opts)

pkg/protocols/common/variables/variables.go

@@ -21,7 +21,7 @@ type Variable struct {
 	utils.InsertionOrderedStringMap `yaml:"-" json:"-"`
 }
 
-func (variables Variable) JSONSchema() *jsonschema.Schema {
+func (variables Variable) JSONSchemaType() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:                 "object",
 		Title:                "variables for the request",

pkg/protocols/dns/dns.go

@@ -60,7 +60,7 @@ type Request struct {
 	// examples:
 	//   - name: Use a retry of 100 to 150 generally
 	//     value: 100
-	TraceMaxRecursion int `yaml:"trace-max-recursion,omitempty" json:"trace-max-recursion,omitempty"  jsonschema:"title=trace-max-recursion level for dns request,description=TraceMaxRecursion is the number of max recursion allowed for trace operations"`
+	TraceMaxRecursion int `yaml:"trace-max-recursion,omitempty"  jsonschema:"title=trace-max-recursion level for dns request,description=TraceMaxRecursion is the number of max recursion allowed for trace operations"`
 
 	// description: |
 	//   Attack is the type of payload combinations to perform.
@@ -83,7 +83,7 @@ type Request struct {
 	Threads   int `yaml:"threads,omitempty" json:"threads,omitempty" jsonschema:"title=threads for sending requests,description=Threads specifies number of threads to use sending requests. This enables Connection Pooling"`
 	generator *generators.PayloadGenerator
 
-	CompiledOperators *operators.Operators `yaml:"-" json:"-"`
+	CompiledOperators *operators.Operators `yaml:"-"`
 	dnsClient         *retryabledns.Client
 	options           *protocols.ExecutorOptions
 
@@ -275,8 +275,6 @@ func questionTypeToInt(questionType string) uint16 {
 		question = dns.TypeTLSA
 	case "ANY":
 		question = dns.TypeANY
-	case "SRV":
-		question = dns.TypeSRV
 	}
 	return question
 }

pkg/protocols/dns/dns_types.go

@@ -37,8 +37,6 @@ const (
 	TLSA
 	// name:ANY
 	ANY
-	// name:SRV
-	SRV
 	limit
 )
 
@@ -56,7 +54,6 @@ var DNSRequestTypeMapping = map[DNSRequestType]string{
 	CAA:   "CAA",
 	TLSA:  "TLSA",
 	ANY:   "ANY",
-	SRV:   "SRV",
 }
 
 // GetSupportedDNSRequestTypes returns list of supported types
@@ -95,7 +92,7 @@ func (holder DNSRequestTypeHolder) String() string {
 	return holder.DNSRequestType.String()
 }
 
-func (holder DNSRequestTypeHolder) JSONSchema() *jsonschema.Schema {
+func (holder DNSRequestTypeHolder) JSONSchemaType() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "type of DNS request to make",

pkg/protocols/dns/dnsclientpool/clientpool.go

@@ -34,7 +34,7 @@ func Init(options *types.Options) error {
 	clientPool = make(map[string]*retryabledns.Client)
 
 	resolvers := defaultResolvers
-	if len(options.InternalResolversList) > 0 {
+	if options.ResolversFile != "" {
 		resolvers = options.InternalResolversList
 	}
 	var err error
@@ -78,7 +78,7 @@ func Get(options *types.Options, configuration *Configuration) (*retryabledns.Cl
 	poolMutex.RUnlock()
 
 	resolvers := defaultResolvers
-	if len(options.InternalResolversList) > 0 {
+	if options.ResolversFile != "" {
 		resolvers = options.InternalResolversList
 	} else if len(configuration.Resolvers) > 0 {
 		resolvers = configuration.Resolvers

pkg/protocols/dns/request.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/miekg/dns"
 	"github.com/pkg/errors"
+	"github.com/remeh/sizedwaitgroup"
 	"go.uber.org/multierr"
 	"golang.org/x/exp/maps"
 
@@ -26,7 +27,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	"github.com/projectdiscovery/retryabledns"
 	iputil "github.com/projectdiscovery/utils/ip"
-	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var _ protocols.Request = &Request{}
@@ -62,15 +62,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 	variablesMap := request.options.Variables.Evaluate(vars)
 	vars = generators.MergeMaps(vars, variablesMap, request.options.Constants)
 
-	// if request threads matches global payload concurrency we follow it
-	shouldFollowGlobal := request.Threads == request.options.Options.PayloadConcurrency
-
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
-		swg, err := syncutil.New(syncutil.WithSize(request.Threads))
-		if err != nil {
-			return err
-		}
+		swg := sizedwaitgroup.New(request.Threads)
 		var multiErr error
 		m := &sync.Mutex{}
 
@@ -79,12 +73,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 			if !ok {
 				break
 			}
-
-			// resize check point - nop if there are no changes
-			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
-				swg.Resize(request.options.Options.PayloadConcurrency)
-			}
-
 			value = generators.MergeMaps(vars, value)
 			swg.Add()
 			go func(newVars map[string]interface{}) {
@@ -152,7 +140,7 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad
 		}
 	}
 
-	request.options.RateLimitTake()
+	request.options.RateLimiter.Take()
 
 	// Send the request to the target servers
 	response, err := dnsClient.Do(compiledRequest)

pkg/protocols/file/request.go

@@ -11,6 +11,7 @@ import (
 	"github.com/docker/go-units"
 	"github.com/mholt/archiver"
 	"github.com/pkg/errors"
+	"github.com/remeh/sizedwaitgroup"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
@@ -23,7 +24,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	sliceutil "github.com/projectdiscovery/utils/slice"
-	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var _ protocols.Request = &Request{}
@@ -47,11 +47,8 @@ var errEmptyResult = errors.New("Empty result")
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
-	wg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize))
-	if err != nil {
-		return err
-	}
-	err = request.getInputPaths(input.MetaInput.Input, func(filePath string) {
+	wg := sizedwaitgroup.New(request.options.Options.BulkSize)
+	err := request.getInputPaths(input.MetaInput.Input, func(filePath string) {
 		wg.Add()
 		func(filePath string) {
 			defer wg.Done()

pkg/protocols/headless/engine/action.go

@@ -1,10 +1,6 @@
 package engine
 
-import (
-	"strings"
-
-	"github.com/invopop/jsonschema"
-)
+import "strings"
 
 // Action is an action taken by the browser to reach a navigation
 //
@@ -33,29 +29,6 @@ type Action struct {
 	ActionType ActionTypeHolder `yaml:"action" json:"action" jsonschema:"title=action to perform,description=Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"`
 }
 
-func (a Action) JSONSchemaExtend(schema *jsonschema.Schema) {
-	argsSchema, ok := schema.Properties.Get("args")
-	if !ok {
-		return
-	}
-	argsSchema.PatternProperties = map[string]*jsonschema.Schema{
-		".*": {
-			OneOf: []*jsonschema.Schema{
-				{
-					Type: "string",
-				},
-				{
-					Type: "integer",
-				},
-				{
-					Type: "boolean",
-				},
-			},
-		},
-	}
-	argsSchema.Ref = ""
-}
-
 // String returns the string representation of an action
 func (a *Action) String() string {
 	builder := &strings.Builder{}

pkg/protocols/headless/engine/action_types.go

@@ -171,7 +171,7 @@ type ActionTypeHolder struct {
 func (holder ActionTypeHolder) String() string {
 	return holder.ActionType.String()
 }
-func (holder ActionTypeHolder) JSONSchema() *jsonschema.Schema {
+func (holder ActionTypeHolder) JSONSchemaType() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "action to perform",

pkg/protocols/headless/engine/page_actions.go

@@ -214,7 +214,7 @@ func geTimeParameter(p *Page, act *Action, parameterName string, defaultValue ti
 }
 
 // ActionAddHeader executes a AddHeader action.
-func (p *Page) ActionAddHeader(act *Action, out map[string]string) error {
+func (p *Page) ActionAddHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error {
 	in := p.getActionArgWithDefaultValues(act, "part")
 
 	args := make(map[string]string)
@@ -225,7 +225,7 @@ func (p *Page) ActionAddHeader(act *Action, out map[string]string) error {
 }
 
 // ActionSetHeader executes a SetHeader action.
-func (p *Page) ActionSetHeader(act *Action, out map[string]string) error {
+func (p *Page) ActionSetHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error {
 	in := p.getActionArgWithDefaultValues(act, "part")
 
 	args := make(map[string]string)
@@ -236,7 +236,7 @@ func (p *Page) ActionSetHeader(act *Action, out map[string]string) error {
 }
 
 // ActionDeleteHeader executes a DeleteHeader action.
-func (p *Page) ActionDeleteHeader(act *Action, out map[string]string) error {
+func (p *Page) ActionDeleteHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error {
 	in := p.getActionArgWithDefaultValues(act, "part")
 
 	args := make(map[string]string)
@@ -343,7 +343,7 @@ func (p *Page) RunScript(action *Action, out map[string]string) error {
 }
 
 // ClickElement executes click actions for an element.
-func (p *Page) ClickElement(act *Action, out map[string]string) error {
+func (p *Page) ClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
 	element, err := p.pageElementBy(act.Data)
 	if err != nil {
 		return errors.Wrap(err, errCouldNotGetElement)
@@ -358,12 +358,12 @@ func (p *Page) ClickElement(act *Action, out map[string]string) error {
 }
 
 // KeyboardAction executes a keyboard action on the page.
-func (p *Page) KeyboardAction(act *Action, out map[string]string) error {
+func (p *Page) KeyboardAction(act *Action, out map[string]string /*TODO review unused parameter*/) error {
 	return p.page.Keyboard.Type([]input.Key(p.getActionArgWithDefaultValues(act, "keys"))...)
 }
 
 // RightClickElement executes right click actions for an element.
-func (p *Page) RightClickElement(act *Action, out map[string]string) error {
+func (p *Page) RightClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
 	element, err := p.pageElementBy(act.Data)
 	if err != nil {
 		return errors.Wrap(err, errCouldNotGetElement)
@@ -441,7 +441,7 @@ func (p *Page) Screenshot(act *Action, out map[string]string) error {
 }
 
 // InputElement executes input element actions for an element.
-func (p *Page) InputElement(act *Action, out map[string]string) error {
+func (p *Page) InputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
 	value := p.getActionArgWithDefaultValues(act, "value")
 	if value == "" {
 		return errinvalidArguments
@@ -460,7 +460,7 @@ func (p *Page) InputElement(act *Action, out map[string]string) error {
 }
 
 // TimeInputElement executes time input on an element
-func (p *Page) TimeInputElement(act *Action, out map[string]string) error {
+func (p *Page) TimeInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
 	value := p.getActionArgWithDefaultValues(act, "value")
 	if value == "" {
 		return errinvalidArguments
@@ -483,7 +483,7 @@ func (p *Page) TimeInputElement(act *Action, out map[string]string) error {
 }
 
 // SelectInputElement executes select input statement action on a element
-func (p *Page) SelectInputElement(act *Action, out map[string]string) error {
+func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
 	value := p.getActionArgWithDefaultValues(act, "value")
 	if value == "" {
 		return errinvalidArguments
@@ -508,7 +508,7 @@ func (p *Page) SelectInputElement(act *Action, out map[string]string) error {
 }
 
 // WaitLoad waits for the page to load
-func (p *Page) WaitLoad(act *Action, out map[string]string) error {
+func (p *Page) WaitLoad(act *Action, out map[string]string /*TODO review unused parameter*/) error {
 	p.page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)()
 
 	// Wait for the window.onload event and also wait for the network requests
@@ -538,7 +538,7 @@ func (p *Page) GetResource(act *Action, out map[string]string) error {
 }
 
 // FilesInput acts with a file input element on page
-func (p *Page) FilesInput(act *Action, out map[string]string) error {
+func (p *Page) FilesInput(act *Action, out map[string]string /*TODO review unused parameter*/) error {
 	element, err := p.pageElementBy(act.Data)
 	if err != nil {
 		return errors.Wrap(err, errCouldNotGetElement)
@@ -589,7 +589,7 @@ func (p *Page) ExtractElement(act *Action, out map[string]string) error {
 }
 
 // WaitEvent waits for an event to happen on the page.
-func (p *Page) WaitEvent(act *Action, out map[string]string) (func() error, error) {
+func (p *Page) WaitEvent(act *Action, out map[string]string /*TODO review unused parameter*/) (func() error, error) {
 	event := p.getActionArgWithDefaultValues(act, "event")
 	if event == "" {
 		return nil, errors.New("event not recognized")
@@ -661,14 +661,14 @@ func (p *Page) pageElementBy(data map[string]string) (*rod.Element, error) {
 }
 
 // DebugAction enables debug action on a page.
-func (p *Page) DebugAction(act *Action, out map[string]string) error {
+func (p *Page) DebugAction(act *Action, out map[string]string /*TODO review unused parameter*/) error {
 	p.instance.browser.engine.SlowMotion(5 * time.Second)
 	p.instance.browser.engine.Trace(true)
 	return nil
 }
 
 // SleepAction sleeps on the page for a specified duration
-func (p *Page) SleepAction(act *Action, out map[string]string) error {
+func (p *Page) SleepAction(act *Action, out map[string]string /*TODO review unused parameter*/) error {
 	seconds := act.Data["duration"]
 	if seconds == "" {
 		seconds = "5"

pkg/protocols/http/http.go

@@ -5,11 +5,9 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/invopop/jsonschema"
 	json "github.com/json-iterator/go"
 	"github.com/pkg/errors"
 
-	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/fuzz"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
@@ -221,29 +219,6 @@ type Request struct {
 	fuzzPreConditionOperator matchers.ConditionType `yaml:"-" json:"-"`
 }
 
-func (e Request) JSONSchemaExtend(schema *jsonschema.Schema) {
-	headersSchema, ok := schema.Properties.Get("headers")
-	if !ok {
-		return
-	}
-	headersSchema.PatternProperties = map[string]*jsonschema.Schema{
-		".*": {
-			OneOf: []*jsonschema.Schema{
-				{
-					Type: "string",
-				},
-				{
-					Type: "integer",
-				},
-				{
-					Type: "boolean",
-				},
-			},
-		},
-	}
-	headersSchema.Ref = ""
-}
-
 // Options returns executer options for http request
 func (r *Request) Options() *protocols.ExecutorOptions {
 	return r.options
@@ -436,34 +411,12 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 		}
 	}
 	if len(request.Payloads) > 0 {
-		// Due to a known issue (https://github.com/projectdiscovery/nuclei/issues/5015),
-		// dynamic extractors cannot be used with payloads. To address this,
-		// execution is handled by the standard engine without concurrency,
-		// achieved by setting the thread count to 0.
-
-		// this limitation will be removed once we have a better way to handle dynamic extractors with payloads
-		hasMultipleRequests := false
-		if len(request.Raw)+len(request.Path) > 1 {
-			hasMultipleRequests = true
-		}
-		// look for dynamic extractor ( internal: true with named extractor)
-		hasNamedInternalExtractor := false
-		for _, extractor := range request.Extractors {
-			if extractor.Internal && extractor.Name != "" {
-				hasNamedInternalExtractor = true
-				break
-			}
-		}
-		if hasNamedInternalExtractor && hasMultipleRequests {
-			gologger.Warning().Label(options.TemplateID).Msgf("Setting thread count to 0 because dynamic extractors are not supported with payloads yet")
-			request.Threads = 0
-		} else {
-			// specifically for http requests high concurrency and and threads will lead to memory exausthion, hence reduce the maximum parallelism
-			if protocolstate.IsLowOnMemory() {
-				request.Threads = protocolstate.GuardThreadsOrDefault(request.Threads)
-			}
-			request.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads)
+		// specifically for http requests high concurrency and and threads will lead to memory exausthion, hence reduce the maximum parallelism
+		if protocolstate.IsLowOnMemory() {
+			request.Threads = protocolstate.GuardThreadsOrDefault(request.Threads)
 		}
+		// if we have payloads, adjust threads if none specified
+		request.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads)
 	}
 
 	return nil

pkg/protocols/http/http_method_types.go

@@ -89,7 +89,7 @@ func (holder HTTPMethodTypeHolder) String() string {
 	return holder.MethodType.String()
 }
 
-func (holder HTTPMethodTypeHolder) JSONSchema() *jsonschema.Schema {
+func (holder HTTPMethodTypeHolder) JSONSchemaType() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "method is the HTTP request method",

pkg/protocols/http/httputils/spm.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"sync"
 
-	syncutil "github.com/projectdiscovery/utils/sync"
+	"github.com/remeh/sizedwaitgroup"
 )
 
 // WorkPoolType is the type of work pool to use
@@ -26,7 +26,7 @@ type StopAtFirstMatchHandler[T any] struct {
 
 	// work pool and its type
 	poolType WorkPoolType
-	sgPool   *syncutil.AdaptiveWaitGroup
+	sgPool   sizedwaitgroup.SizedWaitGroup
 	wgPool   *sync.WaitGroup
 
 	// internal / unexported
@@ -40,13 +40,10 @@ type StopAtFirstMatchHandler[T any] struct {
 // NewBlockingSPMHandler creates a new stop at first match handler
 func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *StopAtFirstMatchHandler[T] {
 	ctx1, cancel := context.WithCancel(ctx)
-
-	awg, _ := syncutil.New(syncutil.WithSize(size))
-
 	s := &StopAtFirstMatchHandler[T]{
 		ResultChan:  make(chan T, 1),
 		poolType:    Blocking,
-		sgPool:      awg,
+		sgPool:      sizedwaitgroup.New(size),
 		internalWg:  &sync.WaitGroup{},
 		ctx:         ctx1,
 		cancel:      cancel,
@@ -143,16 +140,6 @@ func (h *StopAtFirstMatchHandler[T]) Release() {
 	}
 }
 
-func (h *StopAtFirstMatchHandler[T]) Resize(size int) {
-	if h.sgPool.Size != size {
-		h.sgPool.Resize(size)
-	}
-}
-
-func (h *StopAtFirstMatchHandler[T]) Size() int {
-	return h.sgPool.Size
-}
-
 // Wait waits for all work to be done
 func (h *StopAtFirstMatchHandler[T]) Wait() {
 	switch h.poolType {

pkg/protocols/http/request.go

@@ -165,9 +165,6 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 	// Workers that keeps enqueuing new requests
 	maxWorkers := request.Threads
 
-	// if request threads matches global payload concurrency we follow it
-	shouldFollowGlobal := maxWorkers == request.options.Options.PayloadConcurrency
-
 	if protocolstate.IsLowOnMemory() {
 		maxWorkers = protocolstate.GuardThreadsOrDefault(request.Threads)
 	}
@@ -201,12 +198,6 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		if !ok {
 			break
 		}
-
-		// resize check point - nop if there are no changes
-		if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency {
-			spmHandler.Resize(request.options.Options.PayloadConcurrency)
-		}
-
 		ctx := request.newContext(input)
 		generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
 		if err != nil {
@@ -231,7 +222,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 				return
 			case spmHandler.ResultChan <- func() error {
 				// putting ratelimiter here prevents any unnecessary waiting if any
-				request.options.RateLimitTake()
+				request.options.RateLimiter.Take()
 				previous := make(map[string]interface{})
 				return request.executeRequest(input, httpRequest, previous, false, wrappedCallback, 0)
 			}():
@@ -375,7 +366,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 		executeFunc := func(data string, payloads, dynamicValue map[string]interface{}) (bool, error) {
 			hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
 
-			request.options.RateLimitTake()
+			request.options.RateLimiter.Take()
 
 			ctx := request.newContext(input)
 			ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(request.options.Options.Timeout)*time.Second)

pkg/protocols/http/request_fuzz.go

@@ -145,7 +145,7 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
 	if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) {
 		return false
 	}
-	request.options.RateLimitTake()
+	request.options.RateLimiter.Take()
 	req := &generatedRequest{
 		request:        gr.Request,
 		dynamicValues:  gr.DynamicValues,

pkg/protocols/http/signature.go

@@ -51,7 +51,7 @@ type SignatureTypeHolder struct {
 	Value SignatureType
 }
 
-func (holder SignatureTypeHolder) JSONSchema() *jsonschema.Schema {
+func (holder SignatureTypeHolder) JSONSchemaType() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "type of the signature",

pkg/protocols/javascript/js.go

@@ -27,15 +27,13 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
 	protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	errorutil "github.com/projectdiscovery/utils/errors"
-	iputil "github.com/projectdiscovery/utils/ip"
-	syncutil "github.com/projectdiscovery/utils/sync"
 	urlutil "github.com/projectdiscovery/utils/url"
+	"github.com/remeh/sizedwaitgroup"
 )
 
 // Request is a request for the javascript protocol
@@ -406,11 +404,7 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 	requestOptions := request.options
 	gotmatches := &atomic.Bool{}
 
-	// if request threads matches global payload concurrency we follow it
-	shouldFollowGlobal := threads == request.options.Options.PayloadConcurrency
-
-	sg, _ := syncutil.New(syncutil.WithSize(threads))
-
+	sg := sizedwaitgroup.New(threads)
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
 		for {
@@ -418,12 +412,6 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 			if !ok {
 				break
 			}
-
-			// resize check point - nop if there are no changes
-			if shouldFollowGlobal && sg.Size != request.options.Options.PayloadConcurrency {
-				sg.Resize(request.options.Options.PayloadConcurrency)
-			}
-
 			sg.Add()
 			go func() {
 				defer sg.Done()
@@ -532,46 +520,6 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
 		data["stop-at-first-match"] = true
 	}
 
-	// add ip address to data
-	if input.MetaInput.CustomIP != "" {
-		data["ip"] = input.MetaInput.CustomIP
-	} else {
-		// context: https://github.com/projectdiscovery/nuclei/issues/5021
-		hostname := input.MetaInput.Input
-		if strings.Contains(hostname, ":") {
-			host, _, err := net.SplitHostPort(hostname)
-			if err == nil {
-				hostname = host
-			} else {
-				// naive way
-				if !strings.Contains(hostname, "]") {
-					hostname = hostname[:strings.LastIndex(hostname, ":")]
-				}
-			}
-		}
-		data["ip"] = protocolstate.Dialer.GetDialedIP(hostname)
-		// if input itself was an ip, use it
-		if iputil.IsIP(hostname) {
-			data["ip"] = hostname
-		}
-
-		// if ip is not found,this is because ssh and other protocols do not use fastdialer
-		// although its not perfect due to its use case dial and get ip
-		dnsData, err := protocolstate.Dialer.GetDNSData(hostname)
-		if err == nil {
-			for _, v := range dnsData.A {
-				data["ip"] = v
-				break
-			}
-			if data["ip"] == "" {
-				for _, v := range dnsData.AAAA {
-					data["ip"] = v
-					break
-				}
-			}
-		}
-	}
-
 	// add and get values from templatectx
 	request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), data)
 	data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll())

pkg/protocols/network/network.go

@@ -61,7 +61,7 @@ type Request struct {
 	// description: |
 	//   Port is the port to send network requests to. this acts as default port but is overriden if target/input contains
 	// non-http(s) ports like 80,8080,8081 etc
-	Port string `yaml:"port,omitempty" json:"port,omitempty" jsonschema:"title=port to send requests to,description=Port to send network requests to,oneof_type=string;integer"`
+	Port string `yaml:"port,omitempty" json:"port,omitempty" jsonschema:"title=port to send requests to,description=Port to send network requests to"`
 
 	// description:	|
 	//	ExcludePorts is the list of ports to exclude from being scanned . It is intended to be used with `Port` field and contains a list of ports which are ignored/skipped
@@ -91,7 +91,7 @@ type Request struct {
 
 	// Operators for the current request go here.
 	operators.Operators `yaml:",inline,omitempty"`
-	CompiledOperators   *operators.Operators `yaml:"-" json:"-"`
+	CompiledOperators   *operators.Operators `yaml:"-"`
 
 	generator *generators.PayloadGenerator
 	// cache any variables that may be needed for operation.
@@ -128,7 +128,7 @@ type Input struct {
 	// examples:
 	//   - value: "\"TEST\""
 	//   - value: "\"hex_decode('50494e47')\""
-	Data string `yaml:"data,omitempty" json:"data,omitempty" jsonschema:"title=data to send as input,description=Data is the data to send as the input,oneof_type=string;integer"`
+	Data string `yaml:"data,omitempty" json:"data,omitempty" jsonschema:"title=data to send as input,description=Data is the data to send as the input"`
 	// description: |
 	//   Type is the type of input specified in `data` field.
 	//

pkg/protocols/network/network_input_types.go

@@ -66,7 +66,7 @@ func (holder NetworkInputTypeHolder) String() string {
 	return holder.NetworkInputType.String()
 }
 
-func (holder NetworkInputTypeHolder) JSONSchema() *jsonschema.Schema {
+func (holder NetworkInputTypeHolder) JSONSchemaType() *jsonschema.Schema {
 	gotType := &jsonschema.Schema{
 		Type:        "string",
 		Title:       "type is the type of input data",

pkg/protocols/network/networkclientpool/clientpool.go

@@ -11,7 +11,7 @@ var (
 )
 
 // Init initializes the clientpool implementation
-func Init(options *types.Options) error {
+func Init(options *types.Options /*TODO review unused parameter*/) error {
 	// Don't create clients if already created in the past.
 	if normalClient != nil {
 		return nil

pkg/protocols/network/request.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/remeh/sizedwaitgroup"
 	"go.uber.org/multierr"
 	"golang.org/x/exp/maps"
 
@@ -33,7 +34,6 @@ import (
 	errorutil "github.com/projectdiscovery/utils/errors"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	"github.com/projectdiscovery/utils/reader"
-	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var (
@@ -174,29 +174,17 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 		return err
 	}
 
-	// if request threads matches global payload concurrency we follow it
-	shouldFollowGlobal := request.Threads == request.options.Options.PayloadConcurrency
-
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
 		var multiErr error
 		m := &sync.Mutex{}
-		swg, err := syncutil.New(syncutil.WithSize(request.Threads))
-		if err != nil {
-			return err
-		}
+		swg := sizedwaitgroup.New(request.Threads)
 
 		for {
 			value, ok := iterator.Value()
 			if !ok {
 				break
 			}
-
-			// resize check point - nop if there are no changes
-			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
-				swg.Resize(request.options.Options.PayloadConcurrency)
-			}
-
 			value = generators.MergeMaps(value, payloads)
 			swg.Add()
 			go func(vars map[string]interface{}) {

pkg/protocols/offlinehttp/request.go

@@ -6,6 +6,7 @@ import (
 	"os"
 
 	"github.com/pkg/errors"
+	"github.com/remeh/sizedwaitgroup"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
@@ -16,7 +17,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/utils/conversion"
-	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var _ protocols.Request = &Request{}
@@ -29,13 +29,10 @@ func (request *Request) Type() templateTypes.ProtocolType {
 }
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
-func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
-	wg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize))
-	if err != nil {
-		return err
-	}
+func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
+	wg := sizedwaitgroup.New(request.options.Options.BulkSize)
 
-	err = request.getInputPaths(input.MetaInput.Input, func(data string) {
+	err := request.getInputPaths(input.MetaInput.Input, func(data string) {
 		wg.Add()
 
 		go func(data string) {

pkg/protocols/protocols.go

@@ -34,6 +34,9 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 )
 
+// Optional Callback to update Thread count in payloads across all requests
+type PayloadThreadSetterCallback func(opts *ExecutorOptions, totalRequests, currentThreads int) int
+
 var (
 	MaxTemplateFileSizeForEncoding = 1024 * 1024
 )
@@ -111,6 +114,10 @@ type ExecutorOptions struct {
 	// JsCompiler is abstracted javascript compiler which adds node modules and provides execution
 	// environment for javascript templates
 	JsCompiler *compiler.Compiler
+	// Optional Callback function to update Thread count in payloads across all protocols
+	// based on given logic. by default nuclei reverts to using value of `-c` when threads count
+	// is not specified or is 0 in template
+	OverrideThreadsCount PayloadThreadSetterCallback
 	// AuthProvider is a provider for auth strategies
 	AuthProvider authprovider.AuthProvider
 	//TemporaryDirectory is the directory to store temporary files
@@ -121,25 +128,17 @@ type ExecutorOptions struct {
 	ExportReqURLPattern bool
 }
 
-// todo: centralizing components is not feasible with current clogged architecture
-// a possible approach could be an internal event bus with pub-subs? This would be less invasive than
-// reworking dep injection from scratch
-func (eo *ExecutorOptions) RateLimitTake() {
-	if eo.RateLimiter.GetLimit() != uint(eo.Options.RateLimit) {
-		eo.RateLimiter.SetLimit(uint(eo.Options.RateLimit))
-		eo.RateLimiter.SetDuration(eo.Options.RateLimitDuration)
-	}
-	eo.RateLimiter.Take()
-}
-
 // GetThreadsForPayloadRequests returns the number of threads to use as default for
 // given max-request of payloads
 func (e *ExecutorOptions) GetThreadsForNPayloadRequests(totalRequests int, currentThreads int) int {
+	if e.OverrideThreadsCount != nil {
+		return e.OverrideThreadsCount(e, totalRequests, currentThreads)
+	}
 	if currentThreads > 0 {
 		return currentThreads
+	} else {
+		return e.Options.PayloadConcurrency
 	}
-
-	return e.Options.PayloadConcurrency
 }
 
 // CreateTemplateCtxStore creates template context store (which contains templateCtx for every scan)

pkg/protocols/ssl/ssl.go

@@ -282,7 +282,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 	} else {
 		data["ip"] = request.dialer.GetDialedIP(hostname)
 	}
-	data["Port"] = port
 	data["template-path"] = requestOptions.TemplatePath
 	data["template-id"] = requestOptions.TemplateID
 	data["template-info"] = requestOptions.TemplateInfo
@@ -406,9 +405,6 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 	if fields.Port == "80" {
 		fields.Port = "443"
 	}
-	if types.ToString(wrapped.InternalEvent["Port"]) != "" {
-		fields.Port = types.ToString(wrapped.InternalEvent["Port"])
-	}
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),
 		TemplatePath:     types.ToString(wrapped.InternalEvent["template-path"]),

pkg/reporting/exporters/es/elasticsearch.go

@@ -131,8 +131,7 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
 	if err != nil {
 		return err
 	}
-	defer res.Body.Close() 
-	
+
 	b, err = io.ReadAll(res.Body)
 	if err != nil {
 		return errors.New(err.Error() + "error thrown by elasticsearch " + string(b))

pkg/scan/charts/charts.go

@@ -1,87 +0,0 @@
-package charts
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"path/filepath"
-
-	"github.com/labstack/echo/v4"
-	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
-	fileutil "github.com/projectdiscovery/utils/file"
-)
-
-// ScanEventsCharts is a struct for nuclei event charts
-type ScanEventsCharts struct {
-	eventsDir string
-	config    *events.ScanConfig
-	data      []events.ScanEvent
-}
-
-func (sc *ScanEventsCharts) PrintInfo() {
-	fmt.Printf("[+] Scan Info\n")
-	fmt.Printf("  - Name: %s\n", sc.config.Name)
-	fmt.Printf("  - Target Count: %d\n", sc.config.TargetCount)
-	fmt.Printf("  - Template Count: %d\n", sc.config.TemplatesCount)
-	fmt.Printf("  - Template Concurrency: %d\n", sc.config.TemplateConcurrency)
-	fmt.Printf("  - Payload Concurrency: %d\n", sc.config.PayloadConcurrency)
-	fmt.Printf("  - Retries: %v\n", sc.config.Retries)
-	fmt.Printf("  - Total Events: %d\n", len(sc.data))
-	fmt.Println()
-}
-
-// NewScanEventsCharts creates a new nuclei event charts
-func NewScanEventsCharts(eventsDir string) (*ScanEventsCharts, error) {
-	sc := &ScanEventsCharts{eventsDir: eventsDir}
-	if !fileutil.FolderExists(eventsDir) {
-		return nil, fmt.Errorf("events directory does not exist")
-	}
-	// open two files
-	// config.json
-	bin, err := os.ReadFile(filepath.Join(eventsDir, events.ConfigFile))
-	if err != nil {
-		return nil, err
-	}
-	var config events.ScanConfig
-	err = json.Unmarshal(bin, &config)
-	if err != nil {
-		return nil, err
-	}
-	sc.config = &config
-
-	// events.jsonl
-	f, err := os.Open(filepath.Join(eventsDir, events.EventsFile))
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	data := []events.ScanEvent{}
-	dec := json.NewDecoder(f)
-	for {
-		var event events.ScanEvent
-		if err := dec.Decode(&event); err != nil {
-			break
-		}
-		data = append(data, event)
-	}
-	sc.data = data
-
-	if len(data) == 0 {
-		return nil, fmt.Errorf("no events found in the events file")
-	}
-
-	return sc, nil
-}
-
-// Start starts the nuclei event charts server
-func (sc *ScanEventsCharts) Start(addr string) {
-	e := echo.New()
-	e.HideBanner = true
-	e.GET("/concurrency", sc.ConcurrencyVsTime)
-	e.GET("/requests", sc.TotalRequestsOverTime)
-	e.GET("/slow", sc.TopSlowTemplates)
-	e.GET("/rps", sc.RequestsVSInterval)
-	e.GET("/", sc.AllCharts)
-	e.Logger.Fatal(e.Start(addr))
-}

pkg/scan/charts/echarts.go

@@ -1,351 +0,0 @@
-package charts
-
-import (
-	"fmt"
-	"os"
-	"sort"
-	"time"
-
-	"github.com/go-echarts/go-echarts/v2/charts"
-	"github.com/go-echarts/go-echarts/v2/components"
-	"github.com/go-echarts/go-echarts/v2/opts"
-	"github.com/labstack/echo/v4"
-	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
-	sliceutil "github.com/projectdiscovery/utils/slice"
-)
-
-const (
-	TopK         = 50
-	SpacerHeight = "50px"
-)
-
-func (s *ScanEventsCharts) AllCharts(c echo.Context) error {
-	page := s.allCharts(c)
-	return page.Render(c.Response().Writer)
-}
-
-func (s *ScanEventsCharts) GenerateHTML(filePath string) error {
-	page := s.allCharts(nil)
-	output, err := os.Create(filePath)
-	if err != nil {
-		return err
-	}
-	return page.Render(output)
-}
-
-// AllCharts generates all the charts for the scan events and returns a page component
-func (s *ScanEventsCharts) allCharts(c echo.Context) *components.Page {
-	page := components.NewPage()
-	page.PageTitle = "Nuclei Charts"
-	line1 := s.totalRequestsOverTime(c)
-	line1.SetSpacerHeight(SpacerHeight)
-	kline := s.topSlowTemplates(c)
-	kline.SetSpacerHeight(SpacerHeight)
-	line2 := s.requestsVSInterval(c)
-	line2.SetSpacerHeight(SpacerHeight)
-	line3 := s.concurrencyVsTime(c)
-	line3.SetSpacerHeight(SpacerHeight)
-	page.AddCharts(line1, kline, line2, line3)
-	page.Validate()
-	page.SetLayout(components.PageCenterLayout)
-	return page
-}
-
-func (s *ScanEventsCharts) TotalRequestsOverTime(c echo.Context) error {
-	line := s.totalRequestsOverTime(c)
-	return line.Render(c.Response().Writer)
-}
-
-// totalRequestsOverTime generates a line chart showing total requests count over time
-func (s *ScanEventsCharts) totalRequestsOverTime(c echo.Context) *charts.Line {
-	line := charts.NewLine()
-	line.SetCaption("Chart Shows Total Requests Count Over Time (for each/all Protocols)")
-
-	var startTime time.Time = time.Now()
-	var endTime time.Time
-
-	for _, event := range s.data {
-		if event.Time.Before(startTime) {
-			startTime = event.Time
-		}
-		if event.Time.After(endTime) {
-			endTime = event.Time
-		}
-	}
-	data := getCategoryRequestCount(s.data)
-	max := 0
-	for _, v := range data {
-		if len(v) > max {
-			max = len(v)
-		}
-	}
-	line.SetXAxis(time.Now().Format(time.RFC3339))
-	for k, v := range data {
-		lineData := make([]opts.LineData, 0)
-		temp := 0
-		for _, scanEvent := range v {
-			temp += scanEvent.MaxRequests
-			val := scanEvent.Time.Sub(startTime)
-			lineData = append(lineData, opts.LineData{
-				Value: []interface{}{val.Milliseconds(), temp},
-				Name:  scanEvent.TemplateID,
-			})
-		}
-		line.AddSeries(k, lineData, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"}))
-	}
-
-	line.SetGlobalOptions(
-		charts.WithTitleOpts(opts.Title{Title: "Nuclei: total-req vs time"}),
-		charts.WithXAxisOpts(opts.XAxis{Name: "Time", Type: "time", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}),
-		charts.WithYAxisOpts(opts.YAxis{Name: "Requests Sent", Type: "value"}),
-		charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}),
-		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
-		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}),
-		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
-			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
-			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
-			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
-		}}),
-	)
-
-	line.Validate()
-	return line
-}
-
-func (s *ScanEventsCharts) TopSlowTemplates(c echo.Context) error {
-	kline := s.topSlowTemplates(c)
-	return kline.Render(c.Response().Writer)
-}
-
-// topSlowTemplates generates a Kline chart showing the top slow templates by time taken
-func (s *ScanEventsCharts) topSlowTemplates(c echo.Context) *charts.Kline {
-	kline := charts.NewKLine()
-	kline.SetCaption(fmt.Sprintf("Chart Shows Top Slow Templates (by time taken) (Top %v)", TopK))
-
-	ids := map[string][]int64{}
-	var startTime time.Time = time.Now()
-	for _, event := range s.data {
-		if event.Time.Before(startTime) {
-			startTime = event.Time
-		}
-	}
-	for _, event := range s.data {
-		ids[event.TemplateID] = append(ids[event.TemplateID], event.Time.Sub(startTime).Milliseconds())
-	}
-
-	type entry struct {
-		ID        string
-		KlineData opts.KlineData
-		start     int64
-		end       int64
-	}
-	data := []entry{}
-
-	for a, b := range ids {
-		if len(b) < 2 {
-			continue // Prevents index out of range error
-		}
-		d := entry{
-			ID:        a,
-			KlineData: opts.KlineData{Value: []int64{b[0], b[len(b)-1], b[0], b[len(b)-1]}}, // Adjusted to prevent index out of range error
-			start:     b[0],
-			end:       b[len(b)-1],
-		}
-		data = append(data, d)
-	}
-
-	sort.Slice(data, func(i, j int) bool {
-		return data[i].end-data[i].start > data[j].end-data[j].start
-	})
-
-	x := make([]string, 0)
-	y := make([]opts.KlineData, 0)
-	for _, event := range data[:TopK] {
-		x = append(x, event.ID)
-		y = append(y, event.KlineData)
-	}
-
-	kline.SetXAxis(x).AddSeries("templates", y)
-	kline.SetGlobalOptions(
-		charts.WithTitleOpts(opts.Title{Title: fmt.Sprintf("Nuclei: Top %v Slow Templates", TopK)}),
-		charts.WithXAxisOpts(opts.XAxis{
-			Type:      "category",
-			Show:      true,
-			AxisLabel: &opts.AxisLabel{Rotate: 90, Show: true, ShowMinLabel: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (value) { return value; }`)},
-		}),
-		charts.WithYAxisOpts(opts.YAxis{
-			Scale:     true,
-			Type:      "value",
-			Show:      true,
-			AxisLabel: &opts.AxisLabel{Show: true, Formatter: opts.FuncOpts(`function (ms) {  return Math.floor(ms/60000) + 'm' + Math.floor((ms/60000 - Math.floor(ms/60000))*60) + 's'; }`)},
-		}),
-		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
-		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "40%", Top: "10%"}),
-		charts.WithTooltipOpts(opts.Tooltip{Show: true, Trigger: "events.ScanEvent", TriggerOn: "mousemove|click", Enterable: true, Formatter: opts.FuncOpts(`function (params) { return params.name ; }`)}),
-		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
-			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
-			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
-			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
-		}}),
-	)
-
-	return kline
-}
-
-func (s *ScanEventsCharts) RequestsVSInterval(c echo.Context) error {
-	line := s.requestsVSInterval(c)
-	return line.Render(c.Response().Writer)
-}
-
-// requestsVSInterval generates a line chart showing requests per second over time
-func (s *ScanEventsCharts) requestsVSInterval(c echo.Context) *charts.Line {
-	line := charts.NewLine()
-	line.SetCaption("Chart Shows RPS (Requests Per Second) Over Time")
-
-	sort.Slice(s.data, func(i, j int) bool {
-		return s.data[i].Time.Before(s.data[j].Time)
-	})
-
-	var interval time.Duration
-
-	if c != nil {
-		interval, _ = time.ParseDuration(c.QueryParam("interval"))
-	}
-	if interval <= 3 {
-		interval = 5 * time.Second
-	}
-
-	data := []opts.LineData{}
-	temp := 0
-	if len(s.data) > 0 {
-		orig := s.data[0].Time
-		startTime := orig
-		xaxisData := []int64{}
-		for _, v := range s.data {
-			if v.Time.Sub(startTime) > interval {
-				millisec := v.Time.Sub(orig).Milliseconds()
-				xaxisData = append(xaxisData, millisec)
-				data = append(data, opts.LineData{Value: temp, Name: v.Time.Sub(orig).String()})
-				temp = 0
-				startTime = v.Time
-			}
-			temp += 1
-		}
-		// Handle last interval if exists
-		if temp > 0 {
-			millisec := s.data[len(s.data)-1].Time.Sub(orig).Milliseconds()
-			xaxisData = append(xaxisData, millisec)
-			data = append(data, opts.LineData{Value: temp, Name: s.data[len(s.data)-1].Time.Sub(orig).String()})
-		}
-		line.SetXAxis(xaxisData)
-		line.AddSeries("RPS", data, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"}))
-	}
-
-	line.SetGlobalOptions(
-		charts.WithTitleOpts(opts.Title{Title: "Nuclei: Template Execution", Subtitle: "Time Interval: " + interval.String()}),
-		charts.WithXAxisOpts(opts.XAxis{Name: "Time Intervals", Type: "category", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}),
-		charts.WithYAxisOpts(opts.YAxis{Name: "RPS Value", Type: "value", Show: true}),
-		charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}),
-		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
-		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}),
-		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
-			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
-			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
-			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
-		}}),
-	)
-
-	line.Validate()
-	return line
-}
-
-func (s *ScanEventsCharts) ConcurrencyVsTime(c echo.Context) error {
-	line := s.concurrencyVsTime(c)
-	return line.Render(c.Response().Writer)
-}
-
-// concurrencyVsTime generates a line chart showing concurrency (total workers) over time
-func (s *ScanEventsCharts) concurrencyVsTime(c echo.Context) *charts.Line {
-	line := charts.NewLine()
-	line.SetCaption("Chart Shows Concurrency (Total Workers) Over Time")
-
-	dataset := sliceutil.Clone(s.data)
-
-	sort.Slice(dataset, func(i, j int) bool {
-		return dataset[i].Time.Before(dataset[j].Time)
-	})
-
-	var interval time.Duration
-	if c != nil {
-		interval, _ = time.ParseDuration(c.QueryParam("interval"))
-	}
-	if interval <= 3 {
-		interval = 5 * time.Second
-	}
-
-	// create array with time interval as x-axis and worker count as y-axis
-	// entry is a struct with time and poolsize
-	type entry struct {
-		Time     time.Duration
-		poolsize int
-	}
-	allEntries := []entry{}
-
-	dataIndex := 0
-	maxIndex := len(dataset) - 1
-	currEntry := entry{}
-
-	lastTime := dataset[0].Time
-	for dataIndex <= maxIndex {
-		currTime := dataset[dataIndex].Time
-		if currTime.Sub(lastTime) > interval {
-			// next batch
-			currEntry.Time = interval
-			allEntries = append(allEntries, currEntry)
-			lastTime = dataset[dataIndex-1].Time
-		}
-		if dataset[dataIndex].EventType == events.ScanStarted {
-			currEntry.poolsize += 1
-		} else {
-			currEntry.poolsize -= 1
-		}
-		dataIndex += 1
-	}
-
-	plotData := []opts.LineData{}
-	xaxisData := []int64{}
-	tempTime := time.Duration(0)
-	for _, v := range allEntries {
-		tempTime += v.Time
-		plotData = append(plotData, opts.LineData{Value: v.poolsize, Name: tempTime.String()})
-		xaxisData = append(xaxisData, tempTime.Milliseconds())
-	}
-	line.SetXAxis(xaxisData)
-	line.AddSeries("Concurrency", plotData, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"}))
-
-	line.SetGlobalOptions(
-		charts.WithTitleOpts(opts.Title{Title: "Nuclei: WorkerPool", Subtitle: "Time Interval: " + interval.String()}),
-		charts.WithXAxisOpts(opts.XAxis{Name: "Time Intervals", Type: "category", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}),
-		charts.WithYAxisOpts(opts.YAxis{Name: "Total Workers", Type: "value", Show: true}),
-		charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}),
-		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
-		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}),
-		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
-			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
-			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
-			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
-		}}),
-	)
-
-	line.Validate()
-	return line
-}
-
-// getCategoryRequestCount returns a map of category and request count
-func getCategoryRequestCount(values []events.ScanEvent) map[string][]events.ScanEvent {
-	mx := make(map[string][]events.ScanEvent)
-	for _, event := range values {
-		mx[event.TemplateType] = append(mx[event.TemplateType], event)
-	}
-	return mx
-}

pkg/scan/events/scan_noop.go

@@ -1,11 +0,0 @@
-//go:build !stats
-// +build !stats
-
-package events
-
-// AddScanEvent is a no-op function
-func AddScanEvent(event ScanEvent) {
-}
-
-func InitWithConfig(config *ScanConfig, statsDirectory string) {
-}

pkg/scan/events/stats_build.go

@@ -1,80 +0,0 @@
-//go:build stats
-// +build stats
-
-package events
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"path/filepath"
-	"sync"
-	"time"
-)
-
-var _ ScanEventWorker = &ScanStatsWorker{}
-
-var defaultWorker = &ScanStatsWorker{}
-
-// ScanStatsWorker is a worker for scanning stats
-// This tracks basic stats in jsonlines format
-// in given directory or a default directory with name stats_{timestamp} in the current directory
-type ScanStatsWorker struct {
-	config    *ScanConfig
-	m         *sync.Mutex
-	directory string
-	enc       *json.Encoder
-}
-
-// Init initializes the scan stats worker
-func InitWithConfig(config *ScanConfig, statsDirectory string) {
-	currentTime := time.Now().Format("20060102150405")
-	dirName := fmt.Sprintf("nuclei-stats-%s", currentTime)
-	err := os.Mkdir(dirName, 0755)
-	if err != nil {
-		panic(err)
-	}
-	// save the config to the directory
-	bin, err := json.MarshalIndent(config, "", "  ")
-	if err != nil {
-		panic(err)
-	}
-	err = os.WriteFile(filepath.Join(dirName, ConfigFile), bin, 0755)
-	if err != nil {
-		panic(err)
-	}
-	defaultWorker = &ScanStatsWorker{config: config, m: &sync.Mutex{}, directory: dirName}
-	err = defaultWorker.initEventsFile()
-	if err != nil {
-		panic(err)
-	}
-}
-
-// initEventsFile initializes the events file for the worker
-func (s *ScanStatsWorker) initEventsFile() error {
-	f, err := os.Create(filepath.Join(s.directory, EventsFile))
-	if err != nil {
-		return err
-	}
-	s.enc = json.NewEncoder(f)
-	return nil
-}
-
-// AddScanEvent adds a scan event to the worker
-func (s *ScanStatsWorker) AddScanEvent(event ScanEvent) {
-	s.m.Lock()
-	defer s.m.Unlock()
-
-	err := s.enc.Encode(event)
-	if err != nil {
-		panic(err)
-	}
-}
-
-// AddScanEvent adds a scan event to the worker
-func AddScanEvent(event ScanEvent) {
-	if defaultWorker == nil {
-		return
-	}
-	defaultWorker.AddScanEvent(event)
-}

pkg/scan/events/utils.go

@@ -1,45 +0,0 @@
-package events
-
-import (
-	"time"
-)
-
-type ScanEventWorker interface {
-	// AddScanEvent adds a scan event to the worker
-	AddScanEvent(event ScanEvent)
-}
-
-// Track scan start / finish status
-type ScanStatus string
-
-const (
-	ScanStarted  ScanStatus = "scan_start"
-	ScanFinished ScanStatus = "scan_end"
-)
-
-const (
-	ConfigFile = "config.json"
-	EventsFile = "events.jsonl"
-)
-
-// ScanEvent represents a single scan event with its metadata
-type ScanEvent struct {
-	Target       string     `json:"target" yaml:"target"`
-	TemplateType string     `json:"template_type" yaml:"template_type"`
-	TemplateID   string     `json:"template_id" yaml:"template_id"`
-	TemplatePath string     `json:"template_path" yaml:"template_path"`
-	MaxRequests  int        `json:"max_requests" yaml:"max_requests"`
-	Time         time.Time  `json:"time" yaml:"time"`
-	EventType    ScanStatus `json:"event_type" yaml:"event_type"`
-}
-
-// ScanConfig is only in context of scan event analysis
-type ScanConfig struct {
-	Name                string `json:"name" yaml:"name"`
-	TargetCount         int    `json:"target_count" yaml:"target_count"`
-	TemplatesCount      int    `json:"templates_count" yaml:"templates_count"`
-	TemplateConcurrency int    `json:"template_concurrency" yaml:"template_concurrency"`
-	PayloadConcurrency  int    `json:"payload_concurrency" yaml:"payload_concurrency"`
-	JsConcurrency       int    `json:"js_concurrency" yaml:"js_concurrency"`
-	Retries             int    `json:"retries" yaml:"retries"`
-}

pkg/scan/scan_context.go

@@ -10,7 +10,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 )
 
-
 type ScanContextOption func(*ScanContext)
 
 func WithEvents() ScanContextOption {