feat: add proxy server + more enhancements additions

Signed-off-by: Dwi Siswanto <[email protected]>

.github/workflows/compability-check.yaml

@@ -1,28 +0,0 @@
-name: ♾️ Compatibility Check
-
-on:
-  pull_request:
-    types: [opened, synchronize]
-    branches:
-      - dev
-
-jobs:
-  check:
-    if: github.actor == 'dependabot[bot]'
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - uses: actions/checkout@v4
-      - uses: projectdiscovery/actions/setup/git@v1
-      - uses: projectdiscovery/actions/setup/go@v1
-      - run: go mod download && go mod verify && go vet ./...
-      - name: Checks go.mod Integrity
-        run: |
-          git diff --exit-code go.mod >/dev/null || {
-            echo "::warning::go.mod is out of sync. Pushing changes to the branch."
-            git add go.{mod,sum}
-            git commit -m "chore(deps): go mod tidy"
-            git push origin $GITHUB_REF
-          }
-      - uses: projectdiscovery/actions/goreleaser@v1

.github/workflows/compat-checks.yaml

@@ -0,0 +1,19 @@
+name: ♾️ Compatibility Checks
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+    branches:
+      - dev
+
+jobs:
+  check:
+    if: github.actor == 'dependabot[bot]'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: projectdiscovery/actions/setup/go/compat-checks@v1
+        with:
+          release-test: true

.github/workflows/dockerhub-push.yml

@@ -1,47 +0,0 @@
-name: 🐳 Docker Push
-
-on:
-  workflow_run:
-    workflows: ["🎉 Release Binary"]
-    types:
-      - completed
-  workflow_dispatch:
-
-jobs:
-  docker:
-    runs-on: ubuntu-latest-16-cores
-    steps:
-      - name: Git Checkout
-        uses: actions/checkout@v4
-
-      - name: Get GitHub tag
-        id: meta
-        run: |
-          curl --silent "https://api.github.com/repos/projectdiscovery/nuclei/releases/latest" | jq -r .tag_name | xargs -I {} echo TAG={} >> $GITHUB_OUTPUT
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v3 
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: projectdiscovery/nuclei:latest,projectdiscovery/nuclei:${{ steps.meta.outputs.TAG }}
-
-      - name: Update DockerHub Description
-        uses: peter-evans/dockerhub-description@v4
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-          repository: projectdiscovery/nuclei

.github/workflows/generate-pgo.yaml

@@ -19,7 +19,7 @@ jobs:
     strategy:
       matrix:
         targets: [150]
-    runs-on: ubuntu-latest-16-cores
+    runs-on: ubuntu-latest
     if: github.repository == 'projectdiscovery/nuclei'
     permissions:
       contents: write

.github/workflows/perf-regression.yaml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   perf-regression:
-    runs-on: ubuntu-latest-16-cores
+    runs-on: ubuntu-latest
     if: github.repository == 'projectdiscovery/nuclei'
     env:
       BENCH_OUT: "/tmp/bench.out"

.github/workflows/perf-test.yaml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         count: [50, 100, 150]
-    runs-on: ubuntu-latest-16-cores
+    runs-on: ubuntu-latest
     if: github.repository == 'projectdiscovery/nuclei'
     env:
       LIST_FILE: "/tmp/targets-${{ matrix.count }}.txt"

.github/workflows/release.yaml

@@ -14,6 +14,12 @@ jobs:
         with: 
           fetch-depth: 0
       - uses: projectdiscovery/actions/setup/go@v1
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3 
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
       - uses: projectdiscovery/actions/goreleaser@v1
         with: 
           release: true

.github/workflows/tests.yaml

@@ -132,6 +132,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: projectdiscovery/actions/setup/go@v1
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
       - uses: projectdiscovery/actions/goreleaser@v1
 
   flamegraph:

.goreleaser.yml

@@ -1,3 +1,5 @@
+version: 2
+
 before:
   hooks:
     - go mod download
@@ -44,6 +46,72 @@ archives:
 checksum:
   algorithm: sha256
 
+dockers:
+  - image_templates:
+      - "projectdiscovery/{{ .ProjectName }}:{{ .Tag }}-amd64"
+      - "projectdiscovery/{{ .ProjectName }}:v{{ .Major }}.{{ .Minor }}-amd64"
+      - "projectdiscovery/{{ .ProjectName }}:v{{ .Major }}-amd64"
+      - "projectdiscovery/{{ .ProjectName }}:latest-amd64"
+    dockerfile: Dockerfile.goreleaser
+    use: buildx
+    build_flag_templates:
+      - "--pull"
+      - "--platform=linux/amd64"
+      - "--label=org.opencontainers.image.created={{ .Date }}"
+      - "--label=org.opencontainers.image.ref.name={{ .Tag }}"
+      - "--label=org.opencontainers.image.revision={{ .FullCommit }}"
+      - "--label=org.opencontainers.image.version={{ .Version }}"
+    goarch: amd64
+  - image_templates:
+      - "projectdiscovery/{{ .ProjectName }}:{{ .Tag }}-arm64"
+      - "projectdiscovery/{{ .ProjectName }}:v{{ .Major }}.{{ .Minor }}-arm64"
+      - "projectdiscovery/{{ .ProjectName }}:v{{ .Major }}-arm64"
+      - "projectdiscovery/{{ .ProjectName }}:latest-arm64"
+    dockerfile: Dockerfile.goreleaser
+    use: buildx
+    build_flag_templates:
+      - "--pull"
+      - "--platform=linux/arm64"
+      - "--label=org.opencontainers.image.created={{ .Date }}"
+      - "--label=org.opencontainers.image.ref.name={{ .Tag }}"
+      - "--label=org.opencontainers.image.revision={{ .FullCommit }}"
+      - "--label=org.opencontainers.image.version={{ .Version }}"
+    goarch: arm64
+  # # NOTE(dwisiswant0): chromium doesn't support 32-bit on alpine
+  # - image_templates:
+  #     - "projectdiscovery/{{ .ProjectName }}:{{ .Tag }}-386"
+  #     - "projectdiscovery/{{ .ProjectName }}:v{{ .Major }}.{{ .Minor }}-386"
+  #     - "projectdiscovery/{{ .ProjectName }}:v{{ .Major }}-386"
+  #     - "projectdiscovery/{{ .ProjectName }}:latest-386"
+  #   dockerfile: Dockerfile.goreleaser
+  #   use: buildx
+  #   build_flag_templates:
+  #     - "--pull"
+  #     - "--platform=linux/386"
+  #     - "--label=org.opencontainers.image.created={{ .Date }}"
+  #     - "--label=org.opencontainers.image.ref.name={{ .Tag }}"
+  #     - "--label=org.opencontainers.image.revision={{ .FullCommit }}"
+  #     - "--label=org.opencontainers.image.version={{ .Version }}"
+  #   goarch: "386"
+
+docker_manifests:
+  - name_template: "projectdiscovery/{{ .ProjectName }}:{{ .Tag }}"
+    image_templates:
+      - "projectdiscovery/{{ .ProjectName }}:{{ .Tag }}-amd64"
+      - "projectdiscovery/{{ .ProjectName }}:{{ .Tag }}-arm64"
+  - name_template: "projectdiscovery/{{ .ProjectName }}:v{{ .Major }}.{{ .Minor }}"
+    image_templates:
+      - "projectdiscovery/{{ .ProjectName }}:v{{ .Major }}.{{ .Minor }}-amd64"
+      - "projectdiscovery/{{ .ProjectName }}:v{{ .Major }}.{{ .Minor }}-arm64"
+  - name_template: "projectdiscovery/{{ .ProjectName }}:v{{ .Major }}"
+    image_templates:
+      - "projectdiscovery/{{ .ProjectName }}:v{{ .Major }}-amd64"
+      - "projectdiscovery/{{ .ProjectName }}:v{{ .Major }}-arm64"
+  - name_template: "projectdiscovery/{{ .ProjectName }}:latest"
+    image_templates:
+      - "projectdiscovery/{{ .ProjectName }}:latest-amd64"
+      - "projectdiscovery/{{ .ProjectName }}:latest-arm64"
+
 announce:
   slack:
     enabled: true

Dockerfile

@@ -1,15 +1,16 @@
 # Build
-FROM golang:1.22-alpine AS build-env
+FROM golang:1.22-alpine AS builder
+
 RUN apk add build-base
 WORKDIR /app
 COPY . /app
-RUN go mod download
-RUN go build ./cmd/nuclei
+RUN make verify
+RUN make build
 
 # Release
-FROM alpine:3.18.6
-RUN apk upgrade --no-cache \
-    && apk add --no-cache bind-tools chromium ca-certificates
-COPY --from=build-env /app/nuclei /usr/local/bin/
+FROM alpine:latest
+
+RUN apk add --no-cache bind-tools chromium ca-certificates
+COPY --from=builder /app/bin/nuclei /usr/local/bin/
 
 ENTRYPOINT ["nuclei"]

Dockerfile.goreleaser

@@ -0,0 +1,12 @@
+FROM alpine:latest
+
+LABEL org.opencontainers.image.authors="ProjectDiscovery"
+LABEL org.opencontainers.image.description="Nuclei is a fast, customizable vulnerability scanner powered by the global security community and built on a simple YAML-based DSL, enabling collaboration to tackle trending vulnerabilities on the internet. It helps you find vulnerabilities in your applications, APIs, networks, DNS, and cloud configurations."
+LABEL org.opencontainers.image.licenses="MIT"
+LABEL org.opencontainers.image.title="nuclei"
+LABEL org.opencontainers.image.url="https://github.com/projectdiscovery/nuclei"
+
+RUN apk add --no-cache bind-tools chromium ca-certificates
+COPY nuclei /usr/local/bin/
+
+ENTRYPOINT ["nuclei"]

README.md

@@ -395,7 +395,7 @@ nuclei -target https://example.com
 Nuclei can handle bulk scanning by providing a list of targets. You can use a file containing multiple URLs.
 
 ```sh
-nuclei -targets urls.txt
+nuclei -list urls.txt
 ```
 
 ### Network scan

cmd/docgen/docgen.go

@@ -6,53 +6,58 @@ import (
 	"os"
 	"reflect"
 	"regexp"
-	"strings"
 
 	"github.com/invopop/jsonschema"
-
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
 )
 
 var pathRegex = regexp.MustCompile(`github\.com/projectdiscovery/nuclei/v3/(?:internal|pkg)/(?:.*/)?([A-Za-z.]+)`)
 
-func main() {
-	// Generate yaml syntax documentation
-	data, err := templates.GetTemplateDoc().Encode()
+func writeToFile(filename string, data []byte) {
+	file, err := os.Create(filename)
 	if err != nil {
-		log.Fatalf("Could not encode docs: %s\n", err)
+		log.Fatalf("Could not create file %s: %s\n", filename, err)
 	}
+	defer file.Close()
 
+	_, err = file.Write(data)
+	if err != nil {
+		log.Fatalf("Could not write to file %s: %s\n", filename, err)
+	}
+}
+
+func main() {
 	if len(os.Args) < 3 {
 		log.Fatalf("syntax: %s md-docs-file jsonschema-file\n", os.Args[0])
 	}
 
-	err = os.WriteFile(os.Args[1], data, 0644)
+	// Generate YAML documentation
+	data, err := templates.GetTemplateDoc().Encode()
 	if err != nil {
-		log.Fatalf("Could not write docs: %s\n", err)
+		log.Fatalf("Could not encode docs: %s\n", err)
+	}
+	writeToFile(os.Args[1], data)
+
+	// Generate JSON Schema
+	r := &jsonschema.Reflector{
+		Namer: func(t reflect.Type) string {
+			if t.Kind() == reflect.Slice {
+				return ""
+			}
+			return t.String()
+		},
 	}
 
-	// Generate jsonschema
-	r := &jsonschema.Reflector{}
-	r.Namer = func(r reflect.Type) string {
-		if r.Kind() == reflect.Slice {
-			return ""
-		}
-		return r.String()
-	}
 	jsonschemaData := r.Reflect(&templates.Template{})
 
 	var buf bytes.Buffer
 	encoder := json.NewEncoder(&buf)
 	encoder.SetIndent("", "  ")
-	_ = encoder.Encode(jsonschemaData)
+	if err := encoder.Encode(jsonschemaData); err != nil {
+		log.Fatalf("Could not encode JSON schema: %s\n", err)
+	}
 
-	schema := buf.String()
-	for _, match := range pathRegex.FindAllStringSubmatch(schema, -1) {
-		schema = strings.ReplaceAll(schema, match[0], match[1])
-	}
-	err = os.WriteFile(os.Args[2], []byte(schema), 0644)
-	if err != nil {
-		log.Fatalf("Could not write jsonschema: %s\n", err)
-	}
+	schema := pathRegex.ReplaceAllString(buf.String(), "$1")
+	writeToFile(os.Args[2], []byte(schema))
 }

cmd/nuclei/main.go

@@ -196,7 +196,7 @@ func main() {
 	go func() {
 		for range c {
 			gologger.Info().Msgf("CTRL+C pressed: Exiting\n")
-			if options.DASTServer {
+			if options.DASTServer || options.PassiveServer {
 				nucleiRunner.Close()
 				os.Exit(1)
 			}
@@ -375,6 +375,9 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVar(&fuzzFlag, "fuzz", false, "enable loading fuzzing templates (Deprecated: use -dast instead)"),
 		flagSet.BoolVar(&options.DAST, "dast", false, "enable / run dast (fuzz) nuclei templates"),
 		flagSet.BoolVarP(&options.DASTServer, "dast-server", "dts", false, "enable dast server mode (live fuzzing)"),
+		flagSet.BoolVarP(&options.PassiveServer, "passive-server", "psr", false, "enable passive server mode"),
+		flagSet.IntVarP(&options.ProxyServerPort, "proxy-server-port", "psp", 9053, "port to use for the proxy server"),
+		flagSet.StringVarP(&options.ProxyCacheDirectory, "proxy-cache-directory", "pcd", "", "directory to use for the proxy server cache"),
 		flagSet.BoolVarP(&options.DASTReport, "dast-report", "dtr", false, "write dast scan report to file"),
 		flagSet.StringVarP(&options.DASTServerToken, "dast-server-token", "dtst", "", "dast server token (optional)"),
 		flagSet.StringVarP(&options.DASTServerAddress, "dast-server-address", "dtsa", "localhost:9055", "dast server address"),

go.mod

@@ -20,12 +20,12 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.1.1
-	github.com/projectdiscovery/fastdialer v0.3.0
-	github.com/projectdiscovery/hmap v0.0.82
+	github.com/projectdiscovery/fastdialer v0.4.0
+	github.com/projectdiscovery/hmap v0.0.85
 	github.com/projectdiscovery/interactsh v1.2.4
-	github.com/projectdiscovery/rawhttp v0.1.89
-	github.com/projectdiscovery/retryabledns v1.0.94
-	github.com/projectdiscovery/retryablehttp-go v1.0.101
+	github.com/projectdiscovery/rawhttp v0.1.90
+	github.com/projectdiscovery/retryabledns v1.0.96
+	github.com/projectdiscovery/retryablehttp-go v1.0.102
 	github.com/projectdiscovery/yamldoc-go v1.0.6
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -84,24 +84,26 @@ require (
 	github.com/microsoft/go-mssqldb v1.6.0
 	github.com/ory/dockertest/v3 v3.10.0
 	github.com/praetorian-inc/fingerprintx v1.1.9
-	github.com/projectdiscovery/dsl v0.3.18
+	github.com/projectdiscovery/dsl v0.3.21
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
 	github.com/projectdiscovery/goflags v0.1.74
-	github.com/projectdiscovery/gologger v1.1.46
+	github.com/projectdiscovery/gologger v1.1.49
 	github.com/projectdiscovery/gostruct v0.0.2
 	github.com/projectdiscovery/gozero v0.0.3
 	github.com/projectdiscovery/httpx v1.6.10
 	github.com/projectdiscovery/mapcidr v1.1.34
+	github.com/projectdiscovery/martian/v3 v3.0.0-20240219194442-fed3b744f477
 	github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
-	github.com/projectdiscovery/ratelimit v0.0.75
+	github.com/projectdiscovery/proxify v0.0.15
+	github.com/projectdiscovery/ratelimit v0.0.77
 	github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.9
 	github.com/projectdiscovery/uncover v1.0.10
-	github.com/projectdiscovery/useragent v0.0.94
-	github.com/projectdiscovery/utils v0.4.12
-	github.com/projectdiscovery/wappalyzergo v0.2.18
+	github.com/projectdiscovery/useragent v0.0.97
+	github.com/projectdiscovery/utils v0.4.15
+	github.com/projectdiscovery/wappalyzergo v0.2.21
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
 	github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466
@@ -125,9 +127,11 @@ require (
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
 	github.com/STARRY-S/zip v0.2.1 // indirect
+	github.com/Shopify/sarama v1.38.1 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/alecthomas/chroma/v2 v2.14.0 // indirect
 	github.com/andybalholm/brotli v1.1.1 // indirect
+	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
@@ -158,6 +162,10 @@ require (
 	github.com/docker/cli v24.0.5+incompatible // indirect
 	github.com/docker/docker v24.0.9+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/eapache/go-resiliency v1.3.0 // indirect
+	github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
+	github.com/eapache/queue v1.1.0 // indirect
+	github.com/elastic/go-elasticsearch/v7 v7.17.10 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/felixge/fgprof v0.9.5 // indirect
 	github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
@@ -171,6 +179,7 @@ require (
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
+	github.com/goccy/go-yaml v1.11.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
@@ -184,6 +193,7 @@ require (
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/haxii/fastproxy v0.5.37 // indirect
 	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
@@ -219,10 +229,13 @@ require (
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/projectdiscovery/asnmap v1.1.1 // indirect
-	github.com/projectdiscovery/cdncheck v1.1.8 // indirect
+	github.com/projectdiscovery/cdncheck v1.1.11 // indirect
 	github.com/projectdiscovery/freeport v0.0.7 // indirect
 	github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
 	github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
+	github.com/projectdiscovery/roundrobin v0.0.6 // indirect
+	github.com/projectdiscovery/tinydns v0.0.31 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/refraction-networking/utls v1.6.7 // indirect
 	github.com/sashabaranov/go-openai v1.37.0 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
@@ -260,6 +273,7 @@ require (
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
 	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/sync v0.11.0 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
 	mellium.im/sasl v0.3.1 // indirect
 )
@@ -309,7 +323,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/projectdiscovery/blackrock v0.0.1 // indirect
-	github.com/projectdiscovery/networkpolicy v0.1.7
+	github.com/projectdiscovery/networkpolicy v0.1.9
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect

go.sum

@@ -88,7 +88,12 @@ github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mo
 github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg=
 github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
+github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A=
+github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g=
+github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc=
+github.com/Shopify/toxiproxy/v2 v2.5.0/go.mod h1:yhM2epWtAmel9CB8r2+L+PCmhH6yH2pITaPAo7jxJl0=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
@@ -327,9 +332,16 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
+github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0=
+github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 h1:8yY/I9ndfrgrXUbOGObLHKBR4Fl3nZXwM2c7OYTT8hM=
+github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
+github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
+github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elazarl/goproxy v1.2.1 h1:njjgvO6cRG9rIqN2ebkqy6cQz2Njkx7Fsfv/zIZqgug=
 github.com/elazarl/goproxy v1.2.1/go.mod h1:YfEbZtqP4AetfO6d40vWchF3znWX7C7Vd6ZMfdL8z64=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
@@ -438,6 +450,8 @@ github.com/gobwas/ws v1.2.1 h1:F2aeBZrm2NDsc7vbovKrWSogd4wvfAxg0FQ89/iqOTk=
 github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
 github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
 github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I=
+github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
 github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -487,6 +501,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
@@ -592,6 +608,9 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/haxii/fastproxy v0.5.37 h1:grfso8V9sNO8jZjI/vkizFXzz8fE/yrsgl+XxTb3fio=
+github.com/haxii/fastproxy v0.5.37/go.mod h1:VFy3M4EmTbeKu+IccQ6UiJ1W4sPeQ75GV+1JDa8h864=
+github.com/haxii/log v1.0.0/go.mod h1:y9MlOm+u2ny65yQxScWfSGZFOhRVLXz2vJlkiIx2jfI=
 github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk=
 github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA=
 github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A=
@@ -868,14 +887,14 @@ github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kI
 github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60=
 github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ=
 github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
-github.com/projectdiscovery/cdncheck v1.1.8 h1:kOdgKm2QTm0s76jSEHEAQISc5Gb20goJbVLpHqqza+8=
-github.com/projectdiscovery/cdncheck v1.1.8/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U=
+github.com/projectdiscovery/cdncheck v1.1.11 h1:r7hkw0BAfD/YsnKq1fVgZJDfMHhqVmDQroerjlST6oQ=
+github.com/projectdiscovery/cdncheck v1.1.11/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U=
 github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE=
 github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0=
-github.com/projectdiscovery/dsl v0.3.18 h1:UsFiVdqjIm3S1KNQhGUWwgQgCp5N9O39kZmjfazXVQY=
-github.com/projectdiscovery/dsl v0.3.18/go.mod h1:RYA1/QE/L+hFO3haGFc9RU2DQtLoJDszBXYWbD/8T9U=
-github.com/projectdiscovery/fastdialer v0.3.0 h1:/wMptjdsrAU/wiaA/U3lSgYGaYCGJH6xm0mLei6oMxk=
-github.com/projectdiscovery/fastdialer v0.3.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ=
+github.com/projectdiscovery/dsl v0.3.21 h1:z7sI4GSI3kz8CPXeRrnydqZmpR9oKvp6ejEyUk6r7D4=
+github.com/projectdiscovery/dsl v0.3.21/go.mod h1:6+X/o2r3P9x3AK8Ki5cRiOjoPDBXawXwg/QcI9E1i8w=
+github.com/projectdiscovery/fastdialer v0.4.0 h1:licZKyq+Shd5lLDb8uPd60Jp43K4NFE8cr67XD2eg7w=
+github.com/projectdiscovery/fastdialer v0.4.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk=
@@ -884,14 +903,14 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
 github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c=
 github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4=
-github.com/projectdiscovery/gologger v1.1.46 h1:dBbtVJM3Y0rBBukvk2vLmConuMEI3KDj8k9DhECj/bU=
-github.com/projectdiscovery/gologger v1.1.46/go.mod h1:9ZPkQd79jpuy3M4ZENmXx78hobqvumkIgZ03tn4hD/M=
+github.com/projectdiscovery/gologger v1.1.49 h1:1IHvVhRN6evQpxp35e55Vmcq/QjocN3bYEyxY5n5OQA=
+github.com/projectdiscovery/gologger v1.1.49/go.mod h1:askNkcvSKXaRibPvIPuFDJl8qa3yK/EtBgFnCt3Dni0=
 github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
 github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
 github.com/projectdiscovery/gozero v0.0.3 h1:tsYkrSvWw4WdIUJyisd4MB1vRiw1X57TuVVk3p8Z3G8=
 github.com/projectdiscovery/gozero v0.0.3/go.mod h1:MpJ37Dsh94gy2EKqaemdeh+CzduGVB2SDfhr6Upsjew=
-github.com/projectdiscovery/hmap v0.0.82 h1:gPFkeD5sbY3uVIxgtgDkzHmXCo/H7cbhRhdLX//+G1A=
-github.com/projectdiscovery/hmap v0.0.82/go.mod h1:8vSfVgZwCRbHitnZPgpsOw8115/mJX/b4B8QKR88Iak=
+github.com/projectdiscovery/hmap v0.0.85 h1:WXP/gOSzFm8mfMAJpRN1oQ3lYbBHVGvak+nSY5IgRh4=
+github.com/projectdiscovery/hmap v0.0.85/go.mod h1:TdBM3U8x25uBBnsD9OJtAKKoRVPJXubU9L9/1HdTg/c=
 github.com/projectdiscovery/httpx v1.6.10 h1:R08LiWDQKJNmVHcR5TdIGQil1O3ShiFonK6Wf+gzg0k=
 github.com/projectdiscovery/httpx v1.6.10/go.mod h1:drsRPt18IQB/Gfj4D0D9woS3hkY+m6k4HgL09JXX4vg=
 github.com/projectdiscovery/interactsh v1.2.4 h1:WUSj+fxbcV53J64oIAhbYzCKD1w/IyenyRBhkI5jiqI=
@@ -902,34 +921,42 @@ github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScL
 github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
 github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM=
 github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ=
+github.com/projectdiscovery/martian/v3 v3.0.0-20240219194442-fed3b744f477 h1:VJaBELAC5Hw+kc+ylrFF5nSf7Wasnb9mZxMlF/kR3gg=
+github.com/projectdiscovery/martian/v3 v3.0.0-20240219194442-fed3b744f477/go.mod h1:wPvVUl2C/XOFacugXwsUp65GN0upmKfwKMyimA/AAaM=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
-github.com/projectdiscovery/networkpolicy v0.1.7 h1:SIFsv5f8JYDwCUH/AUTeB3dpXthx0AUz2rej64rNWwY=
-github.com/projectdiscovery/networkpolicy v0.1.7/go.mod h1:grubaCHOEgT6a2r/p8DlfOwovEL4N8Fd3yye7/ACwug=
-github.com/projectdiscovery/ratelimit v0.0.75 h1:CcfER34mqU5NjAMI5JN+Sl7slN++m3SKeb6+Ed0KI/c=
-github.com/projectdiscovery/ratelimit v0.0.75/go.mod h1:Gm/NA+pYJMTPMLSurQX95mXb+RDBfLAEONVqzcCW1GU=
-github.com/projectdiscovery/rawhttp v0.1.89 h1:aIkCd0BYlQ11UoPaMP6U5Ujvycch21CD58U1AGxxz6E=
-github.com/projectdiscovery/rawhttp v0.1.89/go.mod h1:q90gCzooR1Xtv8671ipav3tjmXrJx/0aG3HtLNCP6xI=
+github.com/projectdiscovery/networkpolicy v0.1.9 h1:MMFN3zAutxcFmmxuXUcVl1BZJOacUas8NMigdq6Da7U=
+github.com/projectdiscovery/networkpolicy v0.1.9/go.mod h1:tWz6ZlkwpdJ5MML/yE/WE4Q+43nAsPEf9/ffCuWtZo4=
+github.com/projectdiscovery/proxify v0.0.15 h1:kyTedG/pp0YdPMoQl/eaQM1+wzC2Uj8E+RkYudLbeGo=
+github.com/projectdiscovery/proxify v0.0.15/go.mod h1:PIeZTsWGnwdBC9Ppck9bNNqoHX91wcvYHYEcD+2hYRE=
+github.com/projectdiscovery/ratelimit v0.0.77 h1:hdZuXQRENzN7gsZgHPZdMOSpMXMzH7WYDLgvW/S5Nbk=
+github.com/projectdiscovery/ratelimit v0.0.77/go.mod h1:0JGCJGpwtmrroTurx/4XIpBFKdeIObWc7ZRVwpYeQcI=
+github.com/projectdiscovery/rawhttp v0.1.90 h1:LOSZ6PUH08tnKmWsIwvwv1Z/4zkiYKYOSZ6n+8RFKtw=
+github.com/projectdiscovery/rawhttp v0.1.90/go.mod h1:VZYAM25UI/wVB3URZ95ZaftgOnsbphxyAw/XnQRRz4Y=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.94 h1:MvxtRcmvxhxikxT7p/E40hcYRWRiL5fg/JQ8bpBaz+0=
-github.com/projectdiscovery/retryabledns v1.0.94/go.mod h1:croGTyMM4yNlrSWA/X7xNe3c0c7mDmCdbm8goLd8Bak=
-github.com/projectdiscovery/retryablehttp-go v1.0.101 h1:xmoXGVQ7DD/5YvDvtaOExbbF6aXlr5ARjssXgMdtkmY=
-github.com/projectdiscovery/retryablehttp-go v1.0.101/go.mod h1:d+xU7CAHiOL/v+QQIHT4AXbEjTO7o0B5naQQOC0JDhw=
+github.com/projectdiscovery/retryabledns v1.0.96 h1:NeXFOIgLZXstl+v3JNEfU0z0WLvQJTvz4YUFupFruDQ=
+github.com/projectdiscovery/retryabledns v1.0.96/go.mod h1:UCM+4AWmncf1xO/Ku8oYntWLiuaQTwD7TFCfaDKp4ig=
+github.com/projectdiscovery/retryablehttp-go v1.0.102 h1:OgVbzGzFq1a82D/kVoooPSgRHU17RDO/fVWUTfMyDvE=
+github.com/projectdiscovery/retryablehttp-go v1.0.102/go.mod h1:hDyw8H3GOeA0XjpdTdS4JF7qflojp6PAZ9t6vCFFbCU=
+github.com/projectdiscovery/roundrobin v0.0.6 h1:zoJAFRgP9XK7B+iKSjR+djRAuDYxnc57+Fx+qpoPvds=
+github.com/projectdiscovery/roundrobin v0.0.6/go.mod h1:vTxcWqNLyMH6VE2Q/hsNNvDHFLiIzHozC1rLLT/vocQ=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
 github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
 github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
+github.com/projectdiscovery/tinydns v0.0.31 h1:iQL1ze2/4PHWH+Nzsxg35kVIWkvEIatAZCDcP0c1/v4=
+github.com/projectdiscovery/tinydns v0.0.31/go.mod h1:6rj/dEzXZPn+nLl4O1wUMJdfkdUQ9eCV7NsDWPakYv8=
 github.com/projectdiscovery/tlsx v1.1.9 h1:DhErhHCO2+toF5DEX7qe3pkwrIrYlex3F/mzHEUiZHg=
 github.com/projectdiscovery/tlsx v1.1.9/go.mod h1:Jy+r38WyYjapQWaffiKGdYm4Ksdrt8BWtsqA2rUospU=
 github.com/projectdiscovery/uncover v1.0.10 h1:FdnBYgynGUtjIsW5WPIIhadR1Smcghik9cZSMEtILN4=
 github.com/projectdiscovery/uncover v1.0.10/go.mod h1:l7QQ+mBc7bLK4tqYqPyo9nrYdz1K8vaGZWKYihkHmAs=
-github.com/projectdiscovery/useragent v0.0.94 h1:6/JtkkGZveyJ0+IjvA01Py+X2KZGOemQn6yQY1by3AU=
-github.com/projectdiscovery/useragent v0.0.94/go.mod h1:QL4NRHuw0c46XCzstUaHY2pktkcgUsoyMtpyi3a/79A=
-github.com/projectdiscovery/utils v0.4.12 h1:3HE+4Go4iTwipeN2B+tC7xl7KS4BgXgp0BZaQXE2bjM=
-github.com/projectdiscovery/utils v0.4.12/go.mod h1:EDUNBDGTO+Tfl6YQj3ADg97iYp2h8IbCmpP24LMW3+E=
-github.com/projectdiscovery/wappalyzergo v0.2.18 h1:oYI7iOu3m9j58jSbnefop8s1VhkNbH7Z35P3qwQxDtA=
-github.com/projectdiscovery/wappalyzergo v0.2.18/go.mod h1:F8X79ljvmvrG+EIxdxWS9VbdkVTsQupHYz+kXlp8O0o=
+github.com/projectdiscovery/useragent v0.0.97 h1:FoDmkWkLfPZ6Drg1BQEwj5oWUZHQXUO6IerZNzHGAlM=
+github.com/projectdiscovery/useragent v0.0.97/go.mod h1:M7Vc7hbuCuaAS9y0R6XQN+buaA6g2Z9oZMgVSDisn+M=
+github.com/projectdiscovery/utils v0.4.15 h1:6Bh+Bvqc7352rUzQtRtQqNGaVIXv6ZJ4yvE++Jrk58E=
+github.com/projectdiscovery/utils v0.4.15/go.mod h1:y5gnpQn802iEWqf0djTRNskJlS62P5eqe1VS1+ah0tk=
+github.com/projectdiscovery/wappalyzergo v0.2.21 h1:7dcoAM5dM6QSddv5VEnT1ySw3CBlhJpuJVYuwJa0drg=
+github.com/projectdiscovery/wappalyzergo v0.2.21/go.mod h1:F8X79ljvmvrG+EIxdxWS9VbdkVTsQupHYz+kXlp8O0o=
 github.com/projectdiscovery/yamldoc-go v1.0.6 h1:GCEdIRlQjDux28xTXKszM7n3jlMf152d5nqVpVoetas=
 github.com/projectdiscovery/yamldoc-go v1.0.6/go.mod h1:R5lWrNzP+7Oyn77NDVPnBsxx2/FyQZBBkIAaSaCQFxw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -965,6 +992,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
+github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY=
 github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c=
 github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM=
@@ -982,6 +1011,7 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.11.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -1536,6 +1566,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
@@ -1591,6 +1623,8 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20220706185917-7780775163c4 h1:7YDGQC/0sigNGzsEWyb9s72jTxlFdwVEYNJHbfQ+Dtg=
+google.golang.org/genproto v0.0.0-20220706185917-7780775163c4/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
@@ -1608,6 +1642,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w=
+google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

internal/runner/options.go

@@ -235,15 +235,25 @@ func validateMissingS3Options(options *types.Options) []string {
 	if options.AwsBucketName == "" {
 		missing = append(missing, "AWS_TEMPLATE_BUCKET")
 	}
-	if options.AwsAccessKey == "" {
-		missing = append(missing, "AWS_ACCESS_KEY")
-	}
-	if options.AwsSecretKey == "" {
-		missing = append(missing, "AWS_SECRET_KEY")
-	}
-	if options.AwsRegion == "" {
-		missing = append(missing, "AWS_REGION")
+	if options.AwsProfile == "" {
+		var missingCreds []string
+		if options.AwsAccessKey == "" {
+			missingCreds = append(missingCreds, "AWS_ACCESS_KEY")
+		}
+		if options.AwsSecretKey == "" {
+			missingCreds = append(missingCreds, "AWS_SECRET_KEY")
+		}
+		if options.AwsRegion == "" {
+			missingCreds = append(missingCreds, "AWS_REGION")
+		}
+
+		missing = append(missing, missingCreds...)
+
+		if len(missingCreds) > 0 {
+			missing = append(missing, "AWS_PROFILE")
+		}
 	}
+
 	return missing
 }
 
@@ -449,6 +459,7 @@ func readEnvInputVars(options *types.Options) {
 	options.AwsSecretKey = os.Getenv("AWS_SECRET_KEY")
 	options.AwsBucketName = os.Getenv("AWS_TEMPLATE_BUCKET")
 	options.AwsRegion = os.Getenv("AWS_REGION")
+	options.AwsProfile = os.Getenv("AWS_PROFILE")
 
 	// Azure options for downloading templates from an Azure Blob Storage container
 	options.AzureContainerName = os.Getenv("AZURE_CONTAINER_NAME")

internal/runner/runner.go

@@ -476,7 +476,7 @@ func (r *Runner) setupPDCPUpload(writer output.Writer) output.Writer {
 func (r *Runner) RunEnumeration() error {
 	// If the user has asked for DAST server mode, run the live
 	// DAST fuzzing server.
-	if r.options.DASTServer {
+	if r.options.DASTServer || r.options.PassiveServer {
 		execurOpts := &server.NucleiExecutorOptions{
 			Options:            r.options,
 			Output:             r.output,
@@ -501,6 +501,8 @@ func (r *Runner) RunEnumeration() error {
 			InScope:               r.options.Scope,
 			OutScope:              r.options.OutOfScope,
 			NucleiExecutorOptions: execurOpts,
+			ProxyServerPort:       r.options.ProxyServerPort,
+			ProxyCacheDirectory:   r.options.ProxyCacheDirectory,
 		})
 		if err != nil {
 			return err

internal/server/nuclei_sdk.go

@@ -159,7 +159,7 @@ func (n *nucleiExecutor) ExecuteScan(target PostRequestsHandlerRequest) error {
 			Body   string            `json:"body"`
 			Raw    string            `json:"raw"`
 		}{
-			Raw: target.RawHTTP,
+			Raw: target.RawRequest,
 		},
 	}
 
@@ -195,5 +195,4 @@ func (n *nucleiExecutor) Close() {
 	if n.executorOpts.InputHelper != nil {
 		_ = n.executorOpts.InputHelper.Close()
 	}
-
 }

internal/server/passive_nuclei.go

@@ -0,0 +1,180 @@
+package server
+
+import (
+	"context"
+	"log/slog"
+	"net/url"
+	"strings"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/offlinehttp"
+	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
+	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+)
+
+type PassiveNucleiExecutor struct {
+	defaultTemplates   []*templates.Template
+	pathBasedTemplates map[string][]*templates.Template
+}
+
+func NewPassiveNucleiExecutor(templatesDir string) (*PassiveNucleiExecutor, error) {
+	catalog := disk.NewCatalog(templatesDir)
+
+	opts := types.DefaultOptions()
+	opts.ExcludeTags = []string{"cve", "lfi", "xss", "rce", "ssrf", "cmdi", "oast", "ssti", "sqli"}
+	opts.ExcludedTemplates = []string{
+		"http/vulnerabilities/",
+		"http/cves",
+		"http/cnvd",
+		"http/fuzzing",
+	}
+	opts.ExcludeIds = []string{
+		"tech-detect", "fingerprinthub-web-fingerprints", "credentials-disclosure",
+		"aws-detect",
+	}
+	opts.Protocols = templateTypes.ProtocolTypes{templateTypes.HTTPProtocol}
+
+	executorOpts := protocols.ExecutorOptions{
+		Options: opts,
+		Catalog: catalog,
+		Parser:  templates.NewParser(),
+	}
+	if err := protocolinit.Init(opts); err != nil {
+		return nil, err
+	}
+
+	store, err := loader.New(loader.NewConfig(opts, catalog, executorOpts))
+	if err != nil {
+		return nil, err
+	}
+	store.Load()
+
+	loadedTemplates := store.Templates()
+	offlinehttp.RawInputMode = true
+
+	var defaultTemplates []*templates.Template
+	pathBasedTemplates := make(map[string][]*templates.Template)
+
+	for _, tpl := range loadedTemplates {
+		if len(tpl.RequestsWithHTTP) != 1 {
+			continue
+		}
+		if tpl.SelfContained || tpl.Flow != "" {
+			continue
+		}
+		for _, item := range tpl.RequestsWithHTTP {
+			if len(item.Path) == 0 {
+				continue
+			}
+			if err := compileOfflineHTTPRequest(tpl, &executorOpts); err != nil {
+				slog.Warn("Error compiling template", slog.String("id", tpl.ID), slog.String("error", err.Error()))
+				continue
+			}
+			for _, path := range item.Path {
+				// Default templates are not path based
+				if path == "{{BaseURL}}" || path == "{{BaseURL}}/" {
+					defaultTemplates = append(defaultTemplates, tpl)
+					continue
+				}
+				pathCleaned := strings.TrimPrefix(path, "{{BaseURL}}")
+				pathBasedTemplates[pathCleaned] = append(pathBasedTemplates[pathCleaned], tpl)
+			}
+		}
+	}
+	return &PassiveNucleiExecutor{
+		defaultTemplates:   defaultTemplates,
+		pathBasedTemplates: pathBasedTemplates,
+	}, nil
+}
+
+func (e *PassiveNucleiExecutor) Execute(ctx context.Context, input, URL string) ([]*output.ResultEvent, error) {
+	parsedURL, err := url.Parse(URL)
+	if err != nil {
+		return nil, err
+	}
+
+	var allResults []*output.ResultEvent
+	for _, tpl := range e.defaultTemplates {
+		results, err := executeTemplate(ctx, tpl, input, parsedURL)
+		if err != nil {
+			gologger.Warning().Msgf("Error executing template %s: %s\n", tpl.ID, err)
+			return nil, err
+		}
+		allResults = append(allResults, results...)
+	}
+
+	templates, ok := e.pathBasedTemplates[parsedURL.Path]
+	if !ok || len(templates) == 0 {
+		return allResults, nil
+	}
+
+	for _, tpl := range templates {
+		results, err := executeTemplate(ctx, tpl, input, parsedURL)
+		if err != nil {
+			gologger.Warning().Msgf("Error executing template %s: %s\n", tpl.ID, err)
+			continue
+		}
+		allResults = append(allResults, results...)
+	}
+	return allResults, nil
+}
+
+func executeTemplate(ctx context.Context, tpl *templates.Template, input string, parsedURL *url.URL) ([]*output.ResultEvent, error) {
+	results, err := tpl.Executer.ExecuteWithResults(scan.NewScanContext(ctx, contextargs.NewWithInput(ctx, input)))
+	if err != nil {
+		return nil, err
+	}
+	port := parsedURL.Port()
+	if port == "" {
+		if parsedURL.Scheme == "https" {
+			port = "443"
+		} else {
+			port = "80"
+		}
+	}
+	urlString := parsedURL.String()
+	for _, result := range results {
+		result.Info = tpl.Info
+		result.TemplateID = tpl.ID
+		result.TemplatePath = tpl.Path
+		result.URL = urlString
+		result.Host = parsedURL.Host
+		result.Path = parsedURL.Path
+		result.Scheme = parsedURL.Scheme
+		result.Path = parsedURL.Path
+		result.Port = port
+		result.Matched = urlString
+	}
+	return results, nil
+}
+
+// compileOfflineHTTPRequest iterates all requests if offline http mode is
+// specified and collects all matchers for all the base request templates
+// (those with URL {{BaseURL}} and it's slash variation.)
+func compileOfflineHTTPRequest(template *templates.Template, options *protocols.ExecutorOptions) error {
+	operatorsList := []*operators.Operators{}
+
+	for _, req := range template.RequestsHTTP {
+		operatorsList = append(operatorsList, &req.Operators)
+	}
+	if len(operatorsList) > 0 {
+		options.Operators = operatorsList
+		var err error
+
+		request := &offlinehttp.Request{}
+		_ = request.Compile(options)
+		template.Executer, err = tmplexec.NewTemplateExecuter([]protocols.Request{request}, options)
+		return err
+	}
+	return nil
+}

internal/server/proxy.go

@@ -0,0 +1,183 @@
+package server
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"math"
+	"net/http"
+	"net/http/httputil"
+	"os"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/martian/v3"
+	martianlog "github.com/projectdiscovery/martian/v3/log"
+	"github.com/projectdiscovery/proxify"
+	"github.com/projectdiscovery/proxify/pkg/certs"
+	"github.com/projectdiscovery/proxify/pkg/logger/elastic"
+	"github.com/projectdiscovery/proxify/pkg/logger/kafka"
+	"github.com/projectdiscovery/proxify/pkg/types"
+	httputils "github.com/projectdiscovery/utils/http"
+)
+
+// ProxyServer is an intercepting proxy launched through nuclei
+// using proxify for logging requests and responses.
+type ProxyServer struct {
+	ListenAddr string
+
+	proxy *proxify.Proxy
+}
+
+// RequestResponsePair is a pair of request and response
+type RequestResponsePair struct {
+	URL      string `json:"url" yaml:"url"`
+	Request  string `json:"request" yaml:"request"`
+	Response string `json:"response" yaml:"response"`
+	Protocol string `json:"protocol" yaml:"protocol"`
+}
+
+func responsePairFromResp(resp *http.Response) RequestResponsePair {
+	if resp.Request.Method == http.MethodConnect {
+		return RequestResponsePair{}
+	}
+
+	save := resp.Body
+	savecl := resp.ContentLength
+
+	var err error
+	if resp.Body != nil {
+		save, resp.Body, err = drainBody(resp.Body)
+		if err != nil {
+			return RequestResponsePair{}
+		}
+	}
+	chain := httputils.NewResponseChain(resp, -1)
+	defer chain.Close()
+
+	if err := chain.Fill(); err != nil {
+		log.Printf("[error] [proxy] Could not fill response chain: %s\n", err)
+		return RequestResponsePair{}
+	}
+	resp.Body = save
+	resp.ContentLength = savecl
+
+	if !chain.Has() {
+		return RequestResponsePair{}
+	}
+	respDump := chain.FullResponse()
+
+	return RequestResponsePair{
+		Response: respDump.String(),
+	}
+}
+
+type ProxyOptions struct {
+	Port           int
+	CacheDirectory string
+
+	OnIntercepted func(pair RequestResponsePair)
+}
+
+// NewProxyServer creates a new proxy server instance
+func NewProxyServer(opts *ProxyOptions) (*ProxyServer, error) {
+	ps := &ProxyServer{}
+
+	_ = os.MkdirAll(opts.CacheDirectory, 0755)
+	onRequestFunc := func(req *http.Request, ctx *martian.Context) error {
+		dumped, err := httputil.DumpRequestOut(req, true)
+		if err != nil {
+			return nil
+		}
+		ctx.Set(ctx.ID(), RequestResponsePair{
+			Request: string(dumped),
+			URL:     req.URL.String(),
+		})
+		ctx.Set("user-data", types.UserData{})
+		return nil
+	}
+
+	onResponseFunc := func(resp *http.Response, ctx *martian.Context) error {
+		pair := responsePairFromResp(resp)
+		if pair.Response == "" {
+			return nil
+		}
+		req, ok := ctx.Get(ctx.ID())
+		if !ok {
+			log.Printf("[error] [proxy] Could not get request from context\n")
+			return nil
+		}
+		valid, ok := req.(RequestResponsePair)
+		if !ok {
+			log.Printf("[error] [proxy] Could not validate request from context\n")
+			return nil
+		}
+		pair.Request = valid.Request
+		pair.Protocol = "http"
+		pair.URL = valid.URL
+		opts.OnIntercepted(pair)
+		return nil
+	}
+
+	err := certs.LoadCerts(opts.CacheDirectory)
+	if err != nil {
+		return nil, err
+	}
+
+	ps.ListenAddr = fmt.Sprintf("127.0.0.1:%d", opts.Port)
+
+	martianlog.SetLogger(&noopMartianLogger{})
+	proxifyOpts := &proxify.Options{
+		OutputJsonl:                 true,
+		MaxSize:                     math.MaxInt,
+		Verbosity:                   types.VerbosityDefault,
+		CertCacheSize:               256,
+		Directory:                   opts.CacheDirectory,
+		ListenAddrHTTP:              ps.ListenAddr,
+		OnRequestCallback:           onRequestFunc,
+		OnResponseCallback:          onResponseFunc,
+		UpstreamProxyRequestsNumber: 1,
+		Elastic:                     &elastic.Options{},
+		Kafka:                       &kafka.Options{},
+	}
+	proxy, err := proxify.NewProxy(proxifyOpts)
+	if err != nil {
+		return nil, err
+	}
+	ps.proxy = proxy
+
+	gologger.Info().Msgf("Starting proxy server on %s", ps.ListenAddr)
+	go func() {
+		err = proxy.Run()
+		if err != nil {
+			log.Printf("[error] [proxy] Could not run proxy: %s\n", err)
+		}
+	}()
+	return ps, nil
+}
+
+type noopMartianLogger struct{}
+
+func (l *noopMartianLogger) Infof(format string, args ...interface{})  {}
+func (l *noopMartianLogger) Debugf(format string, args ...interface{}) {}
+func (l *noopMartianLogger) Errorf(format string, args ...interface{}) {}
+
+func (p *ProxyServer) Close() {
+	p.proxy.Stop()
+}
+
+// from net/http/httputil.DumpResponse
+func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) {
+	if b == nil || b == http.NoBody {
+		// No copying needed. Preserve the magic sentinel meaning of NoBody.
+		return http.NoBody, http.NoBody, nil
+	}
+	var buf bytes.Buffer
+	if _, err = buf.ReadFrom(b); err != nil {
+		return nil, b, err
+	}
+	if err = b.Close(); err != nil {
+		return nil, b, err
+	}
+	return io.NopCloser(&buf), io.NopCloser(bytes.NewReader(buf.Bytes())), nil
+}

internal/server/requests_worker.go

@@ -1,7 +1,10 @@
 package server
 
 import (
+	"context"
+	"net/url"
 	"path"
+	"strings"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/internal/server/scope"
@@ -11,48 +14,72 @@ import (
 func (s *DASTServer) consumeTaskRequest(req PostRequestsHandlerRequest) {
 	defer s.endpointsInQueue.Add(-1)
 
-	parsedReq, err := types.ParseRawRequestWithURL(req.RawHTTP, req.URL)
+	parsedURL, err := url.Parse(req.URL)
 	if err != nil {
-		gologger.Warning().Msgf("Could not parse raw request: %s\n", err)
+		gologger.Warning().Msgf("Could not parse url: %s\n", err)
 		return
 	}
 
-	if parsedReq.URL.Scheme != "http" && parsedReq.URL.Scheme != "https" {
-		gologger.Warning().Msgf("Invalid scheme: %s\n", parsedReq.URL.Scheme)
+	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
+		gologger.Warning().Msgf("Invalid scheme: %s\n", parsedURL.Scheme)
 		return
 	}
 
 	// Check filenames and don't allow non-interesting files
-	extension := path.Base(parsedReq.URL.Path)
+	extension := path.Base(parsedURL.Path)
 	if extension != "/" && extension != "" && scope.IsUninterestingPath(extension) {
-		gologger.Warning().Msgf("Uninteresting path: %s\n", parsedReq.URL.Path)
+		gologger.Warning().Msgf("Uninteresting path: %s\n", parsedURL.Path)
 		return
 	}
 
-	inScope, err := s.scopeManager.Validate(parsedReq.URL.URL)
+	inScope, err := s.scopeManager.Validate(parsedURL)
 	if err != nil {
 		gologger.Warning().Msgf("Could not validate scope: %s\n", err)
 		return
 	}
 	if !inScope {
-		gologger.Warning().Msgf("Request is out of scope: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String())
+		gologger.Warning().Msgf("Request is out of scope: %s\n", parsedURL.String())
 		return
 	}
 
-	if s.deduplicator.isDuplicate(parsedReq) {
-		gologger.Warning().Msgf("Duplicate request detected: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String())
-		return
-	}
-
-	gologger.Verbose().Msgf("Fuzzing request: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String())
+	gologger.Verbose().Msgf("Fuzzing request: %s\n", parsedURL.String())
 
 	s.endpointsBeingTested.Add(1)
 	defer s.endpointsBeingTested.Add(-1)
 
 	// Fuzz the request finally
-	err = s.nucleiExecutor.ExecuteScan(req)
-	if err != nil {
-		gologger.Warning().Msgf("Could not run nuclei: %s\n", err)
-		return
+	if s.nucleiExecutor != nil && req.RawRequest != "" {
+		parsedReq, err := types.ParseRawRequestWithURL(req.RawRequest, req.URL)
+		if err != nil {
+			gologger.Warning().Msgf("Could not parse raw request: %s\n", err)
+			return
+		}
+
+		if s.deduplicator.isDuplicate(parsedReq) {
+			gologger.Warning().Msgf("Duplicate request detected: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String())
+			return
+		}
+
+		err = s.nucleiExecutor.ExecuteScan(req)
+		if err != nil {
+			gologger.Warning().Msgf("Could not run nuclei: %s\n", err)
+			return
+		}
+	} else if s.passiveNuclei != nil && req.RawResponse != "" {
+		var reqRespBuilder strings.Builder
+		reqRespBuilder.WriteString(req.RawRequest)
+		reqRespBuilder.WriteString("\n\n")
+		reqRespBuilder.WriteString(req.RawResponse)
+
+		results, err := s.passiveNuclei.Execute(context.Background(), reqRespBuilder.String(), req.URL)
+		if err != nil {
+			gologger.Warning().Msgf("Could not run nuclei: %s\n", err)
+			return
+		}
+		for _, result := range results {
+			if err := s.options.OutputWriter.Write(result); err != nil {
+				gologger.Warning().Msgf("Could not write result: %s\n", err)
+			}
+		}
 	}
 }

internal/server/server.go

@@ -37,6 +37,9 @@ type DASTServer struct {
 	endpointsBeingTested atomic.Int64
 
 	nucleiExecutor *nucleiExecutor
+	passiveNuclei  *PassiveNucleiExecutor
+
+	proxyServer *ProxyServer
 }
 
 // Options contains the configuration options for the server.
@@ -50,6 +53,11 @@ type Options struct {
 	// Verbose is a flag that controls verbose output
 	Verbose bool
 
+	// ProxyServerPort is the port to use for the proxy server
+	ProxyServerPort int
+	// ProxyCacheDirectory is the directory to use for the proxy server cache
+	ProxyCacheDirectory string
+
 	// Scope fields for fuzzer
 	InScope  []string
 	OutScope []string
@@ -59,8 +67,7 @@ type Options struct {
 	NucleiExecutorOptions *NucleiExecutorOptions
 }
 
-// New creates a new instance of the DAST server.
-func New(options *Options) (*DASTServer, error) {
+func setupDastDefaults(options *Options) {
 	// If the user has specified no templates, use the default ones
 	// for DAST only.
 	if len(options.Templates) == 0 {
@@ -72,8 +79,19 @@ func New(options *Options) (*DASTServer, error) {
 		options.NucleiExecutorOptions.Options.BulkSize = 1
 		options.NucleiExecutorOptions.Options.TemplateThreads = 1
 	}
-	maxWorkers := env.GetEnvOrDefault[int]("FUZZ_MAX_WORKERS", 1)
-	bufferSize := env.GetEnvOrDefault[int]("FUZZ_BUFFER_SIZE", 10000)
+}
+
+// New creates a new instance of the DAST server.
+func New(options *Options) (*DASTServer, error) {
+	dastServerMode := options.NucleiExecutorOptions.Options.DASTServer
+	passiveServerMode := options.NucleiExecutorOptions.Options.PassiveServer
+
+	if dastServerMode {
+		setupDastDefaults(options)
+	}
+
+	maxWorkers := env.GetEnvOrDefault("FUZZ_MAX_WORKERS", 25)
+	bufferSize := env.GetEnvOrDefault("FUZZ_BUFFER_SIZE", 10000)
 
 	server := &DASTServer{
 		options:      options,
@@ -83,11 +101,40 @@ func New(options *Options) (*DASTServer, error) {
 	}
 	server.setupHandlers(false)
 
-	executor, err := newNucleiExecutor(options.NucleiExecutorOptions)
-	if err != nil {
-		return nil, err
+	if dastServerMode {
+		executor, err := newNucleiExecutor(options.NucleiExecutorOptions)
+		if err != nil {
+			return nil, err
+		}
+		server.nucleiExecutor = executor
+	} else if passiveServerMode {
+		executor, err := NewPassiveNucleiExecutor(config.DefaultConfig.TemplatesDirectory)
+		if err != nil {
+			return nil, err
+		}
+		server.passiveNuclei = executor
+	}
+
+	if dastServerMode || passiveServerMode {
+		if options.ProxyCacheDirectory == "" {
+			options.ProxyCacheDirectory = "proxify-cache"
+		}
+		proxyServer, err := NewProxyServer(&ProxyOptions{
+			Port:           options.ProxyServerPort,
+			CacheDirectory: options.ProxyCacheDirectory,
+			OnIntercepted: func(pair RequestResponsePair) {
+				server.consumeTaskRequest(PostRequestsHandlerRequest{
+					RawRequest:  pair.Request,
+					RawResponse: pair.Response,
+					URL:         pair.URL,
+				})
+			},
+		})
+		if err != nil {
+			return nil, err
+		}
+		server.proxyServer = proxyServer
 	}
-	server.nucleiExecutor = executor
 
 	scopeManager, err := scope.NewManager(
 		options.InScope,
@@ -103,8 +150,8 @@ func New(options *Options) (*DASTServer, error) {
 	if options.Token != "" {
 		builder.WriteString(" (with token)")
 	}
-	gologger.Info().Msgf("DAST Server API: %s", server.buildURL("/fuzz"))
-	gologger.Info().Msgf("DAST Server Stats URL: %s", server.buildURL("/stats"))
+	gologger.Info().Msgf("Server API: %s", server.buildURL("/fuzz"))
+	gologger.Info().Msgf("Server Stats URL: %s", server.buildURL("/stats"))
 
 	return server, nil
 }
@@ -124,7 +171,12 @@ func NewStatsServer(fuzzStatsDB *stats.Tracker) (*DASTServer, error) {
 }
 
 func (s *DASTServer) Close() {
-	s.nucleiExecutor.Close()
+	if s.nucleiExecutor != nil {
+		s.nucleiExecutor.Close()
+	}
+	if s.proxyServer != nil {
+		s.proxyServer.Close()
+	}
 	s.echo.Close()
 	s.tasksPool.StopAndWaitFor(1 * time.Minute)
 }
@@ -160,7 +212,7 @@ func (s *DASTServer) setupHandlers(onlyStats bool) {
 
 	if s.options.Token != "" {
 		e.Use(middleware.KeyAuthWithConfig(middleware.KeyAuthConfig{
-			KeyLookup: "query:token",
+			KeyLookup: "header:X-DAST-Server-Token",
 			Validator: func(key string, c echo.Context) (bool, error) {
 				return key == s.options.Token, nil
 			},
@@ -187,8 +239,9 @@ func (s *DASTServer) Start() error {
 
 // PostReuestsHandlerRequest is the request body for the /fuzz POST handler.
 type PostRequestsHandlerRequest struct {
-	RawHTTP string `json:"raw_http"`
-	URL     string `json:"url"`
+	RawRequest  string `json:"raw_request"`
+	RawResponse string `json:"raw_response"`
+	URL         string `json:"url"`
 }
 
 func (s *DASTServer) handleRequest(c echo.Context) error {
@@ -199,7 +252,7 @@ func (s *DASTServer) handleRequest(c echo.Context) error {
 	}
 
 	// Validate the request
-	if req.RawHTTP == "" || req.URL == "" {
+	if (req.RawRequest == "" || req.URL == "") && req.RawResponse == "" {
 		fmt.Printf("Missing required fields\n")
 		return c.JSON(400, map[string]string{"error": "missing required fields"})
 	}
@@ -253,10 +306,15 @@ func (s *DASTServer) getStats() (StatsResponse, error) {
 		DASTScanStatistics: DASTScanStatistics{
 			EndpointsInQueue:     s.endpointsInQueue.Load(),
 			EndpointsBeingTested: s.endpointsBeingTested.Load(),
-			TotalTemplatesLoaded: int64(len(s.nucleiExecutor.store.Templates())),
 		},
 	}
-	if s.nucleiExecutor.executorOpts.FuzzStatsDB != nil {
+	totalTemplatesLoaded := 0
+	if s.nucleiExecutor != nil {
+		totalTemplatesLoaded = len(s.nucleiExecutor.store.Templates())
+	}
+	resp.DASTScanStatistics.TotalTemplatesLoaded = int64(totalTemplatesLoaded)
+
+	if s.nucleiExecutor != nil && s.nucleiExecutor.executorOpts.FuzzStatsDB != nil {
 		fuzzStats := s.nucleiExecutor.executorOpts.FuzzStatsDB.GetStats()
 		resp.DASTScanSeverityBreakdown = fuzzStats.SeverityCounts
 		resp.DASTScanStatusStatistics = fuzzStats.StatusCodes

pkg/catalog/config/constants.go

@@ -31,7 +31,7 @@ const (
 	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
-	Version = `v3.3.10`
+	Version = `v3.4.2`
 	// Directory Names of custom templates
 	CustomS3TemplatesDirName     = "s3"
 	CustomGitHubTemplatesDirName = "github"

pkg/external/customtemplates/s3.go

@@ -62,7 +62,7 @@ func (bk *customTemplateS3Bucket) Update(ctx context.Context) {
 func NewS3Providers(options *types.Options) ([]*customTemplateS3Bucket, error) {
 	providers := []*customTemplateS3Bucket{}
 	if options.AwsBucketName != "" && !options.AwsTemplateDisableDownload {
-		s3c, err := getS3Client(context.TODO(), options.AwsAccessKey, options.AwsSecretKey, options.AwsRegion)
+		s3c, err := getS3Client(context.TODO(), options.AwsAccessKey, options.AwsSecretKey, options.AwsRegion, options.AwsProfile)
 		if err != nil {
 			return nil, errorutil.NewWithErr(err).Msgf("error downloading s3 bucket %s", options.AwsBucketName)
 		}
@@ -104,10 +104,24 @@ func downloadToFile(downloader *manager.Downloader, targetDirectory, bucket, key
 	return err
 }
 
-func getS3Client(ctx context.Context, accessKey string, secretKey string, region string) (*s3.Client, error) {
-	cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")), config.WithRegion(region))
-	if err != nil {
-		return nil, err
+func getS3Client(ctx context.Context, accessKey string, secretKey string, region string, profile string) (*s3.Client, error) {
+	var cfg aws.Config
+	var err error
+	if profile != "" {
+		cfg, err = config.LoadDefaultConfig(ctx, config.WithSharedConfigProfile(profile))
+		if err != nil {
+			return nil, err
+		}
+	} else if accessKey != "" && secretKey != "" {
+		cfg, err = config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")), config.WithRegion(region))
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		cfg, err = config.LoadDefaultConfig(ctx)
+		if err != nil {
+			return nil, err
+		}
 	}
 	return s3.NewFromConfig(cfg), nil
 }

pkg/output/output.go

@@ -330,7 +330,7 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
 			data = decolorizerRegex.ReplaceAll(data, []byte(""))
 		}
 		if _, writeErr := w.outputFile.Write(data); writeErr != nil {
-			return errors.Wrap(err, "could not write to output")
+			return errors.Wrap(writeErr, "could not write to output")
 		}
 		if w.AddNewLinesOutputFile && w.json {
 			_, _ = w.outputFile.Write([]byte("\n"))

pkg/protocols/network/network.go

@@ -259,3 +259,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 func (request *Request) Requests() int {
 	return len(request.Address)
 }
+
+func (request *Request) SetDialer(dialer *fastdialer.Dialer) {
+	request.dialer = dialer
+}

pkg/protocols/offlinehttp/request.go

@@ -114,7 +114,7 @@ func (request *Request) executeRawInput(data, inputString string, input *context
 		return errors.Wrap(err, "could not read raw http response body")
 	}
 	reqURL := inputString
-	if inputString == "" {
+	if inputString == "" && resp.Request != nil {
 		reqURL = getURLFromRequest(resp.Request)
 	}
 

pkg/protocols/utils/utils.go

@@ -48,7 +48,7 @@ func CalculateContentLength(contentLength, bodyLength int64) int64 {
 	return bodyLength
 }
 
-// headersToString converts http headers to string
+// HeadersToString converts http headers to string
 func HeadersToString(headers http.Header) string {
 	builder := &strings.Builder{}
 

pkg/reporting/trackers/jira/jira.go

@@ -3,6 +3,7 @@ package jira
 import (
 	"fmt"
 	"io"
+	"net/http"
 	"net/url"
 	"strings"
 	"sync"
@@ -68,9 +69,12 @@ type Options struct {
 	// AccountID is the accountID of the jira user.
 	AccountID string `yaml:"account-id" json:"account_id" validate:"required"`
 	// Email is the email of the user for jira instance
-	Email string `yaml:"email" json:"email" validate:"required,email"`
+	Email string `yaml:"email" json:"email"`
+	// PersonalAccessToken is the personal access token for jira instance.
+	// If this is set, Bearer Auth is used instead of Basic Auth.
+	PersonalAccessToken string `yaml:"personal-access-token" json:"personal_access_token"`
 	// Token is the token for jira instance.
-	Token string `yaml:"token" json:"token" validate:"required"`
+	Token string `yaml:"token" json:"token"`
 	// ProjectName is the name of the project.
 	ProjectName string `yaml:"project-name" json:"project_name"`
 	// ProjectID is the ID of the project (optional)
@@ -103,14 +107,28 @@ func New(options *Options) (*Integration, error) {
 	if !options.Cloud {
 		username = options.AccountID
 	}
-	tp := jira.BasicAuthTransport{
-		Username: username,
-		Password: options.Token,
+
+	var httpclient *http.Client
+	if options.PersonalAccessToken != "" {
+		bearerTp := jira.BearerAuthTransport{
+			Token: options.PersonalAccessToken,
+		}
+		if options.HttpClient != nil {
+			bearerTp.Transport = options.HttpClient.HTTPClient.Transport
+		}
+		httpclient = bearerTp.Client()
+	} else {
+		basicTp := jira.BasicAuthTransport{
+			Username: username,
+			Password: options.Token,
+		}
+		if options.HttpClient != nil {
+			basicTp.Transport = options.HttpClient.HTTPClient.Transport
+		}
+		httpclient = basicTp.Client()
 	}
-	if options.HttpClient != nil {
-		tp.Transport = options.HttpClient.HTTPClient.Transport
-	}
-	jiraClient, err := jira.NewClient(tp.Client(), options.URL)
+
+	jiraClient, err := jira.NewClient(httpclient, options.URL)
 	if err != nil {
 		return nil, err
 	}

pkg/templates/templates.go

@@ -317,7 +317,7 @@ func (template *Template) MarshalYAML() ([]byte, error) {
 	return out, multierr.Append(marshalErr, errValidate)
 }
 
-// MarshalYAML forces recursive struct validation after unmarshal operation
+// UnmarshalYAML forces recursive struct validation after unmarshal operation
 func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type Alias Template
 	alias := &Alias{}
@@ -475,7 +475,7 @@ func (template *Template) GetFileImports() []string {
 	return template.ImportedFiles
 }
 
-// addProtocolsToQueue adds protocol requests to the queue and preserves order of the protocols and requests
+// addRequestsToQueue adds protocol requests to the queue and preserves order of the protocols and requests
 func (template *Template) addRequestsToQueue(keys ...string) {
 	for _, key := range keys {
 		switch key {

pkg/types/types.go

@@ -345,6 +345,8 @@ type Options struct {
 	GitLabTemplateRepositoryIDs []int
 	// GitLabTemplateDisableDownload disables downloading templates from custom GitLab repositories
 	GitLabTemplateDisableDownload bool
+	// AWS access profile from ~/.aws/credentials file for downloading templates from S3 bucket
+	AwsProfile string
 	// AWS access key for downloading templates from S3 bucket
 	AwsAccessKey string
 	// AWS secret key for downloading templates from S3 bucket
@@ -425,6 +427,12 @@ type Options struct {
 	DAST bool
 	// DASTServer is the flag to start nuclei as a DAST server
 	DASTServer bool
+	// PassiveServer is the flag to start nuclei as a passive server
+	PassiveServer bool
+	// ProxyServerPort is the port to use for the proxy server
+	ProxyServerPort int
+	// ProxyCacheDirectory is the directory to use for the proxy server cache
+	ProxyCacheDirectory string
 	// DASTServerToken is the token optional for the dast server
 	DASTServerToken string
 	// DASTServerAddress is the address for the dast server