Merge pull request #1038 from projectdiscovery/dev

v2.5.2 Bugfix release

.github/dependabot.yml

@@ -11,6 +11,7 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    target-branch: "dev"
     commit-message:
       prefix: "chore"
       include: "scope"
@@ -20,6 +21,7 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    target-branch: "dev"
     commit-message:
       prefix: "chore"
       include: "scope"
@@ -29,6 +31,7 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    target-branch: "dev"
     commit-message:
       prefix: "chore"
-      include: "scope"
+      include: "scope"

.github/workflows/build-test.yml

@@ -0,0 +1,31 @@
+name: 🔨 Build Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+
+jobs:  
+  build:
+    name: Test Builds
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Test
+        run: go test ./...
+        working-directory: v2/
+
+      - name: Integration Tests
+        run: bash run.sh
+        working-directory: integration_tests/
+
+      - name: Build
+        run: go build .
+        working-directory: v2/cmd/nuclei/

.github/workflows/build.yaml

@@ -1,45 +0,0 @@
-name: Build
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-
-jobs:
-  lint:
-    name: golangci-lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v2
-        with:
-          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.33
-          args: --timeout 5m
-          working-directory: v2/
-        
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.15
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Test
-        run: go test ./...
-        working-directory: v2/
-
-      - name: Integration Tests
-        run: bash run.sh
-        working-directory: integration_tests/
-
-      - name: Build
-        run: go build .
-        working-directory: v2/cmd/nuclei/

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,38 @@
+name: 🚨 CodeQL Analysis
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - dev
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/dockerhub-push.yml

@@ -1,19 +1,34 @@
-# dockerhub-push pushes docker build to dockerhub automatically
-# on the creation of a new release
-name: Publish to Dockerhub on creation of a new release
-on: 
+name: 🌥 Docker Push
+
+on:
   release:
     types: [published]
+  workflow_dispatch:
+
 jobs:
-  update:
+  docker:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@master
-    - name: Publish to Dockerhub Registry
-      #pre: echo ::save-state name=RELEASE_VERSION::$(echo ${GITHUB_REF:10})
-      uses: elgohr/Publish-Docker-Github-Action@master
-      with:
-        name: projectdiscovery/nuclei
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-        tags: "latest" #"latest,${{ env.STATE_RELEASE_VERSION }}"
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64,linux/arm
+          push: true
+          tags: projectdiscovery/nuclei:latest

.github/workflows/functional-test.yml

@@ -0,0 +1,25 @@
+name: 🧪 Functional Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+
+jobs:  
+  functional:
+    name: Functional Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Functional Tests
+        run: |
+          chmod +x run.sh
+          bash run.sh
+        working-directory: v2/cmd/functional-test

.github/workflows/lint-test.yml

@@ -0,0 +1,19 @@
+name: 🙏🏻 Lint Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  lint:
+    name: Lint Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          args: --timeout 5m
+          working-directory: v2/

.github/workflows/publish-docs.yaml

@@ -0,0 +1,51 @@
+name: ⏰ Publish Docs
+
+on:
+  push:
+    branches:
+      - dev
+  workflow_dispatch:
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@master
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+ 
+      - name: "Set up Go"
+        uses: actions/setup-go@v2
+        with: 
+          go-version: 1.17
+
+      - name: Generate YAML Syntax Documentation
+        id: generate-docs
+        run: |
+          if ! which dstdocgen > /dev/null; then
+            echo -e "Command dstdocgen not found! Installing\c"
+            go install github.com/projectdiscovery/yamldoc-go/cmd/docgen/dstdocgen@main
+          fi
+          go generate pkg/templates/templates.go
+          go build -o "cmd/docgen/docgen" cmd/docgen/docgen.go 
+          ./cmd/docgen/docgen ../SYNTAX-REFERENCE.md ../nuclei-jsonschema.json
+          echo "::set-output name=changes::$(git status -s | wc -l)"
+        working-directory: v2
+
+      - name: Commit files
+        if: steps.generate-docs.outputs.changes > 0
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          git add SYNTAX-REFERENCE.md nuclei-jsonschema.json
+          git commit -m "Auto Generate Syntax Docs + JSONSchema [$(date)] :robot:" -a
+
+      - name: Push changes
+        if: steps.generate-docs.outputs.changes > 0
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch: ${{ github.ref }}

.github/workflows/release-binary.yml

@@ -1,8 +1,9 @@
-name: Release
+name: 🎉 Release Binary
 on:
   create:
     tags:
       - v*
+  workflow_dispatch:
 
 jobs: 
   release: 
@@ -17,7 +18,7 @@ jobs:
         name: "Set up Go"
         uses: actions/setup-go@v2
         with: 
-          go-version: 1.15
+          go-version: 1.17
       - 
         env: 
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
@@ -26,5 +27,4 @@ jobs:
         with: 
           args: "release --rm-dist"
           version: latest
-          workdir: v2/
-        
+          workdir: v2/

.gitignore

@@ -1,6 +1,13 @@
-cmd/nuclei/nuclei*
-v2/cmd/nuclei/nuclei
 .idea
-integration_tests/integration-test
 integration_tests/nuclei
-v2/cmd/integration-test/integration-test
+integration_tests/integration-test
+v2/cmd/nuclei/main
+v2/cmd/nuclei/nuclei
+v2/cmd/integration-test/nuclei
+v2/cmd/functional-test/nuclei_dev
+v2/cmd/functional-test/nuclei_main
+v2/cmd/functional-test/functional-test
+v2/cmd/docgen/docgen
+v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class
+v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class
+v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser

.golangci.yml

@@ -1,115 +0,0 @@
-linters-settings:
-  dupl:
-    threshold: 100
-  exhaustive:
-    default-signifies-exhaustive: false
-  # funlen:
-  #   lines: 100
-  #   statements: 50
-  #goconst:
-  #  min-len: 2
-  #  min-occurrences: 2
-  gocritic:
-    enabled-tags:
-      - diagnostic
-      - experimental
-      - opinionated
-      - performance
-      - style
-    disabled-checks:
-      - dupImport # https://github.com/go-critic/go-critic/issues/845
-      - ifElseChain
-  # gocyclo:
-  #   min-complexity: 15
-  goimports:
-    local-prefixes: github.com/golangci/golangci-lint
-  golint:
-    min-confidence: 0
-  gomnd:
-    settings:
-      mnd:
-        # don't include the "operation" and "assign"
-        checks: argument,case,condition,return
-  govet:
-    check-shadowing: true
-    settings:
-      printf:
-        funcs:
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
-  # lll:
-  #   line-length: 140
-  maligned:
-    suggest-new: true
-  misspell:
-    locale: US
-  nolintlint:
-    allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
-    allow-unused: false # report any unused nolint directives
-    require-explanation: false # don't require an explanation for nolint directives
-    require-specific: false # don't require nolint directives to be specific about which linter is being skipped
-
-linters:
-  # please, do not use `enable-all`: it's deprecated and will be removed soon.
-  # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
-  disable-all: true
-  enable:
-    #- bodyclose
-    - deadcode
-    - dogsled
-    - errcheck
-    - exhaustive
-    - gochecknoinits
-    #- goconst
-    - gocritic
-    - gofmt
-    - goimports
-    - golint
-    #- gomnd
-    - goprintffuncname
-    - gosimple
-    - govet
-    - ineffassign
-    - interfacer
-    - maligned
-    - misspell
-    - nakedret
-    - noctx
-    - nolintlint
-    - rowserrcheck
-    - scopelint
-    - staticcheck
-    - structcheck
-    - stylecheck
-    - typecheck
-    - unconvert
-    - unparam
-    - unused
-    - varcheck
-    - whitespace
-
-  # don't enable:
-  # - depguard
-  # - asciicheck
-  # - funlen
-  # - gochecknoglobals
-  # - gocognit
-  # - gocyclo
-  # - godot
-  # - godox
-  # - goerr113
-  # - gosec
-  # - lll
-  # - nestif
-  # - prealloc
-  # - testpackage
-  # - wsl
-
-# golangci.com configuration
-# https://github.com/golangci/golangci/wiki/Configuration
-service:
-  golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly
-  prepare:
-    - echo "here I can run custom commands, but no preparation needed for this repo"

.run/IntegrationTests.run.xml

@@ -0,0 +1,19 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="IntegrationTests" type="ShConfigurationType">
+    <option name="SCRIPT_TEXT" value="" />
+    <option name="INDEPENDENT_SCRIPT_PATH" value="true" />
+    <option name="SCRIPT_PATH" value="$PROJECT_DIR$/integration_tests/run.sh" />
+    <option name="SCRIPT_OPTIONS" value="" />
+    <option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
+    <option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$/integration_tests/" />
+    <option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
+    <option name="INTERPRETER_PATH" value="/bin/zsh" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="EXECUTE_IN_TERMINAL" value="true" />
+    <option name="EXECUTE_SCRIPT_FILE" value="true" />
+    <envs>
+      <env name="DEBUG" value="true" />
+    </envs>
+    <method v="2" />
+  </configuration>
+</component>

.run/RegressionTests.run.xml

@@ -0,0 +1,19 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="RegressionTests" type="ShConfigurationType">
+    <option name="SCRIPT_TEXT" value="" />
+    <option name="INDEPENDENT_SCRIPT_PATH" value="true" />
+    <option name="SCRIPT_PATH" value="$PROJECT_DIR$/v2/cmd/functional-test/run.sh" />
+    <option name="SCRIPT_OPTIONS" value="" />
+    <option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
+    <option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$/v2/cmd/functional-test/" />
+    <option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
+    <option name="INTERPRETER_PATH" value="/bin/zsh" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="EXECUTE_IN_TERMINAL" value="true" />
+    <option name="EXECUTE_SCRIPT_FILE" value="true" />
+    <envs>
+      <env name="DEBUG" value="true" />
+    </envs>
+    <method v="2" />
+  </configuration>
+</component>

.run/UnitTests.run.xml

@@ -0,0 +1,12 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="UnitTests" type="GoTestRunConfiguration" factoryName="Go Test">
+    <module name="nuclei" />
+    <working_directory value="$PROJECT_DIR$/v2" />
+    <go_parameters value="-i" />
+    <kind value="DIRECTORY" />
+    <directory value="$PROJECT_DIR$/v2/" />
+    <filePath value="$PROJECT_DIR$" />
+    <framework value="gotest" />
+    <method v="2" />
+  </configuration>
+</component>

Dockerfile

@@ -1,7 +1,7 @@
-FROM golang:1.15-alpine as build-env
+FROM golang:1.17.1-alpine as build-env
 RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
 
-FROM alpine:latest
+FROM alpine:3.14
 RUN apk add --no-cache bind-tools ca-certificates
 COPY --from=build-env /go/bin/nuclei /usr/local/bin/nuclei
 ENTRYPOINT ["nuclei"]

README.md

@@ -11,8 +11,8 @@
 <a href="https://github.com/projectdiscovery/nuclei/issues"><img src="https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat"></a>
 <a href="https://github.com/projectdiscovery/nuclei/releases"><img src="https://img.shields.io/github/release/projectdiscovery/nuclei"></a>
 <a href="https://twitter.com/pdnuclei"><img src="https://img.shields.io/twitter/follow/pdnuclei.svg?logo=twitter"></a>
-<a href="https://discord.gg/KECAGdH"><img src="https://img.shields.io/discord/695645237418131507.svg?logo=discord"></a>
-<a href="https://github.com/projectdiscovery/nuclei/actions/workflows/build.yaml"><img src="https://github.com/projectdiscovery/nuclei/actions/workflows/build.yaml/badge.svg?branch=master"></a>
+<a href="https://discord.gg/projectdiscovery"><img src="https://img.shields.io/discord/695645237418131507.svg?logo=discord"></a>
+<a href="https://github.com/projectdiscovery/nuclei/actions/workflows/build-test.yml"><img src="https://github.com/projectdiscovery/nuclei/actions/workflows/build-test.yml/badge.svg?branch=master"></a>
 </p>
       
 <p align="center">
@@ -22,15 +22,15 @@
   <a href="#for-developers-and-organisations">For Developers</a> •
   <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">Documentation</a> •
   <a href="#credits">Credits</a> •
-  <a href="#license">License</a> •
-  <a href="https://discord.gg/KECAGdH">Join Discord</a>
+  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">FAQs</a> •
+  <a href="https://discord.gg/projectdiscovery">Join Discord</a>
 </p>
 
 ---
 
 Nuclei is used to send requests across targets based on a template leading to zero false positives and providing fast scanning on large number of hosts. Nuclei offers scanning for a variety of protocols including TCP, DNS, HTTP, File, etc. With powerful and flexible templating, all kinds of security checks can be modelled with Nuclei.
 
-We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-templates) that houses various type of vulnerability templates contributed by **more than 100** security researchers and engineers. It is preloaded with ready to use templates using `-update-templates` flag.
+We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-templates) that houses various type of vulnerability templates contributed by **more than 200** security researchers and engineers.
 
 
 
@@ -45,33 +45,161 @@ We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-temp
 # Install Nuclei
 
 ```sh
-▶ GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
 ```
 
-**More installation [methods can be found here](https://nuclei.projectdiscovery.io/nuclei/get-started/#nuclei-installation).**
+**More installation [methods can be found here](https://nuclei.projectdiscovery.io/nuclei/get-started/).**
 
 <table>
 <tr>
 <td>  
 
-### Download Templates
+### Nuclei Templates
 
-You can download and update the nuclei templates using <ins>*update-templates*</ins> flag of nuclei that downloads all the available **nuclei-templates** from [Github project](https://github.com/projectdiscovery/nuclei-templates), a community curated list of templates that are ready to use.
+Nuclei has had built-in support for automatic template download/update as default since version [v2.5.2](https://github.com/projectdiscovery/nuclei/releases/tag/v2.5.2). [**Nuclei-Templates**](https://github.com/projectdiscovery/nuclei-templates) project provides a community-contributed list of ready-to-use templates that is constantly updated.
 
-`▶ nuclei -update-templates`
+You may still use the `update-templates` flag to update the nuclei templates at any time; You can write your own checks for your individual workflow and needs following Nuclei's [templating guide](https://nuclei.projectdiscovery.io/templating-guide/).
 
-Nuclei is designed to used with custom templates according to the target and workflow, you can write your own checks for your specific workflow and needs, please refer to nuclei [templating guide](https://nuclei.projectdiscovery.io/templating-guide/) to write your own custom templates.
+The YAML DSL reference syntax is available [here](SYNTAX-REFERENCE.md).
 
 </td>
 </tr>
 </table>
 
-### Running Nuclei
-
-Scanning for CVEs on given list of URLs.
+### Usage
 
 ```sh
-▶ nuclei -l target_urls.txt -t cves/
+nuclei -h
+```
+
+This will display help for the tool. Here are all the switches it supports.
+
+
+```yaml
+Nuclei is a fast, template based vulnerability scanner focusing
+on extensive configurability, massive extensibility and ease of use.
+
+Usage:
+  nuclei [flags]
+
+Flags:
+TARGET:
+   -u, -target string[]  target URLs/hosts to scan
+   -l, -list string      path to file containing a list of target URLs/hosts to scan (one per line)
+
+TEMPLATES:
+   -tl                      list all available templates
+   -t, -templates string[]  template or template directory paths to include in the scan
+   -w, -workflows string[]  list of workflows to run
+   -nt, -new-templates      run newly added templates only
+   -validate                validate the passed templates to nuclei
+
+FILTERING:
+   -tags string[]                         execute a subset of templates that contain the provided tags
+   -include-tags string[]                 tags from the default deny list that permit executing more intrusive templates
+   -etags, -exclude-tags string[]         exclude templates with the provided tags
+   -include-templates string[]            templates to be executed even if they are excluded either by default or configuration
+   -exclude-templates, -exclude string[]  template or template directory paths to exclude
+   -severity, -impact value[]             Templates to run based on severity. Possible values: info, low, medium, high, critical
+   -author string[]                       execute templates that are (co-)created by the specified authors
+
+OUTPUT:
+   -o, -output string            output file to write found issues/vulnerabilities
+   -silent                       display findings only
+   -v, -verbose                  show verbose output
+   -vv                           display extra verbose information
+   -nc, -no-color                disable output content coloring (ANSI escape codes)
+   -json                         write output in JSONL(ines) format
+   -irr, -include-rr             include request/response pairs in the JSONL output (for findings only)
+   -nm, -no-meta                 don't display match metadata in CLI output
+   -nts, -no-timestamp           don't display timestamp metadata in CLI output
+   -rdb, -report-db string       local nuclei reporting database (always use this to persist report data)
+   -me, -markdown-export string  directory to export results in markdown format
+   -se, -sarif-export string     file to export results in SARIF format
+
+CONFIGURATIONS:
+   -config string              path to the nuclei configuration file
+   -rc, -report-config string  nuclei reporting module configuration file
+   -H, -header string[]        custom headers in header:value format
+   -V, -var value              custom vars in var=value format
+   -r, -resolvers string       file containing resolver list for nuclei
+   -system-resolvers           use system DNS resolving as error fallback
+   -passive                    enable passive HTTP response processing mode
+   -env-vars                   enable environment variables support
+
+INTERACTSH:
+   -no-interactsh                     disable interactsh server for OOB testing
+   -interactsh-url string             interactsh server url for self-hosted instance (default "https://interact.sh")
+   -interactsh-token string           authentication token for self-hosted interactsh server
+   -interactions-cache-size int       number of requests to keep in the interactions cache (default 5000)
+   -interactions-eviction int         number of seconds to wait before evicting requests from cache (default 60)
+   -interactions-poll-duration int    number of seconds to wait before each interaction poll request (default 5)
+   -interactions-cooldown-period int  extra time for interaction polling before exiting (default 5)
+
+RATE-LIMIT:
+   -rl, -rate-limit int          maximum number of requests to send per second (default 150)
+   -rlm, -rate-limit-minute int  maximum number of requests to send per minute
+   -bs, -bulk-size int           maximum number of hosts to be analyzed in parallel per template (default 25)
+   -c, -concurrency int          maximum number of templates to be executed in parallel (default 10)
+
+OPTIMIZATIONS:
+   -timeout int               time to wait in seconds before timeout (default 5)
+   -retries int               number of times to retry a failed request (default 1)
+   -max-host-error int        max errors for a host before skipping from scan (default 30)
+   -project                   use a project folder to avoid sending same request multiple times
+   -project-path string       set a specific project path (default "$TMPDIR/")
+   -spm, -stop-at-first-path  stop processing HTTP requests after the first match (may break template/workflow logic)
+
+HEADLESS:
+   -headless          enable templates that require headless browser support
+   -page-timeout int  seconds to wait for each page in headless mode (default 20)
+   -show-browser      show the browser on the screen when running templates with headless mode
+
+DEBUG:
+   -debug                     show all requests and responses
+   -debug-req                 show all sent requests
+   -debug-resp                show all received responses
+   -proxy, -proxy-url string  URL of the HTTP proxy server
+   -proxy-socks-url string    URL of the SOCKS proxy server
+   -trace-log string          file to write sent requests trace log
+   -version                   show nuclei version
+   -tv, -templates-version    shows the version of the installed nuclei-templates
+
+UPDATE:
+   -update                        update nuclei to the latest released version
+   -ut, -update-templates         update the community templates to latest released version
+   -nut, -no-update-templates     do not check for nuclei-templates updates
+   -ud, -update-directory string  overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates")
+
+STATISTICS:
+   -stats                    display statistics about the running scan
+   -stats-json               write statistics data to an output file in JSONL(ines) format
+   -si, -stats-interval int  number of seconds to wait between showing a statistics update (default 5)
+   -metrics                  expose nuclei metrics on a port
+   -metrics-port int         port to expose nuclei metrics on (default 9092)
+```
+
+### Running Nuclei
+
+Scanning target domain with [community-curated](https://github.com/projectdiscovery/nuclei-templates) nuclei templates.
+
+```sh
+nuclei -u https://example.com
+```
+
+Scanning target URLs with [community-curated](https://github.com/projectdiscovery/nuclei-templates) nuclei templates.
+
+```sh
+nuclei -list urls.txt
+```
+
+Example of `urls.txt`:
+
+```yaml
+http://example.com
+http://app.example.com
+http://test.example.com
+http://uat.example.com
 ```
 
 **More detailed examples of running nuclei can be found [here](https://nuclei.projectdiscovery.io/nuclei/get-started/#running-nuclei).**
@@ -92,9 +220,9 @@ Nuclei offers great number of features that are helpful for security engineers t
 <tr>
 <td>  
 
-**For bugbounty hunters:**
+**For Bug Bounty hunters:**
 
-Nuclei allows you to customise your testing approach with your own suite of checks and easily run across your bug bounty programs. Moroever, Nuclei can be easily integrated into any continuous scanning workflow.
+Nuclei allows you to customise your testing approach with your own suite of checks and easily run across your bug bounty programs. Moreover, Nuclei can be easily integrated into any continuous scanning workflow.
 
 - Designed to be easily integrated into other tool workflow.
 - Can process thousands of hosts in few minutes.
@@ -110,9 +238,9 @@ Please check our other open-source projects that might fit into your bug bounty
 <tr>
 <td>
   
-**For pentesters:**
+**For Penetration Testers:**
 
-Nuclei immensely improve how you approach security assessment by augmenting the manual repetitve processes. Consultancies are already converting their manual assessment steps with Nuclei, it allows them to run set of their custom assessment approach across thousands of hosts in an automated manner. 
+Nuclei immensely improve how you approach security assessment by augmenting the manual, repetitive processes. Consultancies are already converting their manual assessment steps with Nuclei, it allows them to run set of their custom assessment approach across thousands of hosts in an automated manner. 
 
 Pen-testers get the full power of our public templates and customization capabilities to speed-up their assessment process, and specifically with the regression cycle where you can easily verify the fix.
 
@@ -127,7 +255,7 @@ Pen-testers get the full power of our public templates and customization capabil
 
 # For Developers and Organisations
 
-Nuclei is built with simplicity in mind, with the community backed templates by hundreds of security researchers, it allows you to stay updated with latest security threats using continuous Nuclei scanning on the hosts. It is designed to be easily integrated into regression tests cycle, to verify the fixes and  eliminate vulnerabilities from occuring in future.
+Nuclei is built with simplicity in mind, with the community backed templates by hundreds of security researchers, it allows you to stay updated with the latest security threats using continuous Nuclei scanning on the hosts. It is designed to be easily integrated into regression tests cycle, to verify the fixes and eliminate vulnerabilities from occurring in the future.
 
 - **CI/CD:** Engineers are already utilising Nuclei within their CI/CD pipeline, it allows them to constantly monitor their staging and production environments with customised templates.
 - **Continuous Regression Cycle:** With Nuclei, you can create your custom template on every new identified vulnerability and put into Nuclei engine to eliminate in the continuous regression cycle.
@@ -147,8 +275,10 @@ We have [a discussion thread around this](https://github.com/projectdiscovery/nu
 - [Community Powered Scanning with Nuclei](https://blog.projectdiscovery.io/community-powered-scanning-with-nuclei/)
 - [Nuclei Unleashed - Quickly write complex exploits](https://blog.projectdiscovery.io/nuclei-unleashed-quickly-write-complex-exploits/)
 - [Nuclei - Fuzz all the things](https://blog.projectdiscovery.io/nuclei-fuzz-all-the-things/)
+- [Nuclei + Interactsh Integration for Automating OOB Testing](https://blog.projectdiscovery.io/nuclei-interactsh-integration/)
 - [Weaponizes nuclei Workflows to Pwn All the Things](https://medium.com/@dwisiswant0/weaponizes-nuclei-workflows-to-pwn-all-the-things-cd01223feb77) by [@dwisiswant0](https://github.com/dwisiswant0)
 - [How to Scan Continuously with Nuclei?](https://medium.com/@dwisiswant0/how-to-scan-continuously-with-nuclei-fcb7e9d8b8b9) by [@dwisiswant0](https://github.com/dwisiswant0)
+- [Hack with Automation !!!](https://dhiyaneshgeek.github.io/web/security/2021/07/19/hack-with-automation/) by [@DhiyaneshGeek](https://github.com/DhiyaneshGeek)
 
 ### Credits
 
@@ -161,5 +291,5 @@ Thanks to all the amazing community [contributors for sending PRs](https://githu
 Nuclei is distributed under [MIT License](https://github.com/projectdiscovery/nuclei/blob/master/LICENSE.md)
 
 <h1 align="left">
-  <a href="https://discord.gg/KECAGdH"><img src="static/Join-Discord.png" width="380" alt="Join Discord"></a> <a href="https://nuclei.projectdiscovery.io"><img src="static/check-nuclei-documentation.png" width="380" alt="Check Nuclei Documentation"></a>
+  <a href="https://discord.gg/projectdiscovery"><img src="static/Join-Discord.png" width="380" alt="Join Discord"></a> <a href="https://nuclei.projectdiscovery.io"><img src="static/check-nuclei-documentation.png" width="380" alt="Check Nuclei Documentation"></a>
 </h1>

README_CN.md

@@ -97,7 +97,7 @@ nuclei -h
 |burp-collaborator-biid|使用burp-collaborator插件|nuclei -burp-collaborator-biid XXXX|
 |c|并行的最大模板数量(默认10)|nuclei -c 10|
 |l|对URL列表进行测试|nuclei -l urls.txt|
-|target|对目标进行测试|nuclei -target hxxps://example.com|
+|target|对目标进行测试|nuclei -target hxxps://example.com -target hxxps://example2.com|
 |t|要检测的模板种类|nuclei -t git-core.yaml -t cves/|
 |no-color|输出不显示颜色|nuclei -no-color|
 |no-meta|不显示匹配的元数据|nuclei -no-meta|
@@ -250,4 +250,4 @@ nano ~/nuclei-templates/.nuclei-ignore
 
 --------
 
-Nuclei是由[projectdiscovery](https://projectdiscovery.io)团队用🖤制作的，当然社区也贡献了很多，通过 **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)**文件以获取更多详细信息。
+Nuclei是由[projectdiscovery](https://projectdiscovery.io)团队用🖤制作的，当然社区也贡献了很多，通过 **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)**文件以获取更多详细信息。

SECURITY.md

@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+DO NOT CREATE AN ISSUE to report a security problem. Instead, please send an email to security@projectdiscovery.io, and we will acknowledge it within 3 working days.

integration_tests/http/raw-unsafe-request.yaml

@@ -17,4 +17,4 @@ requests:
     matchers:
       - type: word
         words:
-          - "This is test-raw-unsafe request matcher."
+          - "This is test raw-unsafe-matcher test"

integration_tests/run.sh

@@ -1,11 +1,12 @@
 #!/bin/bash
 
+rm integration-test nuclei 2>/dev/null
 cd ../v2/cmd/nuclei
 go build
-cp nuclei ../../../integration_tests/nuclei 
+mv nuclei ../../../integration_tests/nuclei 
 cd ../integration-test
 go build
-cp integration-test ../../../integration_tests/integration-test 
+mv integration-test ../../../integration_tests/integration-test 
 cd ../../../integration_tests
 ./integration-test
 if [ $? -eq 0 ]

integration_tests/test-issue-tracker-config1.yaml

@@ -0,0 +1,68 @@
+allow-list:
+  severity: high, critical
+deny-list:
+  severity: low
+
+# github contains configuration options for github issue tracker
+github:
+  # base-url is the optional self-hosted github application url
+  base-url: https://localhost:8443/github
+  # username is the username of the github user
+  username: test-username
+  # owner is the owner name of the repository for issues.
+  owner: test-owner
+  # token is the token for github account.
+  token: test-token
+  # project-name is the name of the repository.
+  project-name: test-project
+  # issue-label is the label of the created issue type
+  issue-label: bug
+
+# gitlab contains configuration options for gitlab issue tracker
+gitlab:
+  # base-url is the optional self-hosted gitlab application url
+  base-url: https://localhost:8443/gitlab
+  # username is the username of the gitlab user
+  username: test-username
+  # token is the token for gitlab account.
+  token: test-token
+  # project-id is the ID of the repository.
+  project-id: 1234
+  # issue-label is the label of the created issue type
+  issue-label: bug
+
+# jira contains configuration options for jira issue tracker
+jira:
+  # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
+  cloud: true
+  # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created
+  update-existing: false
+  # URL is the jira application url
+  url: https://localhost/jira
+  # account-id is the account-id of the jira user or username in case of on-prem Jira
+  account-id: test-account-id
+  # email is the email of the user for jira instance
+  email: [email protected]
+  # token is the token for jira instance or password in case of on-prem Jira
+  token: test-token
+  # project-name is the name of the project.
+  project-name: test-project-name
+  # issue-type is the name of the created issue type
+  issue-type: bug
+
+# elasticsearch contains configuration options for elasticsearch exporter
+elasticsearch:
+  # IP for elasticsearch instance
+  ip: 127.0.0.1
+  # Port is the port of elasticsearch instance
+  port: 9200
+  # IndexName is the name of the elasticsearch index
+  index-name: nuclei
+  # SSL enables ssl for elasticsearch connection
+  ssl: false
+  # SSLVerification disables SSL verification for elasticsearch
+  ssl-verification: false
+  # Username for the elasticsearch instance
+  username: test
+  # Password is the password for elasticsearch instance
+  password: test

integration_tests/test-issue-tracker-config2.yaml

@@ -0,0 +1,70 @@
+allow-list:
+  severity:
+    - high
+    - critical
+deny-list:
+  severity: low
+
+# github contains configuration options for github issue tracker
+github:
+  # base-url is the optional self-hosted github application url
+  base-url: https://localhost:8443/github
+  # username is the username of the github user
+  username: test-username
+  # owner is the owner name of the repository for issues.
+  owner: test-owner
+  # token is the token for github account.
+  token: test-token
+  # project-name is the name of the repository.
+  project-name: test-project
+  # issue-label is the label of the created issue type
+  issue-label: bug
+
+# gitlab contains configuration options for gitlab issue tracker
+gitlab:
+  # base-url is the optional self-hosted gitlab application url
+  base-url: https://localhost:8443/gitlab
+  # username is the username of the gitlab user
+  username: test-username
+  # token is the token for gitlab account.
+  token: test-token
+  # project-id is the ID of the repository.
+  project-id: 1234
+  # issue-label is the label of the created issue type
+  issue-label: bug
+
+# jira contains configuration options for jira issue tracker
+jira:
+  # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
+  cloud: true
+  # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created
+  update-existing: false
+  # URL is the jira application url
+  url: https://localhost/jira
+  # account-id is the account-id of the jira user or username in case of on-prem Jira
+  account-id: test-account-id
+  # email is the email of the user for jira instance
+  email: [email protected]
+  # token is the token for jira instance or password in case of on-prem Jira
+  token: test-token
+  # project-name is the name of the project.
+  project-name: test-project-name
+  # issue-type is the name of the created issue type
+  issue-type: bug
+
+# elasticsearch contains configuration options for elasticsearch exporter
+elasticsearch:
+  # IP for elasticsearch instance
+  ip: 127.0.0.1
+  # Port is the port of elasticsearch instance
+  port: 9200
+  # IndexName is the name of the elasticsearch index
+  index-name: nuclei
+  # SSL enables ssl for elasticsearch connection
+  ssl: false
+  # SSLVerification disables SSL verification for elasticsearch
+  ssl-verification: false
+  # Username for the elasticsearch instance
+  username: test
+  # Password is the password for elasticsearch instance
+  password: test

nuclei-jsonschema.json

@@ -0,0 +1,959 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$ref": "#/definitions/templates.Template",
+  "definitions": {
+    "model.Classification": {
+      "properties": {
+        "cve-id": {
+          "$ref": "#/definitions/stringslice.StringSlice",
+          "title": "cve ids for the template",
+          "description": "CVE IDs for the template"
+        },
+        "cwe-id": {
+          "$ref": "#/definitions/stringslice.StringSlice",
+          "title": "cwe ids for the template",
+          "description": "CWE IDs for the template"
+        },
+        "cvss-metrics": {
+          "type": "string",
+          "title": "cvss metrics for the template",
+          "description": "CVSS Metrics for the template",
+          "examples": [
+            "3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"
+          ]
+        },
+        "cvss-score": {
+          "type": "number",
+          "title": "cvss score for the template",
+          "description": "CVSS Score for the template"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "model.Info": {
+      "properties": {
+        "name": {
+          "type": "string",
+          "title": "name of the template",
+          "description": "Name is a short summary of what the template does",
+          "examples": [
+            "Nagios Default Credentials Check"
+          ]
+        },
+        "author": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/stringslice.StringSlice",
+          "title": "author of the template",
+          "description": "Author is the author of the template"
+        },
+        "tags": {
+          "$ref": "#/definitions/stringslice.StringSlice",
+          "title": "tags of the template",
+          "description": "Any tags for the template"
+        },
+        "description": {
+          "type": "string",
+          "title": "description of the template",
+          "description": "In-depth explanation on what the template does",
+          "examples": [
+            "Bower is a package manager which stores package information in the bower.json file"
+          ]
+        },
+        "reference": {
+          "$ref": "#/definitions/stringslice.StringSlice",
+          "title": "references for the template",
+          "description": "Links relevant to the template"
+        },
+        "severity": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/severity.Holder"
+        },
+        "metadata": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object",
+          "title": "additional metadata for the template",
+          "description": "Additional metadata fields for the template"
+        },
+        "classification": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/model.Classification",
+          "title": "classification info for the template",
+          "description": "Classification information for the template"
+        },
+        "remediation": {
+          "type": "string",
+          "title": "remediation steps for the template",
+          "description": "In-depth explanation on how to fix the issues found by the template",
+          "examples": [
+            "Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties"
+          ]
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "severity.Holder": {
+      "enum": [
+        "info",
+        "low",
+        "medium",
+        "high",
+        "critical"
+      ],
+      "type": "string",
+      "title": "severity of the template",
+      "description": "Seriousness of the implications of the template"
+    },
+    "stringslice.StringSlice": {
+      "oneOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "array"
+        }
+      ]
+    },
+    "extractors.Extractor": {
+      "required": [
+        "type"
+      ],
+      "properties": {
+        "name": {
+          "type": "string",
+          "title": "name of the extractor",
+          "description": "Name of the extractor"
+        },
+        "type": {
+          "enum": [
+            "regex",
+            "kval",
+            "json",
+            "xpath"
+          ],
+          "type": "string",
+          "title": "type of the extractor",
+          "description": "Type of the extractor"
+        },
+        "regex": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "regex to extract from part",
+          "description": "Regex to extract from part"
+        },
+        "group": {
+          "type": "integer",
+          "title": "group to extract from regex",
+          "description": "Group to extract from regex"
+        },
+        "kval": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "kval pairs to extract from response",
+          "description": "Kval pairs to extract from response"
+        },
+        "json": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "json jq expressions to extract data",
+          "description": "JSON JQ expressions to evaluate from response part"
+        },
+        "xpath": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "html xpath expressions to extract data",
+          "description": "XPath allows using xpath expressions to extract items from html response"
+        },
+        "attribute": {
+          "type": "string",
+          "title": "optional attribute to extract from xpath",
+          "description": "Optional attribute to extract from response XPath"
+        },
+        "part": {
+          "type": "string",
+          "title": "part of response to extract data from",
+          "description": "Part of the request response to extract data from"
+        },
+        "internal": {
+          "type": "boolean",
+          "title": "mark extracted value for internal variable use",
+          "description": "Internal when set to true will allow using the value extracted in the next request for some protocols"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "matchers.Matcher": {
+      "required": [
+        "type"
+      ],
+      "properties": {
+        "type": {
+          "enum": [
+            "status",
+            "size",
+            "word",
+            "regex",
+            "binary",
+            "dsl"
+          ],
+          "type": "string",
+          "title": "type of matcher",
+          "description": "Type of the matcher"
+        },
+        "condition": {
+          "enum": [
+            "and",
+            "or"
+          ],
+          "type": "string",
+          "title": "condition between matcher variables",
+          "description": "Condition between the matcher variables"
+        },
+        "part": {
+          "type": "string",
+          "title": "part of response to match",
+          "description": "Part of response to match data from"
+        },
+        "negative": {
+          "type": "boolean",
+          "title": "negative specifies if match reversed",
+          "description": "Negative specifies if the match should be reversed. It will only match if the condition is not true"
+        },
+        "name": {
+          "type": "string",
+          "title": "name of the matcher",
+          "description": "Name of the matcher"
+        },
+        "status": {
+          "items": {
+            "type": "integer"
+          },
+          "type": "array",
+          "title": "status to match",
+          "description": "Status to match for the response"
+        },
+        "size": {
+          "items": {
+            "type": "integer"
+          },
+          "type": "array",
+          "title": "acceptable size for response",
+          "description": "Size is the acceptable size for the response"
+        },
+        "words": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "words to match in response",
+          "description": " Words contains word patterns required to be present in the response part"
+        },
+        "regex": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "regex to match in response",
+          "description": "Regex contains regex patterns required to be present in the response part"
+        },
+        "binary": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "binary patterns to match in response",
+          "description": "Binary are the binary patterns required to be present in the response part"
+        },
+        "dsl": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "dsl expressions to match in response",
+          "description": "DSL are the dsl expressions that will be evaluated as part of nuclei matching rules"
+        },
+        "encoding": {
+          "enum": [
+            "hex"
+          ],
+          "type": "string",
+          "title": "encoding for word field",
+          "description": "Optional encoding for the word fields"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "dns.Request": {
+      "properties": {
+        "matchers": {
+          "items": {
+            "$ref": "#/definitions/matchers.Matcher"
+          },
+          "type": "array",
+          "title": "matchers to run on response",
+          "description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
+        },
+        "extractors": {
+          "items": {
+            "$ref": "#/definitions/extractors.Extractor"
+          },
+          "type": "array",
+          "title": "extractors to run on response",
+          "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
+        },
+        "matchers-condition": {
+          "enum": [
+            "and",
+            "or"
+          ],
+          "type": "string",
+          "title": "condition between the matchers",
+          "description": "Conditions between the matchers"
+        },
+        "id": {
+          "type": "string",
+          "title": "id of the dns request",
+          "description": "ID is the optional ID of the DNS Request"
+        },
+        "name": {
+          "type": "string",
+          "title": "hostname to make dns request for",
+          "description": "Name is the Hostname to make DNS request for"
+        },
+        "type": {
+          "enum": [
+            "A",
+            "NS",
+            "DS",
+            "CNAME",
+            "SOA",
+            "PTR",
+            "MX",
+            "TXT",
+            "AAAA"
+          ],
+          "type": "string",
+          "title": "type of dns request to make",
+          "description": "Type is the type of DNS request to make"
+        },
+        "class": {
+          "enum": [
+            "inet",
+            "csnet",
+            "chaos",
+            "hesiod",
+            "none",
+            "any"
+          ],
+          "type": "string",
+          "title": "class of DNS request",
+          "description": "Class is the class of the DNS request"
+        },
+        "retries": {
+          "type": "integer",
+          "title": "retries for dns request",
+          "description": "Retries is the number of retries for the DNS request"
+        },
+        "recursion": {
+          "type": "boolean",
+          "title": "recurse all servers",
+          "description": "Recursion determines if resolver should recurse all records to get fresh results"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "file.Request": {
+      "properties": {
+        "matchers": {
+          "items": {
+            "$ref": "#/definitions/matchers.Matcher"
+          },
+          "type": "array",
+          "title": "matchers to run on response",
+          "description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
+        },
+        "extractors": {
+          "items": {
+            "$ref": "#/definitions/extractors.Extractor"
+          },
+          "type": "array",
+          "title": "extractors to run on response",
+          "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
+        },
+        "matchers-condition": {
+          "enum": [
+            "and",
+            "or"
+          ],
+          "type": "string",
+          "title": "condition between the matchers",
+          "description": "Conditions between the matchers"
+        },
+        "extensions": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "extensions to match",
+          "description": "List of extensions to perform matching on"
+        },
+        "denylist": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "extensions to deny match",
+          "description": "List of file extensions to deny during matching"
+        },
+        "id": {
+          "type": "string",
+          "title": "id of the request",
+          "description": "ID is the optional ID for the request"
+        },
+        "max-size": {
+          "type": "integer",
+          "title": "max size data to run request on",
+          "description": "Maximum size of the file to run request on"
+        },
+        "no-recursive": {
+          "type": "boolean",
+          "title": "do not perform recursion",
+          "description": "Specifies whether to not do recursive checks if folders are provided"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "headless.Request": {
+      "properties": {
+        "id": {
+          "type": "string",
+          "title": "id of the request",
+          "description": "Optional ID of the headless request"
+        },
+        "steps": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/engine.Action"
+          },
+          "type": "array",
+          "title": "list of actions for headless request",
+          "description": "List of actions to run for headless request"
+        },
+        "matchers": {
+          "items": {
+            "$ref": "#/definitions/matchers.Matcher"
+          },
+          "type": "array",
+          "title": "matchers to run on response",
+          "description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
+        },
+        "extractors": {
+          "items": {
+            "$ref": "#/definitions/extractors.Extractor"
+          },
+          "type": "array",
+          "title": "extractors to run on response",
+          "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
+        },
+        "matchers-condition": {
+          "enum": [
+            "and",
+            "or"
+          ],
+          "type": "string",
+          "title": "condition between the matchers",
+          "description": "Conditions between the matchers"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "engine.Action": {
+      "required": [
+        "action"
+      ],
+      "properties": {
+        "args": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object",
+          "title": "arguments for headless action",
+          "description": "Args contain arguments for the headless action"
+        },
+        "name": {
+          "type": "string",
+          "title": "name for headless action",
+          "description": "Name is the name assigned to the headless action"
+        },
+        "description": {
+          "type": "string",
+          "title": "description for headless action",
+          "description": "Description of the headless action"
+        },
+        "action": {
+          "enum": [
+            "navigate",
+            "script",
+            "click",
+            "rightclick",
+            "text",
+            "screenshot",
+            "time",
+            "select",
+            "files",
+            "waitload",
+            "getresource",
+            "extract",
+            "setmethod",
+            "addheader",
+            "setheader",
+            "deleteheader",
+            "setbody",
+            "waitevent",
+            "keyboard",
+            "debug",
+            "sleep"
+          ],
+          "type": "string",
+          "title": "action to perform",
+          "description": "Type of actions to perform"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "http.Request": {
+      "properties": {
+        "matchers": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/matchers.Matcher"
+          },
+          "type": "array",
+          "title": "matchers to run on response",
+          "description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
+        },
+        "extractors": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/extractors.Extractor"
+          },
+          "type": "array",
+          "title": "extractors to run on response",
+          "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
+        },
+        "matchers-condition": {
+          "enum": [
+            "and",
+            "or"
+          ],
+          "type": "string",
+          "title": "condition between the matchers",
+          "description": "Conditions between the matchers"
+        },
+        "path": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "path(s) for the http request",
+          "description": "Path(s) to send http requests to"
+        },
+        "raw": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "description": "HTTP Requests in Raw Format"
+        },
+        "id": {
+          "type": "string",
+          "title": "id for the http request",
+          "description": "ID for the HTTP Request"
+        },
+        "name": {
+          "type": "string",
+          "title": "name for the http request",
+          "description": "Optional name for the HTTP Request"
+        },
+        "attack": {
+          "enum": [
+            "sniper",
+            "pitchfork",
+            "clusterbomb"
+          ],
+          "type": "string",
+          "title": "attack is the payload combination",
+          "description": "Attack is the type of payload combinations to perform"
+        },
+        "method": {
+          "enum": [
+            "GET",
+            "HEAD",
+            "POST",
+            "PUT",
+            "DELETE",
+            "CONNECT",
+            "OPTIONS",
+            "TRACE",
+            "PATCH",
+            "PURGE"
+          ],
+          "type": "string",
+          "title": "method is the http request method",
+          "description": "Method is the HTTP Request Method"
+        },
+        "body": {
+          "type": "string",
+          "title": "body is the http request body",
+          "description": "Body is an optional parameter which contains HTTP Request body"
+        },
+        "payloads": {
+          "patternProperties": {
+            ".*": {
+              "additionalProperties": true
+            }
+          },
+          "type": "object",
+          "title": "payloads for the http request",
+          "description": "Payloads contains any payloads for the current request"
+        },
+        "headers": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object",
+          "title": "headers to send with the http request",
+          "description": "Headers contains HTTP Headers to send with the request"
+        },
+        "race_count": {
+          "type": "integer",
+          "title": "number of times to repeat request in race condition",
+          "description": "Number of times to send a request in Race Condition Attack"
+        },
+        "max-redirects": {
+          "type": "integer",
+          "title": "maximum number of redirects to follow",
+          "description": "Maximum number of redirects that should be followed"
+        },
+        "pipeline-concurrent-connections": {
+          "type": "integer",
+          "title": "number of pipelining connections",
+          "description": "Number of connections to create during pipelining"
+        },
+        "pipeline-requests-per-connection": {
+          "type": "integer",
+          "title": "number of requests to send per pipelining connections",
+          "description": "Number of requests to send per connection when pipelining"
+        },
+        "threads": {
+          "type": "integer",
+          "title": "threads for sending requests",
+          "description": "Threads specifies number of threads to use sending requests. This enables Connection Pooling"
+        },
+        "max-size": {
+          "type": "integer",
+          "title": "maximum http response body size",
+          "description": "Maximum size of http response body to read in bytes"
+        },
+        "cookie-reuse": {
+          "type": "boolean",
+          "title": "optional cookie reuse enable",
+          "description": "Optional setting that enables cookie reuse"
+        },
+        "redirects": {
+          "type": "boolean",
+          "title": "follow http redirects",
+          "description": "Specifies whether redirects should be followed by the HTTP Client"
+        },
+        "pipeline": {
+          "type": "boolean",
+          "title": "perform HTTP 1.1 pipelining",
+          "description": "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"
+        },
+        "unsafe": {
+          "type": "boolean",
+          "title": "use rawhttp non-strict-rfc client",
+          "description": "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests"
+        },
+        "race": {
+          "type": "boolean",
+          "title": "perform race-http request coordination attack",
+          "description": "Race determines if all the request have to be attempted at the same time (Race Condition)"
+        },
+        "req-condition": {
+          "type": "boolean",
+          "title": "preserve request history",
+          "description": "Automatically assigns numbers to requests and preserves their history"
+        },
+        "stop-at-first-match": {
+          "type": "boolean",
+          "title": "stop at first match",
+          "description": "Stop the execution after a match is found"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "network.Input": {
+      "properties": {
+        "data": {
+          "type": "string",
+          "title": "data to send as input",
+          "description": "Data is the data to send as the input"
+        },
+        "type": {
+          "enum": [
+            "hex",
+            "text"
+          ],
+          "type": "string",
+          "title": "type is the type of input data",
+          "description": "Type of input specified in data field"
+        },
+        "read": {
+          "type": "integer",
+          "title": "bytes to read from socket",
+          "description": "Number of bytes to read from socket"
+        },
+        "name": {
+          "type": "string",
+          "title": "optional name for data read",
+          "description": "Optional name of the data read to provide matching on"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "network.Request": {
+      "properties": {
+        "id": {
+          "type": "string",
+          "title": "id of the request",
+          "description": "ID of the network request"
+        },
+        "host": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "host to send requests to",
+          "description": "Host to send network requests to"
+        },
+        "attack": {
+          "enum": [
+            "sniper",
+            "pitchfork",
+            "clusterbomb"
+          ],
+          "type": "string",
+          "title": "attack is the payload combination",
+          "description": "Attack is the type of payload combinations to perform"
+        },
+        "payloads": {
+          "patternProperties": {
+            ".*": {
+              "additionalProperties": true
+            }
+          },
+          "type": "object",
+          "title": "payloads for the network request",
+          "description": "Payloads contains any payloads for the current request"
+        },
+        "inputs": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/network.Input"
+          },
+          "type": "array",
+          "title": "inputs for the network request",
+          "description": "Inputs contains any input/output for the current request"
+        },
+        "read-size": {
+          "type": "integer",
+          "title": "size of network response to read",
+          "description": "Size of response to read at the end. Default is 1024 bytes"
+        },
+        "matchers": {
+          "items": {
+            "$ref": "#/definitions/matchers.Matcher"
+          },
+          "type": "array",
+          "title": "matchers to run on response",
+          "description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
+        },
+        "extractors": {
+          "items": {
+            "$ref": "#/definitions/extractors.Extractor"
+          },
+          "type": "array",
+          "title": "extractors to run on response",
+          "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
+        },
+        "matchers-condition": {
+          "enum": [
+            "and",
+            "or"
+          ],
+          "type": "string",
+          "title": "condition between the matchers",
+          "description": "Conditions between the matchers"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "templates.Template": {
+      "required": [
+        "id",
+        "info"
+      ],
+      "properties": {
+        "id": {
+          "type": "string",
+          "title": "id of the template",
+          "description": "The Unique ID for the template",
+          "examples": [
+            "cve-2021-19520"
+          ]
+        },
+        "info": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/model.Info",
+          "title": "info for the template",
+          "description": "Info contains metadata for the template"
+        },
+        "requests": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/http.Request"
+          },
+          "type": "array",
+          "title": "http requests to make",
+          "description": "HTTP requests to make for the template"
+        },
+        "dns": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/dns.Request"
+          },
+          "type": "array",
+          "title": "dns requests to make",
+          "description": "DNS requests to make for the template"
+        },
+        "file": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/file.Request"
+          },
+          "type": "array",
+          "title": "file requests to make",
+          "description": "File requests to make for the template"
+        },
+        "network": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/network.Request"
+          },
+          "type": "array",
+          "title": "network requests to make",
+          "description": "Network requests to make for the template"
+        },
+        "headless": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/headless.Request"
+          },
+          "type": "array",
+          "title": "headless requests to make",
+          "description": "Headless requests to make for the template"
+        },
+        "workflows": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/workflows.WorkflowTemplate"
+          },
+          "type": "array",
+          "title": "list of workflows to execute",
+          "description": "List of workflows to execute for template"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "workflows.Matcher": {
+      "properties": {
+        "name": {
+          "type": "string",
+          "title": "name of item to match",
+          "description": "Name of item to match"
+        },
+        "subtemplates": {
+          "items": {
+            "$ref": "#/definitions/workflows.WorkflowTemplate"
+          },
+          "type": "array",
+          "title": "templates to run after match",
+          "description": "Templates to run after match"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "workflows.WorkflowTemplate": {
+      "properties": {
+        "template": {
+          "type": "string",
+          "title": "template/directory to execute",
+          "description": "Template or directory to execute as part of workflow"
+        },
+        "tags": {
+          "$ref": "#/definitions/stringslice.StringSlice",
+          "title": "tags to execute",
+          "description": "Tags to run template based on"
+        },
+        "matchers": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/workflows.Matcher"
+          },
+          "type": "array",
+          "title": "name based template result matchers",
+          "description": "Matchers perform name based matching to run subtemplates for a workflow"
+        },
+        "subtemplates": {
+          "items": {
+            "$ref": "#/definitions/workflows.WorkflowTemplate"
+          },
+          "type": "array",
+          "title": "subtemplate based result matchers",
+          "description": "Subtemplates are ran if the template field Template matches"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    }
+  }
+}

v2/.goreleaser.yml

@@ -3,23 +3,33 @@ before:
     - go mod tidy
 
 builds:
-    - binary: nuclei
-      main: cmd/nuclei/main.go
-      goos:
-        - linux
-        - windows
-        - darwin
-      goarch:
-        - amd64
-        - 386
-        - arm
-        - arm64
-  
+- env:
+  - CGO_ENABLED=0
+  goos:
+    - windows
+    - linux
+    - darwin
+  goarch:
+    - amd64
+    - 386
+    - arm
+    - arm64
+
+  ignore:
+    - goos: darwin
+      goarch: '386'
+    - goos: windows
+      goarch: 'arm'
+    - goos: windows
+      goarch: 'arm64'
+
+  binary: '{{ .ProjectName }}'
+  main: cmd/nuclei/main.go
+
 archives:
-    - id: tgz
-      format: tar.gz
-      replacements:
-          darwin: macOS
-      format_overrides:
-          - goos: windows
-            format: zip
+- format: zip
+  replacements:
+      darwin: macOS
+
+checksum:
+  algorithm: sha256

v2/Makefile

@@ -0,0 +1,22 @@
+# Go parameters
+GOCMD=go
+GOBUILD=$(GOCMD) build
+GOMOD=$(GOCMD) mod
+GOTEST=$(GOCMD) test
+GOGET=$(GOCMD) get
+    
+all: build
+build:
+	$(GOBUILD) -v -ldflags="-extldflags=-static" -o "nuclei" cmd/nuclei/main.go
+docs:
+	if ! which dstdocgen > /dev/null; then
+		echo -e "Command not found! Install? (y/n) \c"
+		go get -v github.com/projectdiscovery/yamldoc-go/cmd/docgen/dstdocgen
+	fi
+	$(GOCMD) generate pkg/templates/templates.go
+	$(GOBUILD) -o "cmd/docgen/docgen" cmd/docgen/docgen.go
+	./cmd/docgen/docgen docs.md nuclei-jsonschema.json
+test:
+	$(GOTEST) -v ./...
+tidy:
+	$(GOMOD) tidy

v2/cmd/cve-annotate/main.go

@@ -0,0 +1,162 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/Ice3man543/nvd"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+)
+
+var (
+	input       = flag.String("i", "", "Templates to annotate")
+	templateDir = flag.String("d", "", "Custom template directory for update")
+)
+
+func main() {
+	flag.Parse()
+
+	if *input == "" || *templateDir == "" {
+		log.Fatalf("invalid input, see -h\n")
+	}
+
+	if err := process(); err != nil {
+		log.Fatalf("could not process: %s\n", err)
+	}
+}
+
+func process() error {
+	tempDir, err := ioutil.TempDir("", "nuclei-nvd-%s")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tempDir)
+
+	client, err := nvd.NewClient(tempDir)
+	if err != nil {
+		return err
+	}
+	catalog := catalog.New(*templateDir)
+
+	paths, err := catalog.GetTemplatePath(*input)
+	if err != nil {
+		return err
+	}
+	for _, path := range paths {
+		data, err := ioutil.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		getCVEData(client, path, string(data))
+	}
+	return nil
+}
+
+var (
+	idRegex       = regexp.MustCompile("id: ([C|c][V|v][E|e]-[0-9]+-[0-9]+)")
+	severityRegex = regexp.MustCompile(`severity: ([a-z]+)`)
+)
+
+func getCVEData(client *nvd.Client, filePath, data string) {
+	matches := idRegex.FindAllStringSubmatch(data, 1)
+	if len(matches) == 0 {
+		return
+	}
+	cveName := matches[0][1]
+
+	severityMatches := severityRegex.FindAllStringSubmatch(data, 1)
+	if len(matches) == 0 {
+		return
+	}
+	severityValue := severityMatches[0][1]
+
+	// Skip if there's classification data already
+	if strings.Contains(data, "classification:") {
+		return
+	}
+	cveItem, err := client.FetchCVE(cveName)
+	if err != nil {
+		log.Printf("Could not fetch cve %s: %s\n", cveName, err)
+		return
+	}
+	var cweID []string
+	for _, problemData := range cveItem.CVE.Problemtype.ProblemtypeData {
+		for _, description := range problemData.Description {
+			cweID = append(cweID, description.Value)
+		}
+	}
+	cvssScore := cveItem.Impact.BaseMetricV3.CvssV3.BaseScore
+	cvssMetrics := cveItem.Impact.BaseMetricV3.CvssV3.VectorString
+
+	// Perform some hacky string replacement to place the metadata in templates
+	infoBlockIndexData := data[strings.Index(data, "info:"):]
+	requestsIndex := strings.Index(infoBlockIndexData, "requests:")
+	networkIndex := strings.Index(infoBlockIndexData, "network:")
+	if requestsIndex == -1 && networkIndex == -1 {
+		return
+	}
+	if networkIndex != -1 {
+		requestsIndex = networkIndex
+	}
+	infoBlockData := infoBlockIndexData[:requestsIndex]
+	infoBlockClean := strings.TrimRight(infoBlockData, "\n")
+
+	newInfoBlock := infoBlockClean
+	var changed bool
+
+	if newSeverity := isSeverityMatchingCvssScore(severityValue, cvssScore); newSeverity != "" {
+		changed = true
+		newInfoBlock = strings.ReplaceAll(newInfoBlock, severityMatches[0][0], "severity: "+newSeverity)
+		fmt.Printf("Adjusting severity for %s from %s=>%s (%.2f)\n", filePath, severityValue, newSeverity, cvssScore)
+	}
+	if !strings.Contains(infoBlockClean, "classification") && (cvssScore != 0 && cvssMetrics != "") {
+		changed = true
+		newInfoBlock = newInfoBlock + fmt.Sprintf("\n  classification:\n    cvss-metrics: %s\n    cvss-score: %.2f\n    cve-id: %s", cvssMetrics, cvssScore, cveName)
+		if len(cweID) > 0 && (cweID[0] != "NVD-CWE-Other" && cweID[0] != "NVD-CWE-noinfo") {
+			newInfoBlock = newInfoBlock + fmt.Sprintf("\n    cwe-id: %s", strings.Join(cweID, ","))
+		}
+	}
+	// If there is no description field, fill the description from CVE information
+	if !strings.Contains(infoBlockClean, "description:") && len(cveItem.CVE.Description.DescriptionData) > 0 {
+		changed = true
+		newInfoBlock = newInfoBlock + fmt.Sprintf("\n  description: %s", fmt.Sprintf("%q", cveItem.CVE.Description.DescriptionData[0].Value))
+	}
+	if !strings.Contains(infoBlockClean, "reference:") && len(cveItem.CVE.References.ReferenceData) > 0 {
+		changed = true
+		newInfoBlock = newInfoBlock + "\n  reference:"
+		for _, reference := range cveItem.CVE.References.ReferenceData {
+			newInfoBlock = newInfoBlock + fmt.Sprintf("\n    - %s", reference.URL)
+		}
+	}
+	newTemplate := strings.ReplaceAll(data, infoBlockClean, newInfoBlock)
+	if changed {
+		_ = ioutil.WriteFile(filePath, []byte(newTemplate), 0777)
+		fmt.Printf("Wrote updated template to %s\n", filePath)
+	}
+}
+
+func isSeverityMatchingCvssScore(severity string, score float64) string {
+	if score == 0.0 {
+		return ""
+	}
+	var expected string
+
+	if score >= 0.1 && score <= 3.9 {
+		expected = "low"
+	} else if score >= 4.0 && score <= 6.9 {
+		expected = "medium"
+	} else if score >= 7.0 && score <= 8.9 {
+		expected = "high"
+	} else if score >= 9.0 && score <= 10.0 {
+		expected = "critical"
+	}
+	if expected != "" && expected != severity {
+		return expected
+	}
+	return ""
+}

v2/cmd/docgen/docgen.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/alecthomas/jsonschema"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+)
+
+var pathRegex = regexp.MustCompile(`github.com/projectdiscovery/nuclei/v2/(?:internal|pkg)/(?:.*/)?([A-Za-z\.]+)`)
+
+func main() {
+	// Generate yaml syntax documentation
+	data, err := templates.GetTemplateDoc().Encode()
+	if err != nil {
+		log.Fatalf("Could not encode docs: %s\n", err)
+	}
+	err = ioutil.WriteFile(os.Args[1], data, 0777)
+	if err != nil {
+		log.Fatalf("Could not write docs: %s\n", err)
+	}
+
+	// Generate jsonschema
+	r := &jsonschema.Reflector{
+		PreferYAMLSchema:      true,
+		YAMLEmbeddedStructs:   true,
+		FullyQualifyTypeNames: true,
+	}
+	jsonschemaData := r.Reflect(&templates.Template{})
+
+	var buf bytes.Buffer
+	encoder := json.NewEncoder(&buf)
+	encoder.SetIndent("", "  ")
+	_ = encoder.Encode(jsonschemaData)
+
+	schema := buf.String()
+	for _, match := range pathRegex.FindAllStringSubmatch(schema, -1) {
+		schema = strings.ReplaceAll(schema, match[0], match[1])
+	}
+	err = ioutil.WriteFile(os.Args[2], []byte(schema), 0777)
+	if err != nil {
+		log.Fatalf("Could not write jsonschema: %s\n", err)
+	}
+}

v2/cmd/functional-test/main.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/pkg/errors"
+
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var (
+	success = aurora.Green("[✓]").String()
+	failed  = aurora.Red("[✘]").String()
+	errored = false
+
+	mainNucleiBinary = flag.String("main", "", "Main Branch Nuclei Binary")
+	devNucleiBinary  = flag.String("dev", "", "Dev Branch Nuclei Binary")
+	testcases        = flag.String("testcases", "", "Test cases file for nuclei functional tests")
+)
+
+func main() {
+	flag.Parse()
+
+	if err := runFunctionalTests(); err != nil {
+		log.Fatalf("Could not run functional tests: %s\n", err)
+	}
+	if errored {
+		os.Exit(1)
+	}
+}
+
+func runFunctionalTests() error {
+	file, err := os.Open(*testcases)
+	if err != nil {
+		return errors.Wrap(err, "could not open test cases")
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := strings.TrimSpace(scanner.Text())
+		if text == "" {
+			continue
+		}
+		if err := runIndividualTestCase(text); err != nil {
+			errored = true
+			fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, text, err)
+		} else {
+			fmt.Printf("%s Test \"%s\" passed!\n", success, text)
+		}
+	}
+	return nil
+}
+
+func runIndividualTestCase(testcase string) error {
+	parts := strings.Fields(testcase)
+
+	var finalArgs []string
+	if len(parts) > 1 {
+		finalArgs = parts[1:]
+	}
+	mainOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*mainNucleiBinary, finalArgs)
+	if err != nil {
+		return errors.Wrap(err, "could not run nuclei main test")
+	}
+	devOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*devNucleiBinary, finalArgs)
+	if err != nil {
+		return errors.Wrap(err, "could not run nuclei dev test")
+	}
+	if mainOutput == devOutput {
+		return nil
+	}
+	return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput)
+}

v2/cmd/functional-test/run.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo 'Building functional-test binary'
+go build
+
+echo 'Building Nuclei binary from current branch'
+go build -o nuclei_dev ../nuclei
+
+echo 'Installing latest release of nuclei'
+GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+
+echo 'Starting Nuclei functional test'
+./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt

v2/cmd/functional-test/testcases.txt

@@ -0,0 +1,51 @@
+{{binary}}
+{{binary}} -tags cve
+{{binary}} -tags cve,exposure
+{{binary}} -tags cve,exposure -tags token
+{{binary}} -tags cve,exposure -tags token,logs
+{{binary}} -tags "cve","exposure" -tags "token","logs"
+{{binary}} -tags 'cve','exposure' -tags 'token','logs'
+{{binary}} -tags cve -severity high
+{{binary}} -tags cve,exposure -severity high,critical
+{{binary}} -tags cve,exposure -severity high,critical,medium
+{{binary}} -tags cve -author geeknik
+{{binary}} -tags cve -author geeknik,pdteam
+{{binary}} -tags cve -author geeknik -severity high
+{{binary}} -tags cve
+{{binary}} -tags cve,exposure
+{{binary}} -tags cve,exposure -tags token
+{{binary}} -tags cve,exposure -tags token,logs
+{{binary}} -tags "cve","exposure" -tags "token","logs"
+{{binary}} -tags 'cve','exposure' -tags 'token','logs'
+{{binary}} -tags cve -severity high
+{{binary}} -tags cve,exposure -severity high,critical
+{{binary}} -tags cve,exposure -severity high,critical,medium
+{{binary}} -tags cve -author geeknik
+{{binary}} -tags cve -author geeknik,pdteam
+{{binary}} -tags cve -author geeknik -severity high
+{{binary}} -tags cve,exposure -author geeknik,pdteam -severity high,critical
+{{binary}} -tags "cve,exposure" -author "geeknik,pdteam" -severity high,critical
+{{binary}} -tags cve -etags ssrf
+{{binary}} -tags cve,exposure -etags ssrf,config
+{{binary}} -tags cve,exposure -etags ssrf,config -severity high
+{{binary}} -tags cve,exposure -etags ssrf,config -severity high -author geeknik
+{{binary}} -tags cve,dos,fuzz
+{{binary}} -tags cve -include-tags dos,fuzz
+{{binary}} -tags cve -exclude-tags cve2020
+{{binary}} -tags cve -exclude-templates cves/2020/
+{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml
+{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml -exclude-templates cves/2021/
+{{binary}} -t cves/
+{{binary}} -t cves/ -t exposures/
+{{binary}} -t cves/ -t exposures/ -tags config
+{{binary}} -t cves/ -t exposures/ -tags config,ssrf
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2021/
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2017/CVE-2017-7269.yaml
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -include-templates cves/2017/CVE-2017-7269.yaml
+{{binary}} -w workflows
+{{binary}} -w workflows -author geeknik,pdteam
+{{binary}} -w workflows -severity high,critical
+{{binary}} -w workflows -author geeknik,pdteam  -severity high,critical

v2/cmd/integration-test/dns.go

@@ -10,7 +10,7 @@ var dnsTestCases = map[string]testutils.TestCase{
 
 type dnsBasic struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *dnsBasic) Execute(filePath string) error {
 	var routerErr error
 

v2/cmd/integration-test/http.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/julienschmidt/httprouter"
+
 	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
 )
 
@@ -43,7 +44,7 @@ func httpDebugRequestDump(r *http.Request) {
 
 type httpGetHeaders struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpGetHeaders) Execute(filePath string) error {
 	router := httprouter.New()
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
@@ -67,7 +68,7 @@ func (h *httpGetHeaders) Execute(filePath string) error {
 
 type httpGetQueryString struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpGetQueryString) Execute(filePath string) error {
 	router := httprouter.New()
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
@@ -91,7 +92,7 @@ func (h *httpGetQueryString) Execute(filePath string) error {
 
 type httpGetRedirects struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpGetRedirects) Execute(filePath string) error {
 	router := httprouter.New()
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
@@ -117,7 +118,7 @@ func (h *httpGetRedirects) Execute(filePath string) error {
 
 type httpGet struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpGet) Execute(filePath string) error {
 	router := httprouter.New()
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
@@ -139,7 +140,7 @@ func (h *httpGet) Execute(filePath string) error {
 
 type httpPostBody struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpPostBody) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -172,7 +173,7 @@ func (h *httpPostBody) Execute(filePath string) error {
 
 type httpPostJSONBody struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpPostJSONBody) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -211,7 +212,7 @@ func (h *httpPostJSONBody) Execute(filePath string) error {
 
 type httpPostMultipartBody struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpPostMultipartBody) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -254,7 +255,7 @@ func (h *httpPostMultipartBody) Execute(filePath string) error {
 
 type httpRawDynamicExtractor struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRawDynamicExtractor) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -293,7 +294,7 @@ func (h *httpRawDynamicExtractor) Execute(filePath string) error {
 
 type httpRawGetQuery struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRawGetQuery) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -322,7 +323,7 @@ func (h *httpRawGetQuery) Execute(filePath string) error {
 
 type httpRawGet struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRawGet) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -350,7 +351,7 @@ func (h *httpRawGet) Execute(filePath string) error {
 
 type httpRawPayload struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRawPayload) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -386,7 +387,7 @@ func (h *httpRawPayload) Execute(filePath string) error {
 
 type httpRawPostBody struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRawPostBody) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -419,7 +420,7 @@ func (h *httpRawPostBody) Execute(filePath string) error {
 
 type httpRawCookieReuse struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRawCookieReuse) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -468,14 +469,13 @@ func (h *httpRawCookieReuse) Execute(filePath string) error {
 
 type httpRawUnsafeRequest struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRawUnsafeRequest) Execute(filePath string) error {
 	var routerErr error
 
 	ts := testutils.NewTCPServer(func(conn net.Conn) {
 		defer conn.Close()
-
-		_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 40\r\nContent-Type: text/plain; charset=utf-8\r\nDate: Thu, 25 Feb 2021 17:17:28 GMT\r\n\r\nThis is test-raw-unsafe request matcher.\r\n"))
+		_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 36\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nThis is test raw-unsafe-matcher test"))
 	})
 	defer ts.Close()
 
@@ -494,7 +494,7 @@ func (h *httpRawUnsafeRequest) Execute(filePath string) error {
 
 type httpRequestCondition struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRequestCondition) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error

v2/cmd/integration-test/integration-test.go

@@ -13,6 +13,8 @@ var (
 	debug      = os.Getenv("DEBUG") == "true"
 	customTest = os.Getenv("TEST")
 	protocol   = os.Getenv("PROTO")
+
+	errored = false
 )
 
 func main() {
@@ -33,16 +35,18 @@ func main() {
 				if customTest != "" && !strings.Contains(file, customTest) {
 					continue // only run tests user asked
 				}
-				err := test.Execute(file)
-				if err != nil {
+				if err := test.Execute(file); err != nil {
 					fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, file, err)
-					os.Exit(1)
+					errored = true
 				} else {
 					fmt.Printf("%s Test \"%s\" passed!\n", success, file)
 				}
 			}
 		}
 	}
+	if errored {
+		os.Exit(1)
+	}
 }
 
 func errIncorrectResultsCount(results []string) error {

v2/cmd/integration-test/network.go

@@ -14,7 +14,7 @@ var networkTestcases = map[string]testutils.TestCase{
 
 type networkBasic struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *networkBasic) Execute(filePath string) error {
 	var routerErr error
 
@@ -47,7 +47,7 @@ func (h *networkBasic) Execute(filePath string) error {
 
 type networkMultiStep struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *networkMultiStep) Execute(filePath string) error {
 	var routerErr error
 

v2/cmd/integration-test/workflow.go

@@ -6,6 +6,7 @@ import (
 	"net/http/httptest"
 
 	"github.com/julienschmidt/httprouter"
+
 	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
 )
 
@@ -18,13 +19,13 @@ var workflowTestcases = map[string]testutils.TestCase{
 
 type workflowBasic struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *workflowBasic) Execute(filePath string) error {
 	router := httprouter.New()
-	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		httpDebugRequestDump(r)
 		fmt.Fprintf(w, "This is test matcher text")
-	}))
+	})
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
@@ -40,13 +41,13 @@ func (h *workflowBasic) Execute(filePath string) error {
 
 type workflowConditionMatched struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *workflowConditionMatched) Execute(filePath string) error {
 	router := httprouter.New()
-	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		httpDebugRequestDump(r)
 		fmt.Fprintf(w, "This is test matcher text")
-	}))
+	})
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
@@ -62,13 +63,13 @@ func (h *workflowConditionMatched) Execute(filePath string) error {
 
 type workflowConditionUnmatch struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *workflowConditionUnmatch) Execute(filePath string) error {
 	router := httprouter.New()
-	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		httpDebugRequestDump(r)
 		fmt.Fprintf(w, "This is test matcher text")
-	}))
+	})
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
@@ -84,13 +85,13 @@ func (h *workflowConditionUnmatch) Execute(filePath string) error {
 
 type workflowMatcherName struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *workflowMatcherName) Execute(filePath string) error {
 	router := httprouter.New()
-	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		httpDebugRequestDump(r)
 		fmt.Fprintf(w, "This is test matcher text")
-	}))
+	})
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 

v2/cmd/nuclei/issue-tracker-config.yaml

@@ -1,7 +1,9 @@
+# to specify which severities should be reported
 #allow-list:
-#  severity: high,critical
+#  severity: critical, high
+# to specify which severities should be excluded from reporting
 #deny-list:
-#  severity: low
+#  severity: info, low, medium
 
 # github contains configuration options for github issue tracker
 #github: 
@@ -32,16 +34,37 @@
 #  issue-label: ""
 
 # jira contains configuration options for jira issue tracker
-#jira: 
+#jira:
+#  # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
+#  cloud: true
+#  # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created
+#  update-existing: false
 #  # URL is the jira application url
 #  url: ""
-#  # account-id is the account-id of the jira user
+#  # account-id is the account-id of the jira user or username in case of on-prem Jira
 #  account-id: ""
 #  # email is the email of the user for jira instance
 #  email: ""
-#  # token is the token for jira instance.
+#  # token is the token for jira instance or password in case of on-prem Jira
 #  token: ""
 #  # project-name is the name of the project.
 #  project-name: ""
 #  # issue-type is the name of the created issue type
 #  issue-type: ""
+
+# elasticsearch contains configuration options for elasticsearch exporter
+#elasticsearch:
+#  # IP for elasticsearch instance
+#  ip: 127.0.0.1
+#  # Port is the port of elasticsearch instance
+#  port: 9200
+#  # IndexName is the name of the elasticsearch index
+#  index-name: nuclei
+#  # SSL enables ssl for elasticsearch connection
+#  ssl: false
+#  # SSLVerification disables SSL verification for elasticsearch
+#  ssl-verification: false
+#  # Username for the elasticsearch instance
+#  username: test
+#  # Password is the password for elasticsearch instance
+#  password: test

v2/cmd/nuclei/main.go

@@ -1,12 +1,14 @@
 package main
 
 import (
+	"fmt"
 	"os"
-	"path"
+	"path/filepath"
 
 	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/internal/runner"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
@@ -24,73 +26,165 @@ func main() {
 	if err != nil {
 		gologger.Fatal().Msgf("Could not create runner: %s\n", err)
 	}
-	nucleiRunner.RunEnumeration()
+	if nucleiRunner == nil {
+		return
+	}
+	if err := nucleiRunner.RunEnumeration(); err != nil {
+		gologger.Fatal().Msgf("Could not run nuclei: %s\n", err)
+	}
+	nucleiRunner.Close()
 }
 
 func readConfig() {
 	home, _ := os.UserHomeDir()
-	templatesDirectory := path.Join(home, "nuclei-templates")
+	templatesDirectory := filepath.Join(home, "nuclei-templates")
 
-	set := goflags.New()
-	set.SetDescription(`Nuclei is a fast tool for configurable targeted scanning 
-based on templates offering massive extensibility and ease of use.`)
-	set.StringVar(&cfgFile, "config", "", "Nuclei configuration file")
-	set.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port")
-	set.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on")
-	set.StringVarP(&options.Target, "target", "u", "", "URL to scan with nuclei")
-	set.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "Templates to run, supports single and multiple templates using directory.")
-	set.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "Workflows to run for nuclei")
-	set.StringSliceVarP(&options.ExcludedTemplates, "exclude", "et", []string{}, "Templates to exclude, supports single and multiple templates using directory.")
-	set.StringSliceVarP(&options.Severity, "severity", "impact", []string{}, "Templates to run based on severity, supports single and multiple severity.")
-	set.StringVarP(&options.Targets, "list", "l", "", "List of URLs to run templates on")
-	set.StringVarP(&options.Output, "output", "o", "", "File to write output to (optional)")
-	set.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server")
-	set.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the proxy socks server")
-	set.BoolVar(&options.Silent, "silent", false, "Show only results in output")
-	set.BoolVar(&options.Version, "version", false, "Show version of nuclei")
-	set.BoolVarP(&options.Verbose, "verbose", "v", false, "Show verbose output")
-	set.BoolVarP(&options.NoColor, "no-color", "nc", false, "Disable colors in output")
-	set.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout")
-	set.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
-	set.BoolVarP(&options.RandomAgent, "random-agent", "ra", false, "Use randomly selected HTTP User-Agent header value")
-	set.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "Custom Header.")
-	set.BoolVar(&options.Debug, "debug", false, "Debugging request and responses")
-	set.BoolVar(&options.DebugRequests, "debug-req", false, "Debugging request")
-	set.BoolVar(&options.DebugResponse, "debug-resp", false, "Debugging response")
-	set.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "Download / updates nuclei community templates")
-	set.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log")
-	set.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "Directory storing nuclei-templates")
-	set.BoolVar(&options.JSON, "json", false, "Write json output to files")
-	set.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "Write requests/responses for matches in JSON output")
-	set.BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan")
-	set.BoolVar(&options.TemplateList, "tl", false, "List available templates")
-	set.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "Maximum requests to send per second")
-	set.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "Stop processing http requests at first match (this may break template/workflow logic)")
-	set.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "Maximum Number of hosts analyzed in parallel per template")
-	set.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "Maximum Number of templates executed in parallel")
-	set.BoolVar(&options.Project, "project", false, "Use a project folder to avoid sending same request multiple times")
-	set.StringVar(&options.ProjectPath, "project-path", "", "Use a user defined project folder, temporary folder is used if not specified but enabled")
-	set.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "Don't display metadata for the matches")
-	set.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "Shows the installed nuclei-templates version")
-	set.BoolVar(&options.OfflineHTTP, "passive", false, "Enable Passive HTTP response processing mode")
-	set.StringVarP(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "biid", "", "Burp Collaborator BIID")
-	set.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "Nuclei Reporting Module configuration file")
-	set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database (Always use this to persistent report data)")
-	set.StringSliceVar(&options.Tags, "tags", []string{}, "Tags to execute templates for")
-	set.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "Exclude templates with the provided tags")
-	set.StringVarP(&options.ResolversFile, "resolvers", "r", "", "File containing resolver list for nuclei")
-	set.BoolVar(&options.Headless, "headless", false, "Enable headless browser based templates support")
-	set.BoolVar(&options.ShowBrowser, "show-browser", false, "Show the browser on the screen")
-	set.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "Number of seconds between each stats line")
-	set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback")
-	set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless")
-	set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates")
-	set.StringVarP(&options.DiskExportDirectory, "disk-export", "de", "", "Directory on disk to export reports in markdown to")
-	_ = set.Parse()
+	flagSet := goflags.NewFlagSet()
+	flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing
+on extensive configurability, massive extensibility and ease of use.`)
+
+	/* TODO Important: The defined default values, especially for slice/array types are NOT DEFAULT VALUES, but rather implicit values to which the user input is appended.
+	This can be very confusing and should be addressed
+	*/
+
+	createGroup(flagSet, "input", "Target",
+		flagSet.StringSliceVarP(&options.Targets, "target", "u", []string{}, "target URLs/hosts to scan"),
+		flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
+	)
+
+	createGroup(flagSet, "templates", "Templates",
+		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
+
+		flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"),
+		flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "list of workflows to run"),
+
+		flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run newly added templates only"),
+		flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
+	)
+
+	createGroup(flagSet, "filters", "Filtering",
+		flagSet.NormalizedStringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"),
+		flagSet.NormalizedStringSliceVar(&options.IncludeTags, "include-tags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list
+		flagSet.NormalizedStringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"),
+
+		flagSet.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "templates to be executed even if they are excluded either by default or configuration"),
+		flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"),
+
+		flagSet.VarP(&options.Severities, "impact", "severity", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
+		flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"),
+	)
+
+	createGroup(flagSet, "output", "Output",
+		flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write found issues/vulnerabilities"),
+
+		flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"),
+		flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"),
+		flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display extra verbose information"),
+		flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"),
+
+		flagSet.BoolVar(&options.JSON, "json", false, "write output in JSONL(ines) format"),
+		flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"),
+
+		flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "don't display match metadata"),
+		flagSet.BoolVarP(&options.NoTimestamp, "no-timestamp", "nts", false, "don't display timestamp metadata in CLI output"),
+		flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "local nuclei reporting database (always use this to persist report data)"),
+
+		flagSet.StringVarP(&options.DiskExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"),
+		flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"),
+	)
+
+	createGroup(flagSet, "configs", "Configurations",
+		flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"),
+		flagSet.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking
+
+		flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "custom headers in header:value format"),
+
+		flagSet.RuntimeMapVarP(&options.Vars, "var", "V", []string{}, "custom vars in var=value format"),
+
+		flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"),
+		flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"),
+		flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"),
+		flagSet.BoolVar(&options.EnvironmentVariables, "env-vars", false, "enable environment variables support"),
+	)
+
+	createGroup(flagSet, "interactsh", "interactsh",
+		flagSet.BoolVar(&options.NoInteractsh, "no-interactsh", false, "disable interactsh server for OOB testing"),
+		flagSet.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "interactsh server url for self-hosted instance"),
+		flagSet.StringVar(&options.InteractshToken, "interactsh-token", "", "authentication token for self-hosted interactsh server"),
+		flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"),
+		flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"),
+		flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"),
+		flagSet.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"),
+	)
+
+	createGroup(flagSet, "rate-limit", "Rate-Limit",
+		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
+		flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"),
+		flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
+		flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "maximum number of templates to be executed in parallel"),
+	)
+
+	createGroup(flagSet, "optimization", "Optimizations",
+		flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"),
+		flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"),
+		flagSet.IntVar(&options.MaxHostError, "max-host-error", 30, "max errors for a host before skipping from scan"),
+
+		flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"),
+		flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"),
+
+		flagSet.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "stop processing HTTP requests after the first match (may break template/workflow logic)"),
+	)
+
+	createGroup(flagSet, "headless", "Headless",
+		flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support"),
+		flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"),
+		flagSet.BoolVar(&options.ShowBrowser, "show-browser", false, "show the browser on the screen when running templates with headless mode"),
+	)
+
+	createGroup(flagSet, "debug", "Debug",
+		flagSet.BoolVar(&options.Debug, "debug", false, "show all requests and responses"),
+		flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"),
+		flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"),
+
+		/* TODO why the separation? http://proxy:port vs socks5://proxy:port etc
+		   TODO should auto-set the HTTP_PROXY variable for the process? */
+		flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"),
+		flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"),
+
+		flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"),
+
+		flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"),
+		flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"),
+	)
+
+	createGroup(flagSet, "update", "Update",
+		flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei to the latest released version"),
+		flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update the community templates to latest released version"),
+		flagSet.BoolVarP(&options.NoUpdateTemplates, "no-update-templates", "nut", false, "do not check for nuclei-templates updates"),
+		flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default nuclei-templates directory"),
+	)
+
+	createGroup(flagSet, "stats", "Statistics",
+		flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"),
+		flagSet.BoolVar(&options.StatsJSON, "stats-json", false, "write statistics data to an output file in JSONL(ines) format"),
+		flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"),
+
+		flagSet.BoolVar(&options.Metrics, "metrics", false, "expose nuclei metrics on a port"),
+		flagSet.IntVar(&options.MetricsPort, "metrics-port", 9092, "port to expose nuclei metrics on"),
+	)
+
+	_ = flagSet.Parse()
 
 	if cfgFile != "" {
-		if err := set.MergeConfigFile(cfgFile); err != nil {
+		if err := flagSet.MergeConfigFile(cfgFile); err != nil {
 			gologger.Fatal().Msgf("Could not read config: %s\n", err)
 		}
 	}
 }
+
+func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) {
+	flagSet.SetGroup(groupName, description)
+	for _, currentFlag := range flags {
+		currentFlag.Group(groupName)
+	}
+}

v2/go.mod

@@ -1,58 +1,119 @@
 module github.com/projectdiscovery/nuclei/v2
 
-go 1.15
+go 1.17
 
 require (
-	github.com/Knetic/govaluate v3.0.0+incompatible
-	github.com/andygrunwald/go-jira v1.13.0
+	github.com/Ice3man543/nvd v1.0.8
+	github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
+	github.com/akrylysov/pogreb v0.10.1 // indirect
+	github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c
+	github.com/andygrunwald/go-jira v1.14.0
+	github.com/antchfx/htmlquery v1.2.3
+	github.com/apex/log v1.9.0
 	github.com/blang/semver v3.5.1+incompatible
+	github.com/bluele/gcache v0.0.2
+	github.com/c4milo/unpackit v0.1.0 // indirect
 	github.com/corpix/uarand v0.1.1
-	github.com/fatih/structs v1.1.0 // indirect
-	github.com/go-rod/rod v0.91.1
-	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/go-rod/rod v0.101.7
 	github.com/google/go-github v17.0.0+incompatible
-	github.com/google/go-github/v32 v32.1.0
-	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
-	github.com/json-iterator/go v1.1.10
+	github.com/gosuri/uilive v0.0.4 // indirect
+	github.com/gosuri/uiprogress v0.0.1 // indirect
+	github.com/itchyny/gojq v0.12.4
+	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
+	github.com/karlseguin/ccache v2.0.3+incompatible
 	github.com/karrick/godirwalk v1.16.1
 	github.com/logrusorgru/aurora v2.0.3+incompatible
-	github.com/mattn/go-runewidth v0.0.10 // indirect
-	github.com/miekg/dns v1.1.38
-	github.com/mitchellh/go-ps v1.0.0
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/miekg/dns v1.1.43
 	github.com/olekukonko/tablewriter v0.0.5
+	github.com/owenrumney/go-sarif v1.0.11
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.8
-	github.com/projectdiscovery/collaborator v0.0.2
-	github.com/projectdiscovery/fastdialer v0.0.8
-	github.com/projectdiscovery/goflags v0.0.4
+	github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e
+	github.com/projectdiscovery/goflags v0.0.7
 	github.com/projectdiscovery/gologger v1.1.4
-	github.com/projectdiscovery/hmap v0.0.1
-	github.com/projectdiscovery/rawhttp v0.0.6
-	github.com/projectdiscovery/retryabledns v1.0.10
-	github.com/projectdiscovery/retryablehttp-go v1.0.1
+	github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa
+	github.com/projectdiscovery/interactsh v0.0.4
+	github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77
+	github.com/projectdiscovery/rawhttp v0.0.7
+	github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a
+	github.com/projectdiscovery/retryablehttp-go v1.0.2
+	github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9
+	github.com/projectdiscovery/yamldoc-go v1.0.2
 	github.com/remeh/sizedwaitgroup v1.0.0
-	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/rs/xid v1.2.1
-	github.com/segmentio/ksuid v1.0.3
+	github.com/rs/xid v1.3.0
+	github.com/segmentio/ksuid v1.0.4
+	github.com/shirou/gopsutil/v3 v3.21.7
 	github.com/spaolacci/murmur3 v1.1.0
-	github.com/spf13/cast v1.3.1
+	github.com/spf13/cast v1.4.1
 	github.com/stretchr/testify v1.7.0
 	github.com/syndtr/goleveldb v1.0.0
-	github.com/trivago/tgo v1.0.7 // indirect
+	github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible
 	github.com/valyala/fasttemplate v1.2.1
-	github.com/xanzy/go-gitlab v0.44.0
-	go.uber.org/atomic v1.7.0
-	go.uber.org/multierr v1.6.0
-	go.uber.org/ratelimit v0.1.0
-	golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect
-	golang.org/x/net v0.0.0-20210119194325-5f4716e94777
-	golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
-	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
-	golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b // indirect
-	golang.org/x/text v0.3.4 // indirect
-	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
+	github.com/xanzy/go-gitlab v0.50.3
+	github.com/ysmood/gson v0.6.4 // indirect
+	github.com/ysmood/leakless v0.7.0 // indirect
+	go.uber.org/atomic v1.9.0
+	go.uber.org/multierr v1.7.0
+	go.uber.org/ratelimit v0.2.0
+	golang.org/x/net v0.0.0-20210916014120-12bc252f5db8
+	golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab
+	golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect
+	golang.org/x/text v0.3.7
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 )
+
+require (
+	git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect
+	github.com/PuerkitoBio/goquery v1.6.0 // indirect
+	github.com/StackExchange/wmi v1.2.1 // indirect
+	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
+	github.com/andybalholm/cascadia v1.1.0 // indirect
+	github.com/antchfx/xpath v1.1.6 // indirect
+	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/dsnet/compress v0.0.1 // indirect
+	github.com/eggsampler/acme/v3 v3.2.1 // indirect
+	github.com/fatih/structs v1.1.0 // indirect
+	github.com/go-ole/go-ole v1.2.5 // indirect
+	github.com/golang-jwt/jwt v3.2.1+incompatible // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/go-querystring v1.0.0 // indirect
+	github.com/google/uuid v1.2.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
+	github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
+	github.com/itchyny/timefmt-go v0.1.3 // indirect
+	github.com/jasonlvhit/gocron v0.0.1 // indirect
+	github.com/karlseguin/ccache/v2 v2.0.8 // indirect
+	github.com/klauspost/compress v1.13.4 // indirect
+	github.com/klauspost/pgzip v1.2.5 // indirect
+	github.com/mattn/go-isatty v0.0.13 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect
+	github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect
+	github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect
+	github.com/projectdiscovery/mapcidr v0.0.8 // indirect
+	github.com/projectdiscovery/networkpolicy v0.0.1 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.7 // indirect
+	github.com/tklauser/numcpus v0.2.3 // indirect
+	github.com/trivago/tgo v1.0.7 // indirect
+	github.com/ulikunitz/xz v0.5.10 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/yl2chen/cidranger v1.0.2 // indirect
+	github.com/ysmood/goob v0.3.0 // indirect
+	github.com/zclconf/go-cty v1.8.4 // indirect
+	go.etcd.io/bbolt v1.3.6 // indirect
+	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)

v2/go.sum

@@ -31,18 +31,61 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8=
+git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs=
+github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
+github.com/Ice3man543/nvd v1.0.8 h1:2CBEgOxyWAkQocnnmEMmRtVPWooPRvcuHFLWj48EM4c=
+github.com/Ice3man543/nvd v1.0.8/go.mod h1:0DxLJk6revOcJKiZxa2K+rNF/HO1zJO97lqQtXhXfSc=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
+github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw=
+github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
-github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE=
-github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
+github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94=
+github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
+github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
+github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
+github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
+github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
+github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c h1:oJsq4z4xKgZWWOhrSZuLZ5KyYfRFytddLL1E5+psfIY=
+github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
+github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
+github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/andygrunwald/go-jira v1.14.0 h1:7GT/3qhar2dGJ0kq8w0d63liNyHOnxZsUZ9Pe4+AKBI=
+github.com/andygrunwald/go-jira v1.14.0/go.mod h1:KMo2f4DgMZA1C9FdImuLc04x4WQhn5derQpnsuBFgqE=
+github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M=
+github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
+github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0=
+github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
+github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
+github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA=
+github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
+github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
+github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
+github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
+github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
+github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
+github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og=
+github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I=
+github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r7ZRSxEo=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -51,33 +94,62 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
 github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4=
 github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
 github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU=
+github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ=
+github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
+github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8=
+github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE=
+github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
+github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
+github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
+github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
+github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/eggsampler/acme/v3 v3.2.1 h1:Lfsrg3M2zt00QRnizOFzdpSfsS9oDvPsGrodXS/w1KI=
+github.com/eggsampler/acme/v3 v3.2.1/go.mod h1:/qh0rKC/Dh7Jj+p4So7DbWmFNzC4dpcpK53r226Fhuo=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-rod/rod v0.91.1 h1:7xIlC/bXCXosZqZUl2x6GVB8tv4yMQ4W/ZVdGVa1qYI=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=
+github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI=
+github.com/go-rod/rod v0.101.7 h1:kbI5CNvcRhf7feybBln4xDutsM0mbsF0ENNZfKcF6WA=
+github.com/go-rod/rod v0.101.7/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY=
+github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
+github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -99,11 +171,14 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
 github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -112,19 +187,20 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
-github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
-github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
 github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@@ -133,143 +209,295 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY=
+github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI=
+github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw=
+github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0=
+github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
-github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
 github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
-github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
 github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs=
+github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk=
+github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA=
+github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o=
+github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg=
+github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
+github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
+github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
+github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU=
+github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w=
+github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA=
+github.com/karlseguin/ccache/v2 v2.0.8/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ=
+github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA=
+github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
 github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s=
+github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
+github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA=
+github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
-github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
+github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw=
-github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
+github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
+github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
-github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4=
+github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ=
+github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48=
+github.com/owenrumney/go-sarif v1.0.11 h1:7k4TLSi6h3vAozSECjO0arcQoeUNDMgvA7LDac95sJo=
+github.com/owenrumney/go-sarif v1.0.11/go.mod h1:hTBFbxU7GuVRUvwMx+eStp9M/Oun4xHCS3vqpPvket8=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e h1:7bwaFH1jvtOo5ndhTQgoA349ozhX+1dc4b6tbaPnBOA=
+github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs=
 github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20=
 github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
-github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5zmiJ6tI7+6mB3s=
-github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
-github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0=
-github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4=
-github.com/projectdiscovery/goflags v0.0.4 h1:fWKLMAr3KmPlZxE1b54pfei+vGIUJn9q6aM7woZIbCY=
-github.com/projectdiscovery/goflags v0.0.4/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
+github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U=
+github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc=
+github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0=
+github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges=
+github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ=
+github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk=
+github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
+github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
 github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
-github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
-github.com/projectdiscovery/rawhttp v0.0.6 h1:HbgPB1eKXQVV5F9sq0Uxflm95spWFyZYD8dgFpeOC9M=
-github.com/projectdiscovery/rawhttp v0.0.6/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
-github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
-github.com/projectdiscovery/retryabledns v1.0.10 h1:xJZ2aKoqrNg/OZEw1+4+QIOH40V/WkZDYY1ZZc+uphE=
-github.com/projectdiscovery/retryabledns v1.0.10/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
-github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
+github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
+github.com/projectdiscovery/hmap v0.0.2-0.20210917073634-bfb0e9c03800/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
+github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0=
+github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo=
+github.com/projectdiscovery/interactsh v0.0.4 h1:3BtCZrrTovGYiqdFktXJ4NxKAQFvUvzcEI5pJIuShM8=
+github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE=
+github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE=
+github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
+github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
+github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 h1:VZ9H51A7tI7R/9I5W5l960Nkq7eMJqGd3y1wsuwzdjE=
+github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3/go.mod h1:blmYJkS8lSrrx3QcmcgS2tZIxlojeVmoGeA9twslCBU=
+github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
+github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM=
+github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00=
+github.com/projectdiscovery/mapcidr v0.0.8 h1:16U05F2x3o/jSTsxSCY2hCuCs9xOSwVxjo2zlsL4L4E=
+github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00=
+github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I=
+github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs=
+github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 h1:SNtAiRRrJtDJJDroaa/bFXt/Tix2LA6+rHRib0ORlJQ=
+github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M=
+github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY=
+github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg=
+github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
+github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
+github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
+github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a h1:WJQjr9qi/VjWhdNiGyNqcFi0967Gp0W3I769bCpHOJE=
+github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI=
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
+github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U=
+github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
+github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
+github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
+github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes=
+github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
+github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ=
+github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
-github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
-github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
-github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
+github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
+github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
+github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
+github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c=
+github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shirou/gopsutil/v3 v3.21.7 h1:PnTqQamUjwEDSgn+nBGu0qSDV/CfvyiR/gwTH3i7HTU=
+github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4=
+github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
-github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA=
+github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
-github.com/trivago/tgo v1.0.1/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
+github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
+github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
+github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
+github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc=
+github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
+github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
+github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
+github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible h1:guTq1YxwB8XSILkI9q4IrOmrCOS6Hc1L3hmOhi4Swcs=
+github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible/go.mod h1:waFwwyiAhGey2e+dNoYQ/iLhIcFqhCW7zL/+vDU1WLo=
+github.com/tklauser/go-sysconf v0.3.7 h1:HT7h4+536gjqeq1ZIJPgOl1rg1XFatQGVZWp7Py53eg=
+github.com/tklauser/go-sysconf v0.3.7/go.mod h1:JZIdXh4RmBvZDBZ41ld2bGxRV3n4daiiqA3skYhAoQ4=
+github.com/tklauser/numcpus v0.2.3 h1:nQ0QYpiritP6ViFhrKYsiv6VVxOpum2Gks5GhnJbS/8=
+github.com/tklauser/numcpus v0.2.3/go.mod h1:vpEPS/JC+oZGGQ/My/vJnNsvMDQL6PwOqt8dsCw5j+E=
 github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM=
 github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
+github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
+github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
-github.com/xanzy/go-gitlab v0.44.0 h1:cEiGhqu7EpFGuei2a2etAwB+x6403E5CvpLn35y+GPs=
-github.com/xanzy/go-gitlab v0.44.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
+github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
+github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
+github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
+github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
+github.com/xanzy/go-gitlab v0.50.3 h1:M7ncgNhCN4jaFNyXxarJhCLa9Qi6fdmCxFFhMTQPZiY=
+github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
+github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
 github.com/ysmood/goob v0.3.0 h1:XZ51cZJ4W3WCoCiUktixzMIQF86W7G5VFL4QQ/Q2uS0=
 github.com/ysmood/goob v0.3.0/go.mod h1:S3lq113Y91y1UBf1wj1pFOxeahvfKkCk6mTWTWbDdWs=
-github.com/ysmood/got v0.9.3 h1:qx51X49jL/WAiqZzPTkPZ0zp5pTmrWJa4zYFTYo0gHI=
 github.com/ysmood/got v0.9.3/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
-github.com/ysmood/gotrace v0.2.0 h1:IkTC6rJREwXSaG8yWK+NFwIJGIsxA1DjC6/gxYyQttE=
+github.com/ysmood/got v0.14.1/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
+github.com/ysmood/got v0.15.1 h1:X5jAbMyBf5yeezuFMp9HaMGXZWMSqIQcUlAHI+kJmUs=
+github.com/ysmood/got v0.15.1/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
 github.com/ysmood/gotrace v0.2.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
-github.com/ysmood/gson v0.6.3 h1:4cU+5oOdsyundXHy00t99H0rLXLthuseD3x6W+xmCiU=
+github.com/ysmood/gotrace v0.2.2 h1:006KHGRThSRf8lwh4EyhNmuuq/l+Ygs+JqojkhEG1/E=
+github.com/ysmood/gotrace v0.2.2/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
 github.com/ysmood/gson v0.6.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
-github.com/ysmood/leakless v0.6.12 h1:XxtRYl97bJklfv4BZVdyGnd/y42p6w8lu1hUzfCkT4M=
+github.com/ysmood/gson v0.6.4 h1:Yb6tosv6bk59HqjZu2/7o4BFherpYEMkDkXmlhgryZ4=
+github.com/ysmood/gson v0.6.4/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
 github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
+github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw=
+github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/zclconf/go-cty v1.8.4 h1:pwhhz5P+Fjxse7S7UriBrMu6AUJSZM5pKqGem1PjGAs=
+github.com/zclconf/go-cty v1.8.4/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
+go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
-go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
+go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
+go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
+golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -277,9 +505,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df h1:y7QZzfUiTwWam+xBn29Ulb8CBwVN5UdzmMDavl9Whlw=
-golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -310,10 +535,11 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -332,23 +558,32 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew=
-golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 h1:5vD4XjIc0X5+kHZjx4UecYdjA6mJo+XXNoaW0EjU5Os=
-golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab h1:llrcWN/wOwO+6gAyfBzxb5hZ+c3mriU/0+KNgYu6adA=
+golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -357,24 +592,33 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -387,25 +631,37 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b h1:lAZ0/chPUDWwjqosYR0X4M490zQhMsiJ4K3DbA7o+3g=
-golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw=
+golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
-golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE=
-golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -441,12 +697,16 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs
 golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -527,22 +787,31 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
-google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/corvus-ch/zbase32.v1 v1.0.0 h1:K4u1NprbDNvKPczKfHLbwdOWHTZ0zfv2ow71H1nRnFU=
+gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DSo0nK9U+c0Y=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -552,6 +821,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

v2/internal/collaborator/collaborator.go

@@ -1,72 +0,0 @@
-package collaborator
-
-import (
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/projectdiscovery/collaborator"
-)
-
-var (
-	// PollSeconds is the seconds to poll at.
-	PollSeconds = 5
-	// DefaultMaxBufferLimit is the default request buffer limit
-	DefaultMaxBufferLimit = 150
-	// DefaultPollInterval is the default poll interval for burp collabortor polling.
-	DefaultPollInterval time.Duration = time.Second * time.Duration(PollSeconds)
-	// DefaultCollaborator is the default burp collaborator instance
-	DefaultCollaborator = &Collaborator{Collab: collaborator.NewBurpCollaborator()}
-)
-
-// Collaborator is a client for recording burp collaborator interactions
-type Collaborator struct {
-	sync.RWMutex
-	options *Options // unused
-	Collab  *collaborator.BurpCollaborator
-}
-
-// Options contains configuration options for collaborator client
-type Options struct {
-	BIID           string
-	PollInterval   time.Duration
-	MaxBufferLimit int
-}
-
-// New creates a new collaborator client
-func New(options *Options) *Collaborator {
-	collab := collaborator.NewBurpCollaborator()
-	collab.AddBIID(options.BIID)
-	collab.MaxBufferLimit = options.MaxBufferLimit
-	return &Collaborator{Collab: collab, options: options}
-}
-
-// Poll initiates collaborator polling if any BIIDs were provided
-func (b *Collaborator) Poll() {
-	// if no valid biids were provided just return
-	if len(b.Collab.BIIDs) > 0 {
-		go b.Collab.PollEach(DefaultPollInterval)
-	}
-}
-
-// Has checks if a collabrator hit was found for a URL
-func (b *Collaborator) Has(s string) bool {
-	for _, r := range b.Collab.RespBuffer {
-		for i := 0; i < len(r.Responses); i++ {
-			// search in dns - http - smtp
-			b.RLock()
-			found := strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) ||
-				strings.Contains(r.Responses[i].Data.RequestDecoded, s) ||
-				strings.Contains(r.Responses[i].Data.MessageDecoded, s)
-			b.RUnlock()
-
-			if found {
-				b.Lock()
-				r.Responses = append(r.Responses[:i], r.Responses[i+1:]...)
-				b.Unlock()
-				return true
-			}
-		}
-	}
-	return false
-}

v2/internal/colorizer/colorizer.go

@@ -1,24 +1,41 @@
 package colorizer
 
-import "github.com/logrusorgru/aurora"
+import (
+	"fmt"
 
-// Colorizer returns a colorized severity printer
-type Colorizer struct {
-	Data map[string]string
-}
+	"github.com/logrusorgru/aurora"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+)
 
 const (
 	fgOrange uint8 = 208
 )
 
-// New returns a new severity based colorizer
-func New(colorizer aurora.Aurora) *Colorizer {
-	severityMap := map[string]string{
-		"info":     colorizer.Blue("info").String(),
-		"low":      colorizer.Green("low").String(),
-		"medium":   colorizer.Yellow("medium").String(),
-		"high":     colorizer.Index(fgOrange, "high").String(),
-		"critical": colorizer.Red("critical").String(),
+func GetColor(colorizer aurora.Aurora, templateSeverity fmt.Stringer) string {
+	var method func(arg interface{}) aurora.Value
+	switch templateSeverity {
+	case severity.Info:
+		method = colorizer.Blue
+	case severity.Low:
+		method = colorizer.Green
+	case severity.Medium:
+		method = colorizer.Yellow
+	case severity.High:
+		method = func(stringValue interface{}) aurora.Value { return colorizer.Index(fgOrange, stringValue) }
+	case severity.Critical:
+		method = colorizer.Red
+	default:
+		gologger.Warning().Msgf("The '%s' severity does not have an color associated!", templateSeverity)
+		method = colorizer.White
+	}
+
+	return method(templateSeverity.String()).String()
+}
+
+func New(colorizer aurora.Aurora) func(severity.Severity) string {
+	return func(severity severity.Severity) string {
+		return GetColor(colorizer, severity)
 	}
-	return &Colorizer{Data: severityMap}
 }

v2/internal/runner/banner.go

@@ -1,23 +1,25 @@
 package runner
 
-import "github.com/projectdiscovery/gologger"
+import (
+	"fmt"
 
-const banner = `
-                       __     _
-     ____  __  _______/ /__  (_)
-    / __ \/ / / / ___/ / _ \/ /
-   / / / / /_/ / /__/ /  __/ /
-  /_/ /_/\__,_/\___/_/\___/_/   v2.3.2
-`
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+)
 
-// Version is the current version of nuclei
-const Version = `2.3.2`
+var banner = fmt.Sprintf(`
+                     __     _
+   ____  __  _______/ /__  (_)
+  / __ \/ / / / ___/ / _ \/ /
+ / / / / /_/ / /__/ /  __/ /
+/_/ /_/\__,_/\___/_/\___/_/   %s
+`, config.Version)
 
 // showBanner is used to show the banner to the user
 func showBanner() {
 	gologger.Print().Msgf("%s\n", banner)
 	gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n")
 
-	gologger.Warning().Msgf("Use with caution. You are responsible for your actions\n")
-	gologger.Warning().Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
+	gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n")
+	gologger.Error().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
 }

v2/internal/runner/config.go

@@ -1,130 +0,0 @@
-package runner
-
-import (
-	"bufio"
-	"os"
-	"path"
-	"regexp"
-	"strings"
-	"time"
-
-	jsoniter "github.com/json-iterator/go"
-)
-
-// nucleiConfig contains some configuration options for nuclei
-type nucleiConfig struct {
-	TemplatesDirectory string    `json:"templates-directory,omitempty"`
-	CurrentVersion     string    `json:"current-version,omitempty"`
-	LastChecked        time.Time `json:"last-checked,omitempty"`
-	IgnoreURL          string    `json:"ignore-url,omitempty"`
-	NucleiVersion      string    `json:"nuclei-version,omitempty"`
-	LastCheckedIgnore  time.Time `json:"last-checked-ignore,omitempty"`
-	// IgnorePaths ignores all the paths listed unless specified manually
-	IgnorePaths []string `json:"ignore-paths,omitempty"`
-}
-
-// nucleiConfigFilename is the filename of nuclei configuration file.
-const nucleiConfigFilename = ".templates-config.json"
-
-var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
-
-// readConfiguration reads the nuclei configuration file from disk.
-func readConfiguration() (*nucleiConfig, error) {
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return nil, err
-	}
-	configDir := path.Join(home, "/.config", "/nuclei")
-	_ = os.MkdirAll(configDir, os.ModePerm)
-
-	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
-	file, err := os.Open(templatesConfigFile)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-
-	config := &nucleiConfig{}
-	err = jsoniter.NewDecoder(file).Decode(config)
-	if err != nil {
-		return nil, err
-	}
-	return config, nil
-}
-
-// readConfiguration reads the nuclei configuration file from disk.
-func (r *Runner) writeConfiguration(config *nucleiConfig) error {
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return err
-	}
-	configDir := path.Join(home, "/.config", "/nuclei")
-	_ = os.MkdirAll(configDir, os.ModePerm)
-
-	if config.IgnoreURL == "" {
-		config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
-	}
-	config.LastChecked = time.Now()
-	config.LastCheckedIgnore = time.Now()
-	config.NucleiVersion = Version
-	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
-	file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	err = jsoniter.NewEncoder(file).Encode(config)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-const nucleiIgnoreFile = ".nuclei-ignore"
-
-// readNucleiIgnoreFile reads the nuclei ignore file marking it in map
-func (r *Runner) readNucleiIgnoreFile() {
-	file, err := os.Open(r.getIgnoreFilePath())
-	if err != nil {
-		return
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		text := scanner.Text()
-		if text == "" {
-			continue
-		}
-		if strings.HasPrefix(text, "#") {
-			continue
-		}
-		r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, text)
-	}
-}
-
-// getIgnoreFilePath returns the ignore file path for the runner
-func (r *Runner) getIgnoreFilePath() string {
-	var defIgnoreFilePath string
-
-	home, err := os.UserHomeDir()
-	if err == nil {
-		configDir := path.Join(home, "/.config", "/nuclei")
-		_ = os.MkdirAll(configDir, os.ModePerm)
-
-		defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile)
-	}
-
-	cwd, err := os.Getwd()
-	if err != nil {
-		return defIgnoreFilePath
-	}
-	cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile)
-
-	cwdIfpInfo, err := os.Stat(cwdIgnoreFilePath)
-	if os.IsNotExist(err) || cwdIfpInfo.IsDir() {
-		return defIgnoreFilePath
-	}
-	return cwdIgnoreFilePath
-}

v2/internal/runner/options.go

@@ -5,11 +5,13 @@ import (
 	"errors"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/formatter"
 	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
@@ -19,23 +21,27 @@ func ParseOptions(options *types.Options) {
 	// Check if stdin pipe was given
 	options.Stdin = hasStdin()
 
+	// if VerboseVerbose is set, it implicitly enables the Verbose option as well
+	if options.VerboseVerbose {
+		options.Verbose = true
+	}
+
 	// Read the inputs and configure the logging
 	configureOutput(options)
 
 	// Show the user the banner
 	showBanner()
 
-	options.ExcludeTags = append(options.ExcludeTags, "dos")
 	if options.Version {
-		gologger.Info().Msgf("Current Version: %s\n", Version)
+		gologger.Info().Msgf("Current Version: %s\n", config.Version)
 		os.Exit(0)
 	}
 	if options.TemplatesVersion {
-		config, err := readConfiguration()
+		configuration, err := config.ReadConfiguration()
 		if err != nil {
 			gologger.Fatal().Msgf("Could not read template configuration: %s\n", err)
 		}
-		gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", config.CurrentVersion, config.TemplatesDirectory)
+		gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", configuration.TemplateVersion, configuration.TemplatesDirectory)
 		os.Exit(0)
 	}
 
@@ -76,28 +82,22 @@ func hasStdin() bool {
 
 // validateOptions validates the configuration options passed
 func validateOptions(options *types.Options) error {
-	// Both verbose and silent flags were used
 	if options.Verbose && options.Silent {
 		return errors.New("both verbose and silent mode specified")
 	}
 
-	if !options.TemplateList {
-		// Check if a list of templates was provided and it exists
-		if len(options.Templates) == 0 && !options.NewTemplates && len(options.Workflows) == 0 && len(options.Tags) == 0 && !options.UpdateTemplates {
-			return errors.New("no template/templates provided")
-		}
-	}
-
-	// Validate proxy options if provided
-	err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)")
-	if err != nil {
+	if err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)"); err != nil {
 		return err
 	}
 
-	err = validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)")
-	if err != nil {
+	if err := validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)"); err != nil {
 		return err
 	}
+
+	if options.Validate {
+		validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows)
+	}
+
 	return nil
 }
 
@@ -114,10 +114,10 @@ func isValidURL(urlString string) bool {
 	return err == nil
 }
 
-// configureOutput configures the output on the screen
+// configureOutput configures the output logging levels to be displayed on the screen
 func configureOutput(options *types.Options) {
 	// If the user desires verbose output, show verbose output
-	if options.Verbose {
+	if options.Verbose || options.VerboseVerbose {
 		gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
 	}
 	if options.Debug {
@@ -156,3 +156,21 @@ func loadResolvers(options *types.Options) {
 		}
 	}
 }
+
+func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) {
+	allGivenTemplatePaths := append(templatePaths, workflowPaths...)
+
+	for _, templatePath := range allGivenTemplatePaths {
+		if templatesDirectory != templatePath && filepath.IsAbs(templatePath) {
+			fileInfo, err := os.Stat(templatePath)
+			if err == nil && fileInfo.IsDir() {
+				relativizedPath, err2 := filepath.Rel(templatesDirectory, templatePath)
+				if err2 != nil || (len(relativizedPath) >= 2 && relativizedPath[:2] == "..") {
+					gologger.Warning().Msgf("The given path (%s) is outside the default template directory path (%s)! "+
+						"Referenced sub-templates with relative paths in workflows will be resolved against the default template directory.", templatePath, templatesDirectory)
+					break
+				}
+			}
+		}
+	}
+}

v2/internal/runner/processor.go

@@ -7,13 +7,17 @@ import (
 	"go.uber.org/atomic"
 )
 
-// processTemplateWithList process a template on the URL list
+// processTemplateWithList execute a template against the list of user provided targets
 func (r *Runner) processTemplateWithList(template *templates.Template) bool {
 	results := &atomic.Bool{}
 	wg := sizedwaitgroup.New(r.options.BulkSize)
 	r.hostMap.Scan(func(k, _ []byte) error {
 		URL := string(k)
 
+		// Skip if the host has had errors
+		if r.hostErrors != nil && r.hostErrors.Check(URL) {
+			return nil
+		}
 		wg.Add()
 		go func(URL string) {
 			defer wg.Done()
@@ -37,6 +41,11 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool {
 
 	r.hostMap.Scan(func(k, _ []byte) error {
 		URL := string(k)
+
+		// Skip if the host has had errors
+		if r.hostErrors != nil && r.hostErrors.Check(URL) {
+			return nil
+		}
 		wg.Add()
 		go func(URL string) {
 			defer wg.Done()

v2/internal/runner/runner.go

@@ -4,47 +4,61 @@ import (
 	"bufio"
 	"fmt"
 	"os"
-	"path"
+	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/logrusorgru/aurora"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/hmap/store/hybrid"
-	"github.com/projectdiscovery/nuclei/v2/internal/collaborator"
-	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
-	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
-	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
-	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"github.com/pkg/errors"
 	"github.com/remeh/sizedwaitgroup"
 	"github.com/rs/xid"
 	"go.uber.org/atomic"
 	"go.uber.org/ratelimit"
 	"gopkg.in/yaml.v2"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/hmap/store/hybrid"
+	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
 )
 
 // Runner is a client for running the enumeration process.
 type Runner struct {
 	hostMap         *hybrid.HybridMap
 	output          output.Writer
+	interactsh      *interactsh.Client
 	inputCount      int64
-	templatesConfig *nucleiConfig
+	templatesConfig *config.Config
 	options         *types.Options
 	projectFile     *projectfile.ProjectFile
 	catalog         *catalog.Catalog
 	progress        progress.Progress
 	colorizer       aurora.Aurora
 	issuesClient    *reporting.Client
-	severityColors  *colorizer.Colorizer
+	addColor        func(severity.Severity) string
 	browser         *engine.Browser
 	ratelimiter     ratelimit.Limiter
+	hostErrors      *hosterrorscache.Cache
 }
 
 // New creates a new client for running enumeration process.
@@ -52,6 +66,18 @@ func New(options *types.Options) (*Runner, error) {
 	runner := &Runner{
 		options: options,
 	}
+	if options.UpdateNuclei {
+		if err := updateNucleiVersionToLatest(runner.options.Verbose); err != nil {
+			return nil, err
+		}
+		return nil, nil
+	}
+	if options.Validate {
+		parsers.ShouldValidate = true
+	}
+	if err := runner.updateTemplates(); err != nil {
+		gologger.Warning().Msgf("Could not update templates: %s\n", err)
+	}
 	if options.Headless {
 		browser, err := engine.New(options)
 		if err != nil {
@@ -59,72 +85,60 @@ func New(options *types.Options) (*Runner, error) {
 		}
 		runner.browser = browser
 	}
-	if err := runner.updateTemplates(); err != nil {
-		gologger.Warning().Msgf("Could not update templates: %s\n", err)
-	}
-	// Read nucleiignore file if given a templateconfig
-	if runner.templatesConfig != nil {
-		runner.readNucleiIgnoreFile()
-	}
+
 	runner.catalog = catalog.New(runner.options.TemplatesDirectory)
 
-	var reportingOptions *reporting.Options
-	if options.ReportingConfig != "" {
-		file, err := os.Open(options.ReportingConfig)
-		if err != nil {
-			gologger.Fatal().Msgf("Could not open reporting config file: %s\n", err)
-		}
-
-		reportingOptions = &reporting.Options{}
-		if parseErr := yaml.NewDecoder(file).Decode(reportingOptions); parseErr != nil {
-			file.Close()
-			gologger.Fatal().Msgf("Could not parse reporting config file: %s\n", parseErr)
-		}
-		file.Close()
-	}
-	if options.DiskExportDirectory != "" {
-		if reportingOptions != nil {
-			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
-		} else {
-			reportingOptions = &reporting.Options{}
-			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
-		}
+	reportingOptions, err := createReportingOptions(options)
+	if err != nil {
+		return nil, err
 	}
 	if reportingOptions != nil {
-		if client, err := reporting.New(reportingOptions, options.ReportingDB); err != nil {
-			gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err)
-		} else {
-			runner.issuesClient = client
+		client, err := reporting.New(reportingOptions, options.ReportingDB)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create issue reporting client")
 		}
+		runner.issuesClient = client
 	}
 
 	// output coloring
 	useColor := !options.NoColor
 	runner.colorizer = aurora.NewAurora(useColor)
-	runner.severityColors = colorizer.New(runner.colorizer)
+	runner.addColor = colorizer.New(runner.colorizer)
 
 	if options.TemplateList {
 		runner.listAvailableTemplates()
 		os.Exit(0)
 	}
 
-	if (len(options.Templates) == 0 || !options.NewTemplates || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
+	if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates {
 		os.Exit(0)
 	}
-	if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil {
-		gologger.Fatal().Msgf("Could not create temporary input file: %s\n", err)
-	} else {
-		runner.hostMap = hm
+	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create temporary input file")
 	}
+	runner.hostMap = hm
 
 	runner.inputCount = 0
 	dupeCount := 0
 
-	// Handle single target
-	if options.Target != "" {
-		runner.inputCount++
-		// nolint:errcheck // ignoring error
-		runner.hostMap.Set(options.Target, nil)
+	// Handle multiple targets
+	if len(options.Targets) != 0 {
+		for _, target := range options.Targets {
+			url := strings.TrimSpace(target)
+			if url == "" {
+				continue
+			}
+
+			if _, ok := runner.hostMap.Get(url); ok {
+				dupeCount++
+				continue
+			}
+
+			runner.inputCount++
+			// nolint:errcheck // ignoring error
+			runner.hostMap.Set(url, nil)
+		}
 	}
 
 	// Handle stdin
@@ -135,21 +149,23 @@ func New(options *types.Options) (*Runner, error) {
 			if url == "" {
 				continue
 			}
+
 			if _, ok := runner.hostMap.Get(url); ok {
 				dupeCount++
 				continue
 			}
+
 			runner.inputCount++
 			// nolint:errcheck // ignoring error
 			runner.hostMap.Set(url, nil)
 		}
 	}
 
-	// Handle taget file
-	if options.Targets != "" {
-		input, err := os.Open(options.Targets)
-		if err != nil {
-			gologger.Fatal().Msgf("Could not open targets file '%s': %s\n", options.Targets, err)
+	// Handle target file
+	if options.TargetsFilePath != "" {
+		input, inputErr := os.Open(options.TargetsFilePath)
+		if inputErr != nil {
+			return nil, errors.Wrap(inputErr, "could not open targets file")
 		}
 		scanner := bufio.NewScanner(input)
 		for scanner.Scan() {
@@ -173,34 +189,57 @@ func New(options *types.Options) (*Runner, error) {
 	}
 
 	// Create the output file if asked
-	outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.JSON, options.Output, options.TraceLogFile)
+	outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.Output, options.TraceLogFile)
 	if err != nil {
-		gologger.Fatal().Msgf("Could not create output file '%s': %s\n", options.Output, err)
+		return nil, errors.Wrap(err, "could not create output file")
 	}
 	runner.output = outputWriter
 
+	if options.JSON && options.EnableProgressBar {
+		options.StatsJSON = true
+	}
+	if options.StatsJSON {
+		options.EnableProgressBar = true
+	}
 	// Creates the progress tracking object
 	var progressErr error
-	runner.progress, progressErr = progress.NewStatsTicker(options.StatsInterval, options.EnableProgressBar, options.Metrics, options.MetricsPort)
+	runner.progress, progressErr = progress.NewStatsTicker(options.StatsInterval, options.EnableProgressBar, options.StatsJSON, options.Metrics, options.MetricsPort)
 	if progressErr != nil {
 		return nil, progressErr
 	}
 
-	// create project file if requested or load existing one
+	// create project file if requested or load the existing one
 	if options.Project {
 		var projectFileErr error
-		runner.projectFile, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""})
+		runner.projectFile, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: utils.IsBlank(options.ProjectPath)})
 		if projectFileErr != nil {
 			return nil, projectFileErr
 		}
 	}
 
-	// Enable Polling
-	if options.BurpCollaboratorBiid != "" {
-		collaborator.DefaultCollaborator.Collab.AddBIID(options.BurpCollaboratorBiid)
+	if !options.NoInteractsh {
+		interactshClient, err := interactsh.New(&interactsh.Options{
+			ServerURL:      options.InteractshURL,
+			Authorization:  options.InteractshToken,
+			CacheSize:      int64(options.InteractionsCacheSize),
+			Eviction:       time.Duration(options.InteractionsEviction) * time.Second,
+			ColldownPeriod: time.Duration(options.InteractionsColldownPeriod) * time.Second,
+			PollDuration:   time.Duration(options.InteractionsPollDuration) * time.Second,
+			Output:         runner.output,
+			IssuesClient:   runner.issuesClient,
+			Progress:       runner.progress,
+			Debug:          runner.options.Debug,
+		})
+		if err != nil {
+			gologger.Error().Msgf("Could not create interactsh client: %s", err)
+		} else {
+			runner.interactsh = interactshClient
+		}
 	}
 
-	if options.RateLimit > 0 {
+	if options.RateLimitMinute > 0 {
+		runner.ratelimiter = ratelimit.New(options.RateLimitMinute, ratelimit.Per(60*time.Second))
+	} else if options.RateLimit > 0 {
 		runner.ratelimiter = ratelimit.New(options.RateLimit)
 	} else {
 		runner.ratelimiter = ratelimit.NewUnlimited()
@@ -208,6 +247,40 @@ func New(options *types.Options) (*Runner, error) {
 	return runner, nil
 }
 
+func createReportingOptions(options *types.Options) (*reporting.Options, error) {
+	var reportingOptions *reporting.Options
+	if options.ReportingConfig != "" {
+		file, err := os.Open(options.ReportingConfig)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not open reporting config file")
+		}
+
+		reportingOptions = &reporting.Options{}
+		if parseErr := yaml.NewDecoder(file).Decode(reportingOptions); parseErr != nil {
+			file.Close()
+			return nil, errors.Wrap(parseErr, "could not parse reporting config file")
+		}
+		file.Close()
+	}
+	if options.DiskExportDirectory != "" {
+		if reportingOptions != nil {
+			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
+		} else {
+			reportingOptions = &reporting.Options{}
+			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
+		}
+	}
+	if options.SarifExport != "" {
+		if reportingOptions != nil {
+			reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}
+		} else {
+			reportingOptions = &reporting.Options{}
+			reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}
+		}
+	}
+	return reportingOptions, nil
+}
+
 // Close releases all the resources and cleans up
 func (r *Runner) Close() {
 	if r.output != nil {
@@ -217,77 +290,173 @@ func (r *Runner) Close() {
 	if r.projectFile != nil {
 		r.projectFile.Close()
 	}
+	protocolinit.Close()
 }
 
 // RunEnumeration sets up the input layer for giving input nuclei.
 // binary and runs the actual enumeration
-func (r *Runner) RunEnumeration() {
+func (r *Runner) RunEnumeration() error {
 	defer r.Close()
 
-	// If we have no templates, run on whole template directory with provided tags
-	if len(r.options.Templates) == 0 && len(r.options.Workflows) == 0 && !r.options.NewTemplates && (len(r.options.Tags) > 0 || len(r.options.ExcludeTags) > 0) {
-		r.options.Templates = append(r.options.Templates, r.options.TemplatesDirectory)
-	}
+	// If user asked for new templates to be executed, collect the list from the templates' directory.
 	if r.options.NewTemplates {
 		templatesLoaded, err := r.readNewTemplatesFile()
 		if err != nil {
-			gologger.Warning().Msgf("Could not get newly added templates: %s\n", err)
+			return errors.Wrap(err, "could not get newly added templates")
 		}
 		r.options.Templates = append(r.options.Templates, templatesLoaded...)
 	}
-	includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates)
-	excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates)
-	// defaults to all templates
-	allTemplates := includedTemplates
+	ignoreFile := config.ReadIgnoreFile()
+	r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
+	r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
 
-	if len(excludedTemplates) > 0 {
-		excludedMap := make(map[string]struct{}, len(excludedTemplates))
-		for _, excl := range excludedTemplates {
-			excludedMap[excl] = struct{}{}
-		}
-		// rebuild list with only non-excluded templates
-		allTemplates = []string{}
+	var cache *hosterrorscache.Cache
+	if r.options.MaxHostError > 0 {
+		cache = hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount).SetVerbose(r.options.Verbose)
+	}
+	r.hostErrors = cache
+	executerOpts := protocols.ExecuterOptions{
+		Output:          r.output,
+		Options:         r.options,
+		Progress:        r.progress,
+		Catalog:         r.catalog,
+		IssuesClient:    r.issuesClient,
+		RateLimiter:     r.ratelimiter,
+		Interactsh:      r.interactsh,
+		ProjectFile:     r.projectFile,
+		Browser:         r.browser,
+		HostErrorsCache: cache,
+	}
 
-		for _, incl := range includedTemplates {
-			if _, found := excludedMap[incl]; !found {
-				allTemplates = append(allTemplates, incl)
-			} else {
-				gologger.Warning().Msgf("Excluding '%s'", incl)
-			}
+	workflowLoader, err := parsers.NewLoader(&executerOpts)
+	if err != nil {
+		return errors.Wrap(err, "Could not create loader.")
+	}
+
+	executerOpts.WorkflowLoader = workflowLoader
+
+	loaderConfig := loader.Config{
+		Templates:          r.options.Templates,
+		Workflows:          r.options.Workflows,
+		ExcludeTemplates:   r.options.ExcludedTemplates,
+		Tags:               r.options.Tags,
+		ExcludeTags:        r.options.ExcludeTags,
+		IncludeTemplates:   r.options.IncludeTemplates,
+		Authors:            r.options.Author,
+		Severities:         r.options.Severities,
+		IncludeTags:        r.options.IncludeTags,
+		TemplatesDirectory: r.options.TemplatesDirectory,
+		Catalog:            r.catalog,
+		ExecutorOptions:    executerOpts,
+	}
+	store, err := loader.New(&loaderConfig)
+	if err != nil {
+		return errors.Wrap(err, "could not load templates from config")
+	}
+	store.Load()
+
+	if r.options.Validate {
+		if err := store.ValidateTemplates(r.options.Templates, r.options.Workflows); err != nil {
+			return err
 		}
+		if stats.GetValue(parsers.SyntaxErrorStats) == 0 && stats.GetValue(parsers.SyntaxWarningStats) == 0 {
+			gologger.Info().Msgf("All templates validated successfully\n")
+		} else {
+			return errors.New("encountered errors while performing template validation")
+		}
+		return nil // exit
+	}
+
+	// Display stats for any loaded templates' syntax warnings or errors
+	stats.Display(parsers.SyntaxWarningStats)
+	stats.Display(parsers.SyntaxErrorStats)
+
+	builder := &strings.Builder{}
+	if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" {
+		builder.WriteString(" (")
+
+		if strings.Contains(config.Version, "-dev") {
+			builder.WriteString(r.colorizer.Blue("development").String())
+		} else if config.Version == r.templatesConfig.NucleiLatestVersion {
+			builder.WriteString(r.colorizer.Green("latest").String())
+		} else {
+			builder.WriteString(r.colorizer.Red("outdated").String())
+		}
+		builder.WriteString(")")
+	}
+	messageStr := builder.String()
+	builder.Reset()
+
+	gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr)
+
+	if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { // TODO extract duplicated logic
+		builder.WriteString(" (")
+
+		if r.templatesConfig.TemplateVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
+			builder.WriteString(r.colorizer.Green("latest").String())
+		} else {
+			builder.WriteString(r.colorizer.Red("outdated").String())
+		}
+		builder.WriteString(")")
+	}
+	messageStr = builder.String()
+	builder.Reset()
+
+	if r.templatesConfig != nil {
+		gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.TemplateVersion, messageStr)
+	}
+	if r.interactsh != nil {
+		gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL)
+	}
+	if len(store.Templates()) > 0 {
+		gologger.Info().Msgf("Templates added in last update: %d", r.countNewTemplates())
+		gologger.Info().Msgf("Templates loaded for scan: %d", len(store.Templates()))
+	}
+	if len(store.Workflows()) > 0 {
+		gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows()))
 	}
 
 	// pre-parse all the templates, apply filters
 	finalTemplates := []*templates.Template{}
 
-	workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows)
-	availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false)
-	availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true)
-
-	var unclusteredRequests int64 = 0
-	for _, template := range availableTemplates {
+	var unclusteredRequests int64
+	for _, template := range store.Templates() {
 		// workflows will dynamically adjust the totals while running, as
-		// it can't be know in advance which requests will be called
+		// it can't be known in advance which requests will be called
 		if len(template.Workflows) > 0 {
 			continue
 		}
 		unclusteredRequests += int64(template.TotalRequests) * r.inputCount
 	}
 
-	originalTemplatesCount := len(availableTemplates)
+	if r.options.VerboseVerbose {
+		for _, template := range store.Templates() {
+			r.logAvailableTemplate(template.Path)
+		}
+		for _, template := range store.Workflows() {
+			r.logAvailableTemplate(template.Path)
+		}
+	}
+	templatesMap := make(map[string]*templates.Template)
+	for _, v := range store.Templates() {
+		templatesMap[v.Path] = v
+	}
+	originalTemplatesCount := len(store.Templates())
 	clusterCount := 0
-	clusters := clusterer.Cluster(availableTemplates)
+	clusters := clusterer.Cluster(templatesMap)
 	for _, cluster := range clusters {
 		if len(cluster) > 1 && !r.options.OfflineHTTP {
 			executerOpts := protocols.ExecuterOptions{
-				Output:       r.output,
-				Options:      r.options,
-				Progress:     r.progress,
-				Catalog:      r.catalog,
-				RateLimiter:  r.ratelimiter,
-				IssuesClient: r.issuesClient,
-				Browser:      r.browser,
-				ProjectFile:  r.projectFile,
+				Output:          r.output,
+				Options:         r.options,
+				Progress:        r.progress,
+				Catalog:         r.catalog,
+				RateLimiter:     r.ratelimiter,
+				IssuesClient:    r.issuesClient,
+				Browser:         r.browser,
+				ProjectFile:     r.projectFile,
+				Interactsh:      r.interactsh,
+				HostErrorsCache: cache,
 			}
 			clusterID := fmt.Sprintf("cluster-%s", xid.New().String())
 
@@ -302,11 +471,10 @@ func (r *Runner) RunEnumeration() {
 			finalTemplates = append(finalTemplates, cluster...)
 		}
 	}
-	for _, workflows := range availableWorkflows {
-		finalTemplates = append(finalTemplates, workflows)
-	}
 
-	var totalRequests int64 = 0
+	finalTemplates = append(finalTemplates, store.Workflows()...)
+
+	var totalRequests int64
 	for _, t := range finalTemplates {
 		if len(t.Workflows) > 0 {
 			continue
@@ -314,24 +482,23 @@ func (r *Runner) RunEnumeration() {
 		totalRequests += int64(t.TotalRequests) * r.inputCount
 	}
 	if totalRequests < unclusteredRequests {
-		gologger.Info().Msgf("Reduced %d requests to %d (%d templates clustered)", unclusteredRequests, totalRequests, clusterCount)
+		gologger.Info().Msgf("Templates clustered: %d (Reduced %d HTTP Requests)", clusterCount, unclusteredRequests-totalRequests)
 	}
-	templateCount := originalTemplatesCount + len(availableWorkflows)
+	workflowCount := len(store.Workflows())
+	templateCount := originalTemplatesCount + workflowCount
 
 	// 0 matches means no templates were found in directory
 	if templateCount == 0 {
-		gologger.Fatal().Msgf("Error, no templates were found.\n")
+		return errors.New("no valid templates were found")
 	}
 
-	gologger.Info().Msgf("Using %s rules (%s templates, %s workflows)",
-		r.colorizer.Bold(templateCount).String(),
-		r.colorizer.Bold(templateCount-workflowCount).String(),
-		r.colorizer.Bold(workflowCount).String())
+	/*
+		TODO does it make sense to run the logic below if there are no targets specified?
+		Can we safely assume the user is just experimenting with the template/workflow filters before running them?
+	*/
 
 	results := &atomic.Bool{}
 	wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
-	// Starts polling or ignore
-	collaborator.DefaultCollaborator.Poll()
 
 	// tracks global progress and captures stdout/stderr until p.Wait finishes
 	r.progress.Init(r.inputCount, templateCount, totalRequests)
@@ -349,27 +516,33 @@ func (r *Runner) RunEnumeration() {
 		}(t)
 	}
 	wgtemplates.Wait()
+
+	if r.interactsh != nil {
+		matched := r.interactsh.Close()
+		if matched {
+			results.CAS(false, true)
+		}
+	}
 	r.progress.Stop()
 
 	if r.issuesClient != nil {
 		r.issuesClient.Close()
 	}
 	if !results.Load() {
-		if r.output != nil {
-			r.output.Close()
-			os.Remove(r.options.Output)
-		}
 		gologger.Info().Msgf("No results found. Better luck next time!")
 	}
-
 	if r.browser != nil {
 		r.browser.Close()
 	}
+	return nil
 }
 
 // readNewTemplatesFile reads newly added templates from directory if it exists
 func (r *Runner) readNewTemplatesFile() ([]string, error) {
-	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	if r.templatesConfig == nil {
+		return nil, nil
+	}
+	additionsFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
 	file, err := os.Open(additionsFile)
 	if err != nil {
 		return nil, err
@@ -387,3 +560,27 @@ func (r *Runner) readNewTemplatesFile() ([]string, error) {
 	}
 	return templatesList, nil
 }
+
+// countNewTemplates returns the number of newly added templates
+func (r *Runner) countNewTemplates() int {
+	if r.templatesConfig == nil {
+		return 0
+	}
+	additionsFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	file, err := os.Open(additionsFile)
+	if err != nil {
+		return 0
+	}
+	defer file.Close()
+
+	count := 0
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text == "" {
+			continue
+		}
+		count++
+	}
+	return count
+}

v2/internal/runner/runner_test.go

@@ -0,0 +1,26 @@
+package runner
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+func Test_createReportingOptions(t *testing.T) {
+	var options types.Options
+	options.ReportingConfig = "../../../integration_tests/test-issue-tracker-config1.yaml"
+	resultOptions, err := createReportingOptions(&options)
+
+	assert.Nil(t, err)
+	assert.Equal(t, resultOptions.AllowList.Severities, severity.Severities{severity.High, severity.Critical})
+	assert.Equal(t, resultOptions.DenyList.Severities, severity.Severities{severity.Low})
+
+	options.ReportingConfig = "../../../integration_tests/test-issue-tracker-config2.yaml"
+	resultOptions2, err := createReportingOptions(&options)
+	assert.Nil(t, err)
+	assert.Equal(t, resultOptions2.AllowList.Severities, resultOptions.AllowList.Severities)
+	assert.Equal(t, resultOptions2.DenyList.Severities, resultOptions.DenyList.Severities)
+}

v2/internal/runner/templates.go

@@ -6,97 +6,52 @@ import (
 	"strings"
 
 	"github.com/karrick/godirwalk"
+
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
-// getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered
-// by severity, along with a flag indicating if workflows are present.
-func (r *Runner) getParsedTemplatesFor(templatePaths, severities []string, workflows bool) (parsedTemplates map[string]*templates.Template, workflowCount int) {
-	filterBySeverity := len(severities) > 0
-
-	if !workflows {
-		gologger.Info().Msgf("Loading templates...")
-	} else {
-		gologger.Info().Msgf("Loading workflows...")
-	}
-
-	parsedTemplates = make(map[string]*templates.Template)
-	for _, match := range templatePaths {
-		t, err := r.parseTemplateFile(match)
-		if err != nil {
-			gologger.Warning().Msgf("Could not parse file '%s': %s\n", match, err)
-			continue
-		}
-		if t == nil {
-			continue
-		}
-		if len(t.Workflows) == 0 && workflows {
-			continue // don't print if user only wants to run workflows
-		}
-		if len(t.Workflows) > 0 && !workflows {
-			continue // don't print workflow if user only wants to run templates
-		}
-		if len(t.Workflows) > 0 {
-			workflowCount++
-		}
-		sev := strings.ToLower(types.ToString(t.Info["severity"]))
-		if !filterBySeverity || hasMatchingSeverity(sev, severities) {
-			parsedTemplates[t.ID] = t
-			gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), sev))
-		} else {
-			gologger.Warning().Msgf("Excluding template %s due to severity filter (%s not in [%s])", t.ID, sev, severities)
-		}
-	}
-	return parsedTemplates, workflowCount
-}
-
-// parseTemplateFile returns the parsed template file
-func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
-	executerOpts := protocols.ExecuterOptions{
-		Output:       r.output,
-		Options:      r.options,
-		Progress:     r.progress,
-		Catalog:      r.catalog,
-		IssuesClient: r.issuesClient,
-		RateLimiter:  r.ratelimiter,
-		ProjectFile:  r.projectFile,
-		Browser:      r.browser,
-	}
-	template, err := templates.Parse(file, executerOpts)
-	if err != nil {
-		return nil, err
-	}
-	if template == nil {
-		return nil, nil
-	}
-	return template, nil
-}
-
-func (r *Runner) templateLogMsg(id, name, author, severity string) string {
+func (r *Runner) templateLogMsg(id, name string, authors []string, templateSeverity severity.Severity) string {
 	// Display the message for the template
-	message := fmt.Sprintf("[%s] %s (%s)",
+	return fmt.Sprintf("[%s] %s (%s) [%s]",
 		r.colorizer.BrightBlue(id).String(),
 		r.colorizer.Bold(name).String(),
-		r.colorizer.BrightYellow("@"+author).String())
-	if severity != "" {
-		message += " [" + r.severityColors.Data[severity] + "]"
+		r.colorizer.BrightYellow(appendAtSignToAuthors(authors)).String(),
+		r.addColor(templateSeverity))
+}
+
+// appendAtSignToAuthors appends @ before each author and returns the final string
+func appendAtSignToAuthors(authors []string) string {
+	if len(authors) == 0 {
+		return "@none"
 	}
-	return message
+
+	values := make([]string, 0, len(authors))
+	for _, k := range authors {
+		if !strings.HasPrefix(k, "@") {
+			values = append(values, fmt.Sprintf("@%s", k))
+		} else {
+			values = append(values, k)
+		}
+	}
+	return strings.Join(values, ",")
 }
 
 func (r *Runner) logAvailableTemplate(tplPath string) {
-	t, err := r.parseTemplateFile(tplPath)
+	t, err := parsers.ParseTemplate(tplPath)
 	if err != nil {
 		gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
 	} else {
-		gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
+		gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID,
+			types.ToString(t.Info.Name),
+			t.Info.Authors.ToSlice(),
+			t.Info.SeverityHolder.Severity))
 	}
 }
 
-// ListAvailableTemplates prints available templates to stdout
+// listAvailableTemplates prints available templates to stdout
 func (r *Runner) listAvailableTemplates() {
 	if r.templatesConfig == nil {
 		return
@@ -109,7 +64,7 @@ func (r *Runner) listAvailableTemplates() {
 
 	gologger.Print().Msgf(
 		"\nListing available v.%s nuclei templates for %s",
-		r.templatesConfig.CurrentVersion,
+		r.templatesConfig.TemplateVersion,
 		r.templatesConfig.TemplatesDirectory,
 	)
 	err := directoryWalker(
@@ -129,38 +84,12 @@ func (r *Runner) listAvailableTemplates() {
 	}
 }
 
-func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool {
-	for _, s := range allowedSeverities {
-		finalSeverities := []string{}
-		if strings.Contains(s, ",") {
-			finalSeverities = strings.Split(s, ",")
-		} else {
-			finalSeverities = append(finalSeverities, s)
-		}
-
-		for _, sev := range finalSeverities {
-			sev = strings.ToLower(sev)
-			if sev != "" && strings.HasPrefix(templateSeverity, sev) {
-				return true
-			}
-		}
-	}
-	return false
-}
-
 func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error {
-	err := godirwalk.Walk(fsPath, &godirwalk.Options{
+	return godirwalk.Walk(fsPath, &godirwalk.Options{
 		Callback: callback,
 		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
 			return godirwalk.SkipNode
 		},
 		Unsorted: true,
 	})
-
-	// directory couldn't be walked
-	if err != nil {
-		return err
-	}
-
-	return nil
 }

v2/internal/runner/templates_test.go

@@ -0,0 +1,25 @@
+package runner
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_appendAtSignToAuthors(t *testing.T) {
+	result := appendAtSignToAuthors([]string{"user1", "user2", "user3"})
+	assert.Equal(t, result, "@user1,@user2,@user3")
+}
+
+func Test_appendAtSignToMissingAuthors(t *testing.T) {
+	result := appendAtSignToAuthors([]string{})
+	assert.Equal(t, result, "@none")
+
+	result = appendAtSignToAuthors(nil)
+	assert.Equal(t, result, "@none")
+}
+
+func Test_appendAtSignToOneAuthor(t *testing.T) {
+	result := appendAtSignToAuthors([]string{"user1"})
+	assert.Equal(t, result, "@user1")
+}

v2/internal/runner/update.go

@@ -12,135 +12,117 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"path"
 	"path/filepath"
+	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
-	"time"
 
+	"github.com/apex/log"
 	"github.com/blang/semver"
-	"github.com/google/go-github/v32/github"
+	"github.com/google/go-github/github"
 	"github.com/olekukonko/tablewriter"
 	"github.com/pkg/errors"
+
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei-updatecheck-api/client"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+
+	"github.com/tj/go-update"
+	"github.com/tj/go-update/progress"
+	githubUpdateStore "github.com/tj/go-update/stores/github"
 )
 
 const (
-	userName = "projectdiscovery"
-	repoName = "nuclei-templates"
+	userName             = "projectdiscovery"
+	repoName             = "nuclei-templates"
+	nucleiIgnoreFile     = ".nuclei-ignore"
+	nucleiConfigFilename = ".templates-config.json"
 )
 
+var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
+
 // updateTemplates checks if the default list of nuclei-templates
-// exist in the users home directory, if not the latest revision
-// is downloaded from github.
+// exist in the user's home directory, if not the latest revision
+// is downloaded from GitHub.
 //
-// If the path exists but is not latest, the new version is downloaded
-// from github and replaced with the templates directory.
+// If the path exists but does not contain the latest version of public templates,
+// the new version is downloaded from GitHub to the templates' directory, overwriting the old content.
 func (r *Runner) updateTemplates() error {
 	home, err := os.UserHomeDir()
 	if err != nil {
 		return err
 	}
-	configDir := path.Join(home, "/.config", "/nuclei")
+	configDir := filepath.Join(home, ".config", "nuclei")
 	_ = os.MkdirAll(configDir, os.ModePerm)
 
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
-	if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
-		config, readErr := readConfiguration()
-		if err != nil {
-			return readErr
-		}
-		r.templatesConfig = config
+	if err := r.readInternalConfigurationFile(home, configDir); err != nil {
+		return errors.Wrap(err, "could not read configuration file")
 	}
 
-	ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
+	// If the config doesn't exist, create it now.
 	if r.templatesConfig == nil {
-		currentConfig := &nucleiConfig{
-			TemplatesDirectory: path.Join(home, "nuclei-templates"),
-			IgnoreURL:          ignoreURL,
-			NucleiVersion:      Version,
+		currentConfig := &config.Config{
+			TemplatesDirectory: filepath.Join(home, "nuclei-templates"),
+			NucleiVersion:      config.Version,
 		}
-		if writeErr := r.writeConfiguration(currentConfig); writeErr != nil {
+		if writeErr := config.WriteConfiguration(currentConfig); writeErr != nil {
 			return errors.Wrap(writeErr, "could not write template configuration")
 		}
 		r.templatesConfig = currentConfig
 	}
-	// Check if last checked for nuclei-ignore is more than 1 hours.
-	// and if true, run the check.
-	if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
-		if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" {
-			ignoreURL = r.templatesConfig.IgnoreURL
-		}
-		gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
 
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
-		if reqErr == nil {
-			resp, httpGet := http.DefaultClient.Do(req)
-			if httpGet != nil {
-				if resp != nil && resp.Body != nil {
-					resp.Body.Close()
-				}
-				gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
-			} else {
-				data, _ := ioutil.ReadAll(resp.Body)
-				resp.Body.Close()
-
-				if len(data) > 0 {
-					_ = ioutil.WriteFile(path.Join(configDir, nucleiIgnoreFile), data, 0644)
-				}
-				if r.templatesConfig != nil {
-					r.templatesConfig.LastCheckedIgnore = time.Now()
-				}
-			}
-		}
-		cancel()
+	if r.options.NoUpdateTemplates && !r.options.UpdateTemplates {
+		return nil
 	}
+	client.InitNucleiVersion(config.Version)
+	r.fetchLatestVersionsFromGithub(configDir) // also fetch the latest versions
 
 	ctx := context.Background()
-	if r.templatesConfig.CurrentVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
-		if !r.options.UpdateTemplates {
-			gologger.Warning().Msgf("nuclei-templates are not installed (or indexed), use update-templates flag.\n")
-			return nil
+
+	var noTemplatesFound bool
+	if _, err := os.Stat(r.templatesConfig.TemplatesDirectory); os.IsNotExist(err) {
+		noTemplatesFound = true
+	}
+
+	if r.templatesConfig.TemplateVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) || noTemplatesFound {
+		gologger.Info().Msgf("nuclei-templates are not installed, installing...\n")
+
+		// Use the custom location if the user has given a template directory
+		r.templatesConfig = &config.Config{
+			TemplatesDirectory: filepath.Join(home, "nuclei-templates"),
+		}
+		if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != filepath.Join(home, "nuclei-templates") {
+			r.templatesConfig.TemplatesDirectory, _ = filepath.Abs(r.options.TemplatesDirectory)
+		}
+		r.fetchLatestVersionsFromGithub(configDir) // also fetch the latest versions
+
+		version, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion)
+		if err != nil {
+			return err
 		}
 
-		// Use custom location if user has given a template directory
-		r.templatesConfig = &nucleiConfig{
-			TemplatesDirectory: path.Join(home, "nuclei-templates"),
-		}
-		if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") {
-			r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
-		}
-
-		// Download the repository and also write the revision to a HEAD file.
-		version, asset, getErr := r.getLatestReleaseFromGithub()
+		// Download the repository and write the revision to a HEAD file.
+		asset, getErr := r.getLatestReleaseFromGithub(r.templatesConfig.NucleiTemplatesLatestVersion)
 		if getErr != nil {
 			return getErr
 		}
 		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
 
-		_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
-		if err != nil {
+		if _, err := r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()); err != nil {
 			return err
 		}
-		r.templatesConfig.CurrentVersion = version.String()
+		r.templatesConfig.TemplateVersion = version.String()
 
-		err = r.writeConfiguration(r.templatesConfig)
-		if err != nil {
+		if err := config.WriteConfiguration(r.templatesConfig); err != nil {
 			return err
 		}
-		gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). Enjoy!\n", version.String())
-		return nil
-	}
-
-	// Check if last checked is more than 24 hours.
-	// If not, return since we don't want to do anything now.
-	if time.Since(r.templatesConfig.LastChecked) < 24*time.Hour && !r.options.UpdateTemplates {
+		gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). GoodLuck!\n", version.String())
 		return nil
 	}
 
 	// Get the configuration currently on disk.
-	verText := r.templatesConfig.CurrentVersion
+	verText := r.templatesConfig.TemplateVersion
 	indices := reVersion.FindStringIndex(verText)
 	if indices == nil {
 		return fmt.Errorf("invalid release found with tag %s", err)
@@ -154,77 +136,90 @@ func (r *Runner) updateTemplates() error {
 		return err
 	}
 
-	version, asset, err := r.getLatestReleaseFromGithub()
+	version, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion)
 	if err != nil {
 		return err
 	}
 
 	if version.EQ(oldVersion) {
-		gologger.Info().Msgf("Your nuclei-templates are up to date: v%s\n", oldVersion.String())
-		return r.writeConfiguration(r.templatesConfig)
+		if r.options.UpdateTemplates {
+			gologger.Info().Msgf("No new updates found for nuclei templates")
+		}
+		return config.WriteConfiguration(r.templatesConfig)
 	}
 
 	if version.GT(oldVersion) {
-		if !r.options.UpdateTemplates {
-			gologger.Warning().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
-			return r.writeConfiguration(r.templatesConfig)
-		}
+		gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
+		gologger.Info().Msgf("Downloading latest release...")
 
 		if r.options.TemplatesDirectory != "" {
 			r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
 		}
-		r.templatesConfig.CurrentVersion = version.String()
+		r.templatesConfig.TemplateVersion = version.String()
 
 		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
-		_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
+
+		asset, err := r.getLatestReleaseFromGithub(r.templatesConfig.NucleiTemplatesLatestVersion)
 		if err != nil {
 			return err
 		}
-		err = r.writeConfiguration(r.templatesConfig)
-		if err != nil {
+		if _, err := r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()); err != nil {
 			return err
 		}
-		gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). Enjoy!\n", version.String())
+		if err := config.WriteConfiguration(r.templatesConfig); err != nil {
+			return err
+		}
+		gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", version.String())
 	}
 	return nil
 }
 
-// getLatestReleaseFromGithub returns the latest release from github
-func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.RepositoryRelease, error) {
+// readInternalConfigurationFile reads the internal configuration file for nuclei
+func (r *Runner) readInternalConfigurationFile(home, configDir string) error {
+	templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename)
+	if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
+		configuration, readErr := config.ReadConfiguration()
+		if readErr != nil {
+			return readErr
+		}
+		r.templatesConfig = configuration
+
+		if configuration.TemplatesDirectory != "" && configuration.TemplatesDirectory != filepath.Join(home, "nuclei-templates") {
+			r.options.TemplatesDirectory = configuration.TemplatesDirectory
+		}
+	}
+	return nil
+}
+
+// checkNucleiIgnoreFileUpdates checks .nuclei-ignore file for updates from GitHub
+func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool {
+	data, err := client.GetLatestIgnoreFile()
+	if err != nil {
+		return false
+	}
+	if len(data) > 0 {
+		_ = ioutil.WriteFile(filepath.Join(configDir, nucleiIgnoreFile), data, 0644)
+	}
+	if r.templatesConfig != nil {
+		if err := config.WriteConfiguration(r.templatesConfig); err != nil {
+			gologger.Warning().Msgf("Could not get ignore-file from server: %s", err)
+		}
+	}
+	return true
+}
+
+// getLatestReleaseFromGithub returns the latest release from GitHub
+func (r *Runner) getLatestReleaseFromGithub(latestTag string) (*github.RepositoryRelease, error) {
 	client := github.NewClient(nil)
 
-	rels, _, err := client.Repositories.ListReleases(context.Background(), userName, repoName, nil)
+	release, _, err := client.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag)
 	if err != nil {
-		return semver.Version{}, nil, err
+		return nil, err
 	}
-
-	// Find the most recent version based on semantic versioning.
-	var latestRelease semver.Version
-	var latestPublish *github.RepositoryRelease
-	for _, release := range rels {
-		verText := release.GetTagName()
-		indices := reVersion.FindStringIndex(verText)
-		if indices == nil {
-			return semver.Version{}, nil, fmt.Errorf("invalid release found with tag %s", err)
-		}
-		if indices[0] > 0 {
-			verText = verText[indices[0]:]
-		}
-
-		ver, err := semver.Make(verText)
-		if err != nil {
-			return semver.Version{}, nil, err
-		}
-
-		if latestPublish == nil || ver.GTE(latestRelease) {
-			latestRelease = ver
-			latestPublish = release
-		}
+	if release == nil {
+		return nil, errors.New("no version found for the templates")
 	}
-	if latestPublish == nil {
-		return semver.Version{}, nil, errors.New("no version found for the templates")
-	}
-	return latestRelease, latestPublish, nil
+	return release, nil
 }
 
 // downloadReleaseAndUnzip downloads and unzips the release in a directory
@@ -249,38 +244,38 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU
 	}
 
 	reader := bytes.NewReader(buf)
-	z, err := zip.NewReader(reader, reader.Size())
+	zipReader, err := zip.NewReader(reader, reader.Size())
 	if err != nil {
 		return nil, fmt.Errorf("failed to uncompress zip file: %s", err)
 	}
 
-	// Create the template folder if it doesn't exists
-	err = os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm)
-	if err != nil {
+	// Create the template folder if it doesn't exist
+	if err := os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm); err != nil {
 		return nil, fmt.Errorf("failed to create template base folder: %s", err)
 	}
 
-	results, err := r.compareAndWriteTemplates(z)
+	results, err := r.compareAndWriteTemplates(zipReader)
 	if err != nil {
 		return nil, fmt.Errorf("failed to write templates: %s", err)
 	}
 
-	r.printUpdateChangelog(results, version)
-	checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
-	err = writeTemplatesChecksum(checksumFile, results.checksums)
-	if err != nil {
+	if r.options.Verbose {
+		r.printUpdateChangelog(results, version)
+	}
+	checksumFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
+	if err := writeTemplatesChecksum(checksumFile, results.checksums); err != nil {
 		return nil, errors.Wrap(err, "could not write checksum")
 	}
 
 	// Write the additions to a cached file for new runs.
-	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	additionsFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
 	buffer := &bytes.Buffer{}
 	for _, addition := range results.additions {
 		buffer.WriteString(addition)
 		buffer.WriteString("\n")
 	}
-	err = ioutil.WriteFile(additionsFile, buffer.Bytes(), os.ModePerm)
-	if err != nil {
+
+	if err := ioutil.WriteFile(additionsFile, buffer.Bytes(), os.ModePerm); err != nil {
 		return nil, errors.Wrap(err, "could not write new additions file")
 	}
 	return results, err
@@ -294,9 +289,8 @@ type templateUpdateResults struct {
 	checksums     map[string]string
 }
 
-// compareAndWriteTemplates compares and returns the stats of a template
-// update operations.
-func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults, error) {
+// compareAndWriteTemplates compares and returns the stats of a template update operations.
+func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdateResults, error) {
 	results := &templateUpdateResults{
 		checksums: make(map[string]string),
 	}
@@ -306,115 +300,114 @@ func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults
 	// If the path isn't found in new update after being read from the previous checksum,
 	// it is removed. This allows us fine-grained control over the download process
 	// as well as solves a long problem with nuclei-template updates.
-	checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
-	previousChecksum, _ := readPreviousTemplatesChecksum(checksumFile)
-	for _, file := range z.File {
-		directory, name := filepath.Split(file.Name)
+	checksumFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
+	templateChecksumsMap, _ := createTemplateChecksumsMap(checksumFile)
+	for _, zipTemplateFile := range zipReader.File {
+		directory, name := filepath.Split(zipTemplateFile.Name)
 		if name == "" {
 			continue
 		}
-		paths := strings.Split(directory, "/")
-		finalPath := strings.Join(paths[1:], "/")
+		paths := strings.Split(directory, string(os.PathSeparator))
+		finalPath := filepath.Join(paths[1:]...)
 
-		if (!strings.EqualFold(name, ".nuclei-ignore") && strings.HasPrefix(name, ".")) || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") {
+		if strings.HasPrefix(name, ".") || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") {
 			continue
 		}
 		results.totalCount++
-		templateDirectory := path.Join(r.templatesConfig.TemplatesDirectory, finalPath)
-		err := os.MkdirAll(templateDirectory, os.ModePerm)
-		if err != nil {
+		templateDirectory := filepath.Join(r.templatesConfig.TemplatesDirectory, finalPath)
+		if err := os.MkdirAll(templateDirectory, os.ModePerm); err != nil {
 			return nil, fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err)
 		}
 
-		templatePath := path.Join(templateDirectory, name)
+		templatePath := filepath.Join(templateDirectory, name)
 
 		isAddition := false
 		if _, statErr := os.Stat(templatePath); os.IsNotExist(statErr) {
 			isAddition = true
 		}
-		f, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777)
+		templateFile, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777)
 		if err != nil {
-			f.Close()
+			templateFile.Close()
 			return nil, fmt.Errorf("could not create uncompressed file: %s", err)
 		}
 
-		reader, err := file.Open()
+		zipTemplateFileReader, err := zipTemplateFile.Open()
 		if err != nil {
-			f.Close()
+			templateFile.Close()
 			return nil, fmt.Errorf("could not open archive to extract file: %s", err)
 		}
 		hasher := md5.New()
 
 		// Save file and also read into hasher for md5
-		_, err = io.Copy(f, io.TeeReader(reader, hasher))
-		if err != nil {
-			f.Close()
+		if _, err := io.Copy(templateFile, io.TeeReader(zipTemplateFileReader, hasher)); err != nil {
+			templateFile.Close()
 			return nil, fmt.Errorf("could not write template file: %s", err)
 		}
-		f.Close()
+		templateFile.Close()
 
-		oldChecksum, checksumOK := previousChecksum[templatePath]
+		oldChecksum, checksumOK := templateChecksumsMap[templatePath]
 
 		checksum := hex.EncodeToString(hasher.Sum(nil))
 		if isAddition {
-			results.additions = append(results.additions, path.Join(finalPath, name))
+			results.additions = append(results.additions, filepath.Join(finalPath, name))
 		} else if checksumOK && oldChecksum[0] != checksum {
-			results.modifications = append(results.modifications, path.Join(finalPath, name))
+			results.modifications = append(results.modifications, filepath.Join(finalPath, name))
 		}
 		results.checksums[templatePath] = checksum
 	}
 
-	// If we don't find a previous file in new download and it hasn't been
-	// changed on the disk, delete it.
-	for k, v := range previousChecksum {
-		_, ok := results.checksums[k]
-		if !ok && v[0] == v[1] {
-			os.Remove(k)
-			results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(k, r.templatesConfig.TemplatesDirectory), "/"))
+	// If we don't find the previous file in the newly downloaded list,
+	// and it hasn't been changed on the disk, delete it.
+	for templatePath, templateChecksums := range templateChecksumsMap {
+		_, ok := results.checksums[templatePath]
+		if !ok && templateChecksums[0] == templateChecksums[1] {
+			os.Remove(templatePath)
+			results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(templatePath, r.templatesConfig.TemplatesDirectory), string(os.PathSeparator)))
 		}
 	}
 	return results, nil
 }
 
-// readPreviousTemplatesChecksum reads the previous checksum file from the disk.
-//
-// It reads two checksums, the first checksum is what we expect and the second is
-// the actual checksum of the file on disk currently.
-func readPreviousTemplatesChecksum(file string) (map[string][2]string, error) {
-	f, err := os.Open(file)
+// createTemplateChecksumsMap reads the previous checksum file from the disk.
+// Creates a map of template paths and their previous and currently calculated checksums as values.
+func createTemplateChecksumsMap(checksumsFilePath string) (map[string][2]string, error) {
+	checksumFile, err := os.Open(checksumsFilePath)
 	if err != nil {
 		return nil, err
 	}
-	defer f.Close()
-	scanner := bufio.NewScanner(f)
+	defer checksumFile.Close()
+	scanner := bufio.NewScanner(checksumFile)
 
-	checksum := make(map[string][2]string)
+	templatePathChecksumsMap := make(map[string][2]string)
 	for scanner.Scan() {
 		text := scanner.Text()
 		if text == "" {
 			continue
 		}
+
 		parts := strings.Split(text, ",")
 		if len(parts) < 2 {
 			continue
 		}
-		values := [2]string{parts[1]}
+		templatePath := parts[0]
+		expectedTemplateChecksum := parts[1]
 
-		f, err := os.Open(parts[0])
+		templateFile, err := os.Open(templatePath)
 		if err != nil {
 			return nil, err
 		}
 
 		hasher := md5.New()
-		if _, err := io.Copy(hasher, f); err != nil {
+		if _, err := io.Copy(hasher, templateFile); err != nil {
 			return nil, err
 		}
-		f.Close()
+		templateFile.Close()
 
+		values := [2]string{expectedTemplateChecksum}
 		values[1] = hex.EncodeToString(hasher.Sum(nil))
-		checksum[parts[0]] = values
+		templatePathChecksumsMap[templatePath] = values
 	}
-	return checksum, nil
+	return templatePathChecksumsMap, nil
 }
 
 // writeTemplatesChecksum writes the nuclei-templates checksum data to disk.
@@ -441,7 +434,7 @@ func writeTemplatesChecksum(file string, checksum map[string]string) error {
 }
 
 func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version string) {
-	if len(results.additions) > 0 {
+	if len(results.additions) > 0 && r.options.Verbose {
 		gologger.Print().Msgf("\nNewly added templates: \n\n")
 
 		for _, addition := range results.additions {
@@ -460,3 +453,77 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st
 	}
 	table.Render()
 }
+
+// fetchLatestVersionsFromGithub fetches the latest versions of nuclei repos from GitHub
+//
+// This fetches latest nuclei/templates/ignore from https://version-check.nuclei.sh/versions
+// If you want to disable this automatic update check, use -nut flag.
+func (r *Runner) fetchLatestVersionsFromGithub(configDir string) {
+	versions, err := client.GetLatestNucleiTemplatesVersion()
+	if err != nil {
+		gologger.Warning().Msgf("Could not fetch latest releases: %s", err)
+		return
+	}
+	if r.templatesConfig != nil {
+		r.templatesConfig.NucleiLatestVersion = versions.Nuclei
+		r.templatesConfig.NucleiTemplatesLatestVersion = versions.Templates
+
+		// If the fetch has resulted in new version of ignore file, update.
+		if r.templatesConfig.NucleiIgnoreHash == "" || r.templatesConfig.NucleiIgnoreHash != versions.IgnoreHash {
+			r.templatesConfig.NucleiIgnoreHash = versions.IgnoreHash
+			r.checkNucleiIgnoreFileUpdates(configDir)
+		}
+	}
+}
+
+// updateNucleiVersionToLatest implements nuclei auto-update using GitHub Releases.
+func updateNucleiVersionToLatest(verbose bool) error {
+	if verbose {
+		log.SetLevel(log.DebugLevel)
+	}
+	var command string
+	switch runtime.GOOS {
+	case "windows":
+		command = "nuclei.exe"
+	default:
+		command = "nuclei"
+	}
+	m := &update.Manager{
+		Command: command,
+		Store: &githubUpdateStore.Store{
+			Owner:   "projectdiscovery",
+			Repo:    "nuclei",
+			Version: config.Version,
+		},
+	}
+	releases, err := m.LatestReleases()
+	if err != nil {
+		return errors.Wrap(err, "could not fetch latest release")
+	}
+	if len(releases) == 0 {
+		gologger.Info().Msgf("No new updates found for nuclei engine!")
+		return nil
+	}
+
+	latest := releases[0]
+	var currentOS string
+	switch runtime.GOOS {
+	case "darwin":
+		currentOS = "macOS"
+	default:
+		currentOS = runtime.GOOS
+	}
+	final := latest.FindZip(currentOS, runtime.GOARCH)
+	if final == nil {
+		return fmt.Errorf("no compatible binary found for %s/%s", currentOS, runtime.GOARCH)
+	}
+	tarball, err := final.DownloadProxy(progress.Reader)
+	if err != nil {
+		return errors.Wrap(err, "could not download latest release")
+	}
+	if err := m.Install(tarball); err != nil {
+		return errors.Wrap(err, "could not install latest release")
+	}
+	gologger.Info().Msgf("Successfully updated to Nuclei %s\n", latest.Version)
+	return nil
+}

v2/internal/runner/update_test.go

@@ -8,13 +8,13 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 	"testing"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/stretchr/testify/require"
 )
 
@@ -25,7 +25,7 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
 	require.Nil(t, err, "could not create temp directory")
 	defer os.RemoveAll(baseTemplates)
 
-	err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
+	err = ioutil.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
 	require.Nil(t, err, "could not create write base file")
 
 	err = zipFromDirectory("base.zip", baseTemplates)
@@ -41,8 +41,7 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
 	require.Nil(t, err, "could not create temp directory")
 	defer os.RemoveAll(templatesDirectory)
 
-	r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}}
-
+	r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}, options: testutils.DefaultOptions}
 	results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
 	require.Nil(t, err, "could not download release and unzip")
 	require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")
@@ -51,9 +50,9 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
 	require.Nil(t, err, "could not create temp directory")
 	defer os.RemoveAll(newTempDir)
 
-	err = ioutil.WriteFile(path.Join(newTempDir, "base.yaml"), []byte("id: test"), 0777)
+	err = ioutil.WriteFile(filepath.Join(newTempDir, "base.yaml"), []byte("id: test"), 0777)
 	require.Nil(t, err, "could not create base file")
-	err = ioutil.WriteFile(path.Join(newTempDir, "new.yaml"), []byte("id: test"), 0777)
+	err = ioutil.WriteFile(filepath.Join(newTempDir, "new.yaml"), []byte("id: test"), 0777)
 	require.Nil(t, err, "could not create new file")
 
 	err = zipFromDirectory("new.zip", newTempDir)
@@ -78,7 +77,7 @@ func TestDownloadReleaseAndUnzipDeletion(t *testing.T) {
 	require.Nil(t, err, "could not create temp directory")
 	defer os.RemoveAll(baseTemplates)
 
-	err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
+	err = ioutil.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
 	require.Nil(t, err, "could not create write base file")
 
 	err = zipFromDirectory("base.zip", baseTemplates)
@@ -94,7 +93,7 @@ func TestDownloadReleaseAndUnzipDeletion(t *testing.T) {
 	require.Nil(t, err, "could not create temp directory")
 	defer os.RemoveAll(templatesDirectory)
 
-	r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}}
+	r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}, options: testutils.DefaultOptions}
 
 	results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
 	require.Nil(t, err, "could not download release and unzip")

v2/internal/testutils/integration.go

@@ -1,15 +1,17 @@
 package testutils
 
 import (
+	"errors"
 	"net"
 	"os"
 	"os/exec"
+	"regexp"
 	"strings"
 )
 
 // RunNucleiAndGetResults returns a list of results for a template
 func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
-	cmd := exec.Command("./nuclei", "-t", template, "-target", url)
+	cmd := exec.Command("./nuclei", "-t", template, "-target", url, "-silent")
 	if debug {
 		cmd = exec.Command("./nuclei", "-t", template, "-target", url, "-debug")
 		cmd.Stderr = os.Stderr
@@ -30,9 +32,26 @@ func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) (
 	return parts, nil
 }
 
+var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded[^:]*: (\d+)`)
+
+// RunNucleiBinaryAndGetLoadedTemplates returns a list of results for a template
+func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, args []string) (string, error) {
+	cmd := exec.Command(nucleiBinary, args...)
+
+	data, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", err
+	}
+	matches := templateLoaded.FindAllStringSubmatch(string(data), -1)
+	if len(matches) == 0 {
+		return "", errors.New("no matches found")
+	}
+	return matches[0][1], nil
+}
+
 // RunNucleiWorkflowAndGetResults returns a list of results for a workflow
 func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
-	cmd := exec.Command("./nuclei", "-w", template, "-target", url)
+	cmd := exec.Command("./nuclei", "-w", template, "-target", url, "-silent")
 	if debug {
 		cmd = exec.Command("./nuclei", "-w", template, "-target", url, "-debug")
 		cmd.Stderr = os.Stderr

v2/internal/testutils/testutils.go

@@ -2,14 +2,17 @@ package testutils
 
 import (
 	"github.com/logrusorgru/aurora"
+	"go.uber.org/ratelimit"
+
 	"github.com/projectdiscovery/gologger/levels"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
-	"go.uber.org/ratelimit"
 )
 
 // Init initializes the protocols and their configurations
@@ -19,44 +22,42 @@ func Init(options *types.Options) {
 
 // DefaultOptions is the default options structure for nuclei during mocking.
 var DefaultOptions = &types.Options{
-	RandomAgent:          false,
-	Metrics:              false,
-	Debug:                false,
-	DebugRequests:        false,
-	DebugResponse:        false,
-	Silent:               false,
-	Version:              false,
-	Verbose:              false,
-	NoColor:              true,
-	UpdateTemplates:      false,
-	JSON:                 false,
-	JSONRequests:         false,
-	EnableProgressBar:    false,
-	TemplatesVersion:     false,
-	TemplateList:         false,
-	Stdin:                false,
-	StopAtFirstMatch:     false,
-	NoMeta:               false,
-	Project:              false,
-	MetricsPort:          0,
-	BulkSize:             25,
-	TemplateThreads:      10,
-	Timeout:              5,
-	Retries:              1,
-	RateLimit:            150,
-	BurpCollaboratorBiid: "",
-	ProjectPath:          "",
-	Severity:             []string{},
-	Target:               "",
-	Targets:              "",
-	Output:               "",
-	ProxyURL:             "",
-	ProxySocksURL:        "",
-	TemplatesDirectory:   "",
-	TraceLogFile:         "",
-	Templates:            []string{},
-	ExcludedTemplates:    []string{},
-	CustomHeaders:        []string{},
+	Metrics:            false,
+	Debug:              false,
+	DebugRequests:      false,
+	DebugResponse:      false,
+	Silent:             false,
+	Version:            false,
+	Verbose:            false,
+	NoColor:            true,
+	UpdateTemplates:    false,
+	JSON:               false,
+	JSONRequests:       false,
+	EnableProgressBar:  false,
+	TemplatesVersion:   false,
+	TemplateList:       false,
+	Stdin:              false,
+	StopAtFirstMatch:   false,
+	NoMeta:             false,
+	Project:            false,
+	MetricsPort:        0,
+	BulkSize:           25,
+	TemplateThreads:    10,
+	Timeout:            5,
+	Retries:            1,
+	RateLimit:          150,
+	ProjectPath:        "",
+	Severities:         severity.Severities{},
+	Targets:            []string{},
+	TargetsFilePath:    "",
+	Output:             "",
+	ProxyURL:           "",
+	ProxySocksURL:      "",
+	TemplatesDirectory: "",
+	TraceLogFile:       "",
+	Templates:          []string{},
+	ExcludedTemplates:  []string{},
+	CustomHeaders:      []string{},
 }
 
 // MockOutputWriter is a mocked output writer.
@@ -97,13 +98,13 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro
 // TemplateInfo contains info for a mock executed template.
 type TemplateInfo struct {
 	ID   string
-	Info map[string]interface{}
+	Info model.Info
 	Path string
 }
 
 // NewMockExecuterOptions creates a new mock executeroptions struct
 func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecuterOptions {
-	progressImpl, _ := progress.NewStatsTicker(0, false, false, 0)
+	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
 	executerOpts := &protocols.ExecuterOptions{
 		TemplateID:   info.ID,
 		TemplateInfo: info.Info,

v2/pkg/catalog/catalogue.go

@@ -2,13 +2,11 @@ package catalog
 
 // Catalog is a template catalog helper implementation
 type Catalog struct {
-	ignoreFiles        []string
 	templatesDirectory string
 }
 
 // New creates a new Catalog structure using provided input items
 func New(directory string) *Catalog {
 	catalog := &Catalog{templatesDirectory: directory}
-	catalog.readNucleiIgnoreFile()
 	return catalog
 }

v2/pkg/catalog/config/config.go

@@ -0,0 +1,125 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+	"gopkg.in/yaml.v2"
+)
+
+// Config contains the internal nuclei engine configuration
+type Config struct {
+	TemplatesDirectory string `json:"nuclei-templates-directory,omitempty"`
+	TemplateVersion    string `json:"nuclei-templates-version,omitempty"`
+	NucleiVersion      string `json:"nuclei-version,omitempty"`
+	NucleiIgnoreHash   string `json:"nuclei-ignore-hash,omitempty"`
+
+	NucleiLatestVersion          string `json:"nuclei-latest-version"`
+	NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"`
+}
+
+// nucleiConfigFilename is the filename of nuclei configuration file.
+const nucleiConfigFilename = ".templates-config.json"
+
+// Version is the current version of nuclei
+const Version = `2.5.2`
+
+func getConfigDetails() (string, error) {
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		return "", errors.Wrap(err, "could not get home directory")
+	}
+	configDir := filepath.Join(homeDir, ".config", "nuclei")
+	_ = os.MkdirAll(configDir, os.ModePerm)
+	templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename)
+	return templatesConfigFile, nil
+}
+
+// ReadConfiguration reads the nuclei configuration file from disk.
+func ReadConfiguration() (*Config, error) {
+	templatesConfigFile, err := getConfigDetails()
+	if err != nil {
+		return nil, err
+	}
+
+	file, err := os.Open(templatesConfigFile)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	config := &Config{}
+	if err := jsoniter.NewDecoder(file).Decode(config); err != nil {
+		return nil, err
+	}
+	return config, nil
+}
+
+// WriteConfiguration writes the updated nuclei configuration to disk
+func WriteConfiguration(config *Config) error {
+	config.NucleiVersion = Version
+
+	templatesConfigFile, err := getConfigDetails()
+	if err != nil {
+		return err
+	}
+	file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	err = jsoniter.NewEncoder(file).Encode(config)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+const nucleiIgnoreFile = ".nuclei-ignore"
+
+// IgnoreFile is an internal nuclei template blocking configuration file
+type IgnoreFile struct {
+	Tags  []string `yaml:"tags"`
+	Files []string `yaml:"files"`
+}
+
+// ReadIgnoreFile reads the nuclei ignore file returning blocked tags and paths
+func ReadIgnoreFile() IgnoreFile {
+	file, err := os.Open(getIgnoreFilePath())
+	if err != nil {
+		gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err)
+		return IgnoreFile{}
+	}
+	defer file.Close()
+
+	ignore := IgnoreFile{}
+	if err := yaml.NewDecoder(file).Decode(&ignore); err != nil {
+		gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err)
+		return IgnoreFile{}
+	}
+	return ignore
+}
+
+// getIgnoreFilePath returns the ignore file path for the runner
+func getIgnoreFilePath() string {
+	var defIgnoreFilePath string
+
+	home, err := os.UserHomeDir()
+	if err == nil {
+		configDir := filepath.Join(home, ".config", "nuclei")
+		_ = os.MkdirAll(configDir, os.ModePerm)
+
+		defIgnoreFilePath = filepath.Join(configDir, nucleiIgnoreFile)
+		return defIgnoreFilePath
+	}
+	cwd, err := os.Getwd()
+	if err != nil {
+		return defIgnoreFilePath
+	}
+	cwdIgnoreFilePath := filepath.Join(cwd, nucleiIgnoreFile)
+	return cwdIgnoreFilePath
+}

v2/pkg/catalog/find.go

@@ -2,7 +2,6 @@ package catalog
 
 import (
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 
@@ -29,9 +28,6 @@ func (c *Catalog) GetTemplatesPath(definitions []string) []string {
 			}
 		}
 	}
-	if len(allTemplates) > 0 {
-		gologger.Verbose().Msgf("Identified %d templates", len(allTemplates))
-	}
 	return allTemplates
 }
 
@@ -86,12 +82,12 @@ func (c *Catalog) GetTemplatePath(target string) ([]string, error) {
 // before doing any operations on them regardless of them being blob, folders, files, etc.
 func (c *Catalog) convertPathToAbsolute(t string) (string, error) {
 	if strings.Contains(t, "*") {
-		file := path.Base(t)
-		absPath, err := c.ResolvePath(path.Dir(t), "")
+		file := filepath.Base(t)
+		absPath, err := c.ResolvePath(filepath.Dir(t), "")
 		if err != nil {
 			return "", err
 		}
-		return path.Join(absPath, file), nil
+		return filepath.Join(absPath, file), nil
 	}
 	return c.ResolvePath(t, "")
 }
@@ -139,10 +135,6 @@ func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]stru
 		},
 		Callback: func(path string, d *godirwalk.Dirent) error {
 			if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
-				if c.checkIfInNucleiIgnore(path) {
-					return nil
-				}
-
 				if _, ok := processed[path]; !ok {
 					results = append(results, path)
 					processed[path] = struct{}{}

v2/pkg/catalog/ignore.go

@@ -1,84 +0,0 @@
-package catalog
-
-import (
-	"bufio"
-	"os"
-	"path"
-	"strings"
-
-	"github.com/projectdiscovery/gologger"
-)
-
-const nucleiIgnoreFile = ".nuclei-ignore"
-
-// readNucleiIgnoreFile reads the nuclei ignore file marking it in map
-func (c *Catalog) readNucleiIgnoreFile() {
-	file, err := os.Open(path.Join(c.templatesDirectory, nucleiIgnoreFile))
-	if err != nil {
-		return
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		text := scanner.Text()
-		if text == "" {
-			continue
-		}
-		if strings.HasPrefix(text, "#") {
-			continue
-		}
-		c.ignoreFiles = append(c.ignoreFiles, text)
-	}
-}
-
-// checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules.
-func (c *Catalog) checkIfInNucleiIgnore(item string) bool {
-	if c.templatesDirectory == "" {
-		return false
-	}
-
-	matched := false
-	for _, paths := range c.ignoreFiles {
-		if !strings.HasSuffix(paths, ".yaml") {
-			if strings.HasSuffix(strings.TrimSuffix(item, "/"), strings.TrimSuffix(paths, "/")) {
-				matched = true
-				break
-			}
-		} else if strings.HasSuffix(item, paths) {
-			matched = true
-			break
-		}
-	}
-	if matched {
-		gologger.Error().Msgf("Excluding %s due to nuclei-ignore filter", item)
-		return true
-	}
-	return false
-}
-
-// ignoreFilesWithExcludes ignores results with exclude paths
-func (c *Catalog) ignoreFilesWithExcludes(results, excluded []string) []string {
-	var templates []string
-
-	for _, result := range results {
-		matched := false
-		for _, paths := range excluded {
-			if !strings.HasSuffix(paths, ".yaml") {
-				if strings.HasSuffix(strings.TrimSuffix(result, "/"), strings.TrimSuffix(paths, "/")) {
-					matched = true
-					break
-				}
-			} else if strings.HasSuffix(result, paths) {
-				matched = true
-				break
-			}
-		}
-		if !matched {
-			templates = append(templates, result)
-		} else {
-			gologger.Error().Msgf("Excluding %s due to excludes filter", result)
-		}
-	}
-	return templates
-}

v2/pkg/catalog/ignore_test.go

@@ -1,40 +0,0 @@
-package catalog
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestIgnoreFilesIgnore(t *testing.T) {
-	c := &Catalog{
-		ignoreFiles:        []string{"workflows/", "cves/2020/cve-2020-5432.yaml"},
-		templatesDirectory: "test",
-	}
-	tests := []struct {
-		path   string
-		ignore bool
-	}{
-		{"workflows/", true},
-		{"misc", false},
-		{"cves/", false},
-		{"cves/2020/cve-2020-5432.yaml", true},
-		{"/Users/test/nuclei-templates/workflows/", true},
-		{"/Users/test/nuclei-templates/misc", false},
-		{"/Users/test/nuclei-templates/cves/", false},
-		{"/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", true},
-	}
-	for _, test := range tests {
-		require.Equal(t, test.ignore, c.checkIfInNucleiIgnore(test.path), fmt.Sprintf("could not ignore file correctly: %v", test))
-	}
-}
-
-func TestExcludeFilesIgnore(t *testing.T) {
-	c := &Catalog{}
-	excludes := []string{"workflows/", "cves/2020/cve-2020-5432.yaml"}
-	paths := []string{"/Users/test/nuclei-templates/workflows/", "/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", "/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"}
-
-	data := c.ignoreFilesWithExcludes(paths, excludes)
-	require.Equal(t, []string{"/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"}, data, "could not exclude correct files")
-}

v2/pkg/catalog/loader/filter/path_filter.go

@@ -0,0 +1,45 @@
+package filter
+
+import "github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+
+// PathFilter is a path based template filter
+type PathFilter struct {
+	excludedTemplates          []string
+	alwaysIncludedTemplatesMap map[string]struct{}
+}
+
+// PathFilterConfig contains configuration options for Path based templates Filter
+type PathFilterConfig struct {
+	IncludedTemplates []string
+	ExcludedTemplates []string
+}
+
+// NewPathFilter creates a new path filter from provided config
+func NewPathFilter(config *PathFilterConfig, catalogClient *catalog.Catalog) *PathFilter {
+	filter := &PathFilter{
+		excludedTemplates:          catalogClient.GetTemplatesPath(config.ExcludedTemplates),
+		alwaysIncludedTemplatesMap: make(map[string]struct{}),
+	}
+
+	alwaysIncludeTemplates := catalogClient.GetTemplatesPath(config.IncludedTemplates)
+	for _, tpl := range alwaysIncludeTemplates {
+		filter.alwaysIncludedTemplatesMap[tpl] = struct{}{}
+	}
+	return filter
+}
+
+// Match performs match for path filter on templates and returns final list
+func (p *PathFilter) Match(templates []string) map[string]struct{} {
+	templatesMap := make(map[string]struct{})
+	for _, tpl := range templates {
+		templatesMap[tpl] = struct{}{}
+	}
+	for _, template := range p.excludedTemplates {
+		if _, ok := p.alwaysIncludedTemplatesMap[template]; ok {
+			continue
+		} else {
+			delete(templatesMap, template)
+		}
+	}
+	return templatesMap
+}

v2/pkg/catalog/loader/filter/tag_filter.go

@@ -0,0 +1,194 @@
+package filter
+
+import (
+	"errors"
+	"strings"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+)
+
+// TagFilter is used to filter nuclei templates for tag based execution
+type TagFilter struct {
+	allowedTags map[string]struct{}
+	severities  map[severity.Severity]struct{}
+	authors     map[string]struct{}
+	block       map[string]struct{}
+	matchAllows map[string]struct{}
+}
+
+// ErrExcluded is returned for excluded templates
+var ErrExcluded = errors.New("the template was excluded")
+
+// Match filters templates based on user provided tags, authors, extraTags and severity.
+// If the template contains tags specified in the deny-list, it will not be matched
+// unless it is explicitly specified by user using the includeTags (matchAllows field).
+// Matching rule: (tag1 OR tag2...) AND (author1 OR author2...) AND (severity1 OR severity2...) AND (extraTags1 OR extraTags2...)
+// Returns true if the template matches the filter criteria, false otherwise.
+func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string) (bool, error) {
+	for _, templateTag := range templateTags {
+		_, blocked := tagFilter.block[templateTag]
+		_, allowed := tagFilter.matchAllows[templateTag]
+
+		if blocked && !allowed { // the whitelist has precedence over the blacklist
+			return false, ErrExcluded
+		}
+	}
+
+	if !isExtraTagMatch(extraTags, templateTags) {
+		return false, nil
+	}
+
+	if !isTagMatch(tagFilter, templateTags) {
+		return false, nil
+	}
+
+	if !isAuthorMatch(tagFilter, templateAuthors) {
+		return false, nil
+	}
+
+	if !isSeverityMatch(tagFilter, templateSeverity) {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func isSeverityMatch(tagFilter *TagFilter, templateSeverity severity.Severity) bool {
+	if len(tagFilter.severities) == 0 || templateSeverity == severity.Undefined {
+		return true
+	}
+
+	if _, ok := tagFilter.severities[templateSeverity]; ok {
+		return true
+	}
+
+	return false
+}
+
+func isAuthorMatch(tagFilter *TagFilter, templateAuthors []string) bool {
+	if len(tagFilter.authors) == 0 {
+		return true
+	}
+
+	templateAuthorMap := toMap(templateAuthors)
+	for requiredAuthor := range tagFilter.authors {
+		if _, ok := templateAuthorMap[requiredAuthor]; ok {
+			return true
+		}
+	}
+
+	return false
+}
+
+func isExtraTagMatch(extraTags []string, templateTags []string) bool {
+	if len(extraTags) == 0 {
+		return true
+	}
+
+	templatesTagMap := toMap(templateTags)
+	for _, extraTag := range extraTags {
+		if _, ok := templatesTagMap[extraTag]; ok {
+			return true
+		}
+	}
+
+	return false
+}
+
+func isTagMatch(tagFilter *TagFilter, templateTags []string) bool {
+	if len(tagFilter.allowedTags) == 0 {
+		return true
+	}
+
+	for _, templateTag := range templateTags {
+		if _, ok := tagFilter.allowedTags[templateTag]; ok {
+			return true
+		}
+	}
+
+	return false
+}
+
+type Config struct {
+	Tags        []string
+	ExcludeTags []string
+	Authors     []string
+	Severities  severity.Severities
+	IncludeTags []string
+}
+
+// New returns a tag filter for nuclei tag based execution
+//
+// It takes into account Tags, Severities, Authors, IncludeTags, ExcludeTags.
+func New(config *Config) *TagFilter {
+	filter := &TagFilter{
+		allowedTags: make(map[string]struct{}),
+		authors:     make(map[string]struct{}),
+		severities:  make(map[severity.Severity]struct{}),
+		block:       make(map[string]struct{}),
+		matchAllows: make(map[string]struct{}),
+	}
+	for _, tag := range config.ExcludeTags {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.block[val]; !ok {
+				filter.block[val] = struct{}{}
+			}
+		}
+	}
+	for _, tag := range config.Severities {
+		if _, ok := filter.severities[tag]; !ok {
+			filter.severities[tag] = struct{}{}
+		}
+	}
+	for _, tag := range config.Authors {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.authors[val]; !ok {
+				filter.authors[val] = struct{}{}
+			}
+		}
+	}
+	for _, tag := range config.Tags {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.allowedTags[val]; !ok {
+				filter.allowedTags[val] = struct{}{}
+			}
+			delete(filter.block, val)
+		}
+	}
+	for _, tag := range config.IncludeTags {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.matchAllows[val]; !ok {
+				filter.matchAllows[val] = struct{}{}
+			}
+			delete(filter.block, val)
+		}
+	}
+	return filter
+}
+
+/*
+TODO similar logic is used over and over again. It should be extracted and reused
+Changing []string and string data types that hold string slices to StringSlice would be the preferred solution,
+which implicitly does the normalization before any other calls starting to use it.
+*/
+func splitCommaTrim(value string) []string {
+	if !strings.Contains(value, ",") {
+		return []string{strings.ToLower(value)}
+	}
+	splitted := strings.Split(value, ",")
+	final := make([]string, len(splitted))
+	for i, value := range splitted {
+		final[i] = strings.ToLower(strings.TrimSpace(value))
+	}
+	return final
+}
+
+func toMap(slice []string) map[string]struct{} {
+	result := make(map[string]struct{}, len(slice))
+	for _, value := range slice {
+		if _, ok := result[value]; !ok {
+			result[value] = struct{}{}
+		}
+	}
+	return result
+}

v2/pkg/catalog/loader/filter/tag_filter_test.go

@@ -0,0 +1,102 @@
+package filter
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+)
+
+func TestTagBasedFilter(t *testing.T) {
+	{
+		filter := New(&Config{
+			Tags: []string{"cves", "2021", "jira"},
+		})
+
+		t.Run("true", func(t *testing.T) {
+			matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil)
+			require.True(t, matched, "could not get correct match")
+		})
+		t.Run("false", func(t *testing.T) {
+			matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil)
+			require.False(t, matched, "could not get correct match")
+		})
+		t.Run("match-extra-tags-positive", func(t *testing.T) {
+			matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"})
+			require.True(t, matched, "could not get correct match")
+		})
+		t.Run("match-extra-tags-negative", func(t *testing.T) {
+			matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"})
+			require.False(t, matched, "could not get correct match")
+		})
+	}
+
+	t.Run("not-match-excludes", func(t *testing.T) {
+		filter := New(&Config{
+			ExcludeTags: []string{"dos"},
+		})
+		matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil)
+		require.False(t, matched, "could not get correct match")
+		require.Equal(t, ErrExcluded, err, "could not get correct error")
+	})
+	t.Run("match-includes", func(t *testing.T) {
+		filter := New(&Config{
+			Tags:        []string{"cves", "fuzz"},
+			ExcludeTags: []string{"dos", "fuzz"},
+			IncludeTags: []string{"fuzz"},
+		})
+		matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
+		require.Nil(t, err, "could not get match")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("match-includes", func(t *testing.T) {
+		filter := New(&Config{
+			Tags:        []string{"fuzz"},
+			ExcludeTags: []string{"fuzz"},
+		})
+		matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
+		require.Nil(t, err, "could not get match")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("match-author", func(t *testing.T) {
+		filter := New(&Config{
+			Authors: []string{"pdteam"},
+		})
+		matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("match-severity", func(t *testing.T) {
+		filter := New(&Config{
+			Severities: severity.Severities{severity.High},
+		})
+		matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil)
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("match-exclude-with-tags", func(t *testing.T) {
+		filter := New(&Config{
+			Tags:        []string{"tag"},
+			ExcludeTags: []string{"another"},
+		})
+		matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil)
+		require.False(t, matched, "could not get correct match")
+	})
+	t.Run("match-conditions", func(t *testing.T) {
+		filter := New(&Config{
+			Authors:    []string{"pdteam"},
+			Tags:       []string{"jira"},
+			Severities: severity.Severities{severity.High},
+		})
+		matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil)
+		require.True(t, matched, "could not get correct match")
+
+		matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil)
+		require.False(t, matched, "could not get correct match")
+
+		matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil)
+		require.False(t, matched, "could not get correct match")
+
+		matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil)
+		require.False(t, matched, "could not get correct match")
+	})
+}

v2/pkg/catalog/loader/loader.go

@@ -0,0 +1,200 @@
+package loader
+
+import (
+	"errors"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+)
+
+// Config contains the configuration options for the loader
+type Config struct {
+	Templates        []string
+	Workflows        []string
+	ExcludeTemplates []string
+	IncludeTemplates []string
+
+	Tags        []string
+	ExcludeTags []string
+	Authors     []string
+	Severities  severity.Severities
+	IncludeTags []string
+
+	Catalog            *catalog.Catalog
+	ExecutorOptions    protocols.ExecuterOptions
+	TemplatesDirectory string
+}
+
+// Store is a storage for loaded nuclei templates
+type Store struct {
+	tagFilter      *filter.TagFilter
+	pathFilter     *filter.PathFilter
+	config         *Config
+	finalTemplates []string
+
+	templates []*templates.Template
+	workflows []*templates.Template
+
+	preprocessor templates.Preprocessor
+}
+
+// New creates a new template store based on provided configuration
+func New(config *Config) (*Store, error) {
+	// Create a tag filter based on provided configuration
+	store := &Store{
+		config: config,
+		tagFilter: filter.New(&filter.Config{
+			Tags:        config.Tags,
+			ExcludeTags: config.ExcludeTags,
+			Authors:     config.Authors,
+			Severities:  config.Severities,
+			IncludeTags: config.IncludeTags,
+		}),
+		pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{
+			IncludedTemplates: config.IncludeTemplates,
+			ExcludedTemplates: config.ExcludeTemplates,
+		}, config.Catalog),
+	}
+
+	// Handle a case with no templates or workflows, where we use base directory
+	if len(config.Templates) == 0 && len(config.Workflows) == 0 {
+		config.Templates = append(config.Templates, config.TemplatesDirectory)
+	}
+	store.finalTemplates = append(store.finalTemplates, config.Templates...)
+	return store, nil
+}
+
+// Templates returns all the templates in the store
+func (store *Store) Templates() []*templates.Template {
+	return store.templates
+}
+
+// Workflows returns all the workflows in the store
+func (store *Store) Workflows() []*templates.Template {
+	return store.workflows
+}
+
+// RegisterPreprocessor allows a custom preprocessor to be passed to the store to run against templates
+func (store *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
+	store.preprocessor = preprocessor
+}
+
+// Load loads all the templates from a store, performs filtering and returns
+// the complete compiled templates for a nuclei execution configuration.
+func (store *Store) Load() {
+	store.templates = store.LoadTemplates(store.finalTemplates)
+	store.workflows = store.LoadWorkflows(store.config.Workflows)
+}
+
+// ValidateTemplates takes a list of templates and validates them
+// erroring out on discovering any faulty templates.
+func (store *Store) ValidateTemplates(templatesList, workflowsList []string) error {
+	templatePaths := store.config.Catalog.GetTemplatesPath(templatesList)
+	workflowPaths := store.config.Catalog.GetTemplatesPath(workflowsList)
+
+	filteredTemplatePaths := store.pathFilter.Match(templatePaths)
+	filteredWorkflowPaths := store.pathFilter.Match(workflowPaths)
+
+	if areTemplatesValid(store, filteredTemplatePaths) && areWorkflowsValid(store, filteredWorkflowPaths) {
+		return nil
+	}
+	return errors.New("an error occurred during templates validation")
+}
+
+func areWorkflowsValid(store *Store, filteredWorkflowPaths map[string]struct{}) bool {
+	return areWorkflowOrTemplatesValid(store, filteredWorkflowPaths, true, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) {
+		return parsers.LoadWorkflow(templatePath)
+	})
+}
+
+func areTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}) bool {
+	return areWorkflowOrTemplatesValid(store, filteredTemplatePaths, false, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) {
+		return parsers.LoadTemplate(templatePath, store.tagFilter, nil)
+	})
+}
+
+func areWorkflowOrTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}, isWorkflow bool, load func(templatePath string, tagFilter *filter.TagFilter) (bool, error)) bool {
+	areTemplatesValid := true
+	for templatePath := range filteredTemplatePaths {
+		if _, err := load(templatePath, store.tagFilter); err != nil {
+			if isParsingError("Error occurred loading template %s: %s\n", templatePath, err) {
+				areTemplatesValid = false
+				continue
+			}
+		}
+
+		template, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
+		if err != nil {
+			if isParsingError("Error occurred parsing template %s: %s\n", templatePath, err) {
+				areTemplatesValid = false
+			}
+		} else {
+			if !isWorkflow && len(template.Workflows) > 0 {
+				return true
+			}
+		}
+	}
+	return areTemplatesValid
+}
+
+func isParsingError(message string, template string, err error) bool {
+	if err == templates.ErrCreateTemplateExecutor {
+		return false
+	}
+	if err == filter.ErrExcluded {
+		return false
+	}
+	gologger.Error().Msgf(message, template, err)
+	return true
+}
+
+// LoadTemplates takes a list of templates and returns paths for them
+func (store *Store) LoadTemplates(templatesList []string) []*templates.Template {
+	includedTemplates := store.config.Catalog.GetTemplatesPath(templatesList)
+	templatePathMap := store.pathFilter.Match(includedTemplates)
+
+	loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
+	for templatePath := range templatePathMap {
+		loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil)
+		if err != nil {
+			gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
+		}
+		if loaded {
+			parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
+			if err != nil {
+				gologger.Warning().Msgf("Could not parse template %s: %s\n", templatePath, err)
+			} else if parsed != nil {
+				loadedTemplates = append(loadedTemplates, parsed)
+			}
+		}
+	}
+	return loadedTemplates
+}
+
+// LoadWorkflows takes a list of workflows and returns paths for them
+func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
+	includedWorkflows := store.config.Catalog.GetTemplatesPath(workflowsList)
+	workflowPathMap := store.pathFilter.Match(includedWorkflows)
+
+	loadedWorkflows := make([]*templates.Template, 0, len(workflowPathMap))
+	for workflowPath := range workflowPathMap {
+		loaded, err := parsers.LoadWorkflow(workflowPath)
+		if err != nil {
+			gologger.Warning().Msgf("Could not load workflow %s: %s\n", workflowPath, err)
+		}
+		if loaded {
+			parsed, err := templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions)
+			if err != nil {
+				gologger.Warning().Msgf("Could not parse workflow %s: %s\n", workflowPath, err)
+			} else if parsed != nil {
+				loadedWorkflows = append(loadedWorkflows, parsed)
+			}
+		}
+	}
+	return loadedWorkflows
+}

v2/pkg/catalog/loader/loader_test.go

@@ -0,0 +1,40 @@
+package loader
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadTemplates(t *testing.T) {
+	store, err := New(&Config{
+		Templates: []string{"cves/CVE-2021-21315.yaml"},
+	})
+	require.Nil(t, err, "could not load templates")
+	require.Equal(t, []string{"cves/CVE-2021-21315.yaml"}, store.finalTemplates, "could not get correct templates")
+
+	templatesDirectory := "/test"
+	t.Run("blank", func(t *testing.T) {
+		store, err := New(&Config{
+			TemplatesDirectory: templatesDirectory,
+		})
+		require.Nil(t, err, "could not load templates")
+		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
+	})
+	t.Run("only-tags", func(t *testing.T) {
+		store, err := New(&Config{
+			Tags:               []string{"cves"},
+			TemplatesDirectory: templatesDirectory,
+		})
+		require.Nil(t, err, "could not load templates")
+		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
+	})
+	t.Run("tags-with-path", func(t *testing.T) {
+		store, err := New(&Config{
+			Tags:               []string{"cves"},
+			TemplatesDirectory: templatesDirectory,
+		})
+		require.Nil(t, err, "could not load templates")
+		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
+	})
+}

v2/pkg/catalog/path.go

@@ -3,9 +3,7 @@ package catalog
 import (
 	"fmt"
 	"os"
-	"path"
 	"path/filepath"
-	"strings"
 )
 
 // ResolvePath resolves the path to an absolute one in various ways.
@@ -14,12 +12,12 @@ import (
 // or checking the nuclei templates directory. If a second path is given,
 // it also tries to find paths relative to that second path.
 func (c *Catalog) ResolvePath(templateName, second string) (string, error) {
-	if strings.HasPrefix(templateName, "/") || strings.Contains(templateName, ":\\") {
+	if filepath.IsAbs(templateName) {
 		return templateName, nil
 	}
 
 	if second != "" {
-		secondBasePath := path.Join(filepath.Dir(second), templateName)
+		secondBasePath := filepath.Join(filepath.Dir(second), templateName)
 		if _, err := os.Stat(secondBasePath); !os.IsNotExist(err) {
 			return secondBasePath, nil
 		}
@@ -30,13 +28,13 @@ func (c *Catalog) ResolvePath(templateName, second string) (string, error) {
 		return "", err
 	}
 
-	templatePath := path.Join(curDirectory, templateName)
+	templatePath := filepath.Join(curDirectory, templateName)
 	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
 		return templatePath, nil
 	}
 
 	if c.templatesDirectory != "" {
-		templatePath := path.Join(c.templatesDirectory, templateName)
+		templatePath := filepath.Join(c.templatesDirectory, templateName)
 		if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
 			return templatePath, nil
 		}

v2/pkg/model/model.go

@@ -0,0 +1,105 @@
+package model
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice"
+)
+
+// Info contains metadata information about a template
+type Info struct {
+	// description: |
+	//   Name should be good short summary that identifies what the template does.
+	//
+	// examples:
+	//   - value: "\"bower.json file disclosure\""
+	//   - value: "\"Nagios Default Credentials Check\""
+	Name string `json:"name,omitempty" yaml:"name,omitempty" jsonschema:"title=name of the template,description=Name is a short summary of what the template does,example=Nagios Default Credentials Check"`
+	// description: |
+	//   Author of the template.
+	//
+	//   Multiple values can also be specified separated by commas.
+	// examples:
+	//   - value: "\"<username>\""
+	Authors stringslice.StringSlice `json:"author,omitempty" yaml:"author,omitempty" jsonschema:"title=author of the template,description=Author is the author of the template,example=username"`
+	// description: |
+	//   Any tags for the template.
+	//
+	//   Multiple values can also be specified separated by commas.
+	//
+	// examples:
+	//   - name: Example tags
+	//     value: "\"cve,cve2019,grafana,auth-bypass,dos\""
+	Tags stringslice.StringSlice `json:"tags,omitempty" yaml:"tags,omitempty" jsonschema:"title=tags of the template,description=Any tags for the template"`
+	// description: |
+	//   Description of the template.
+	//
+	//   You can go in-depth here on what the template actually does.
+	//
+	// examples:
+	//   - value: "\"Bower is a package manager which stores package information in the bower.json file\""
+	//   - value: "\"Subversion ALM for the enterprise before 8.8.2 allows reflected XSS at multiple locations\""
+	Description string `json:"description,omitempty" yaml:"description,omitempty" jsonschema:"title=description of the template,description=In-depth explanation on what the template does,example=Bower is a package manager which stores package information in the bower.json file"`
+	// description: |
+	//   References for the template.
+	//
+	//   This should contain links relevant to the template.
+	//
+	// examples:
+	//   - value: >
+	//       []string{"https://github.com/strapi/strapi", "https://github.com/getgrav/grav"}
+	Reference stringslice.StringSlice `json:"reference,omitempty" yaml:"reference,omitempty" jsonschema:"title=references for the template,description=Links relevant to the template"`
+	// description: |
+	//   Severity of the template.
+	//
+	// values:
+	//   - info
+	//   - low
+	//   - medium
+	//   - high
+	//   - critical
+	SeverityHolder severity.Holder `json:"severity,omitempty" yaml:"severity,omitempty"`
+	// description: |
+	//   Metadata of the template.
+	//
+	// examples:
+	//   - value: >
+	//       map[string]string{"customField1":"customValue1"}
+	Metadata map[string]string `json:"metadata,omitempty" yaml:"metadata,omitempty" jsonschema:"title=additional metadata for the template,description=Additional metadata fields for the template"`
+
+	// description: |
+	//   Classification contains classification information about the template.
+	Classification *Classification `json:"classification,omitempty" yaml:"classification,omitempty" jsonschema:"title=classification info for the template,description=Classification information for the template"`
+
+	// description: |
+	//   Remediation steps for the template.
+	//
+	//   You can go in-depth here on how to mitigate the problem found by this template.
+	//
+	// examples:
+	//   - value: "\"Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties\""
+	Remediation string `json:"remediation,omitempty" yaml:"remediation,omitempty" jsonschema:"title=remediation steps for the template,description=In-depth explanation on how to fix the issues found by the template,example=Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties"`
+}
+
+// Classification contains the vulnerability classification data for a template.
+type Classification struct {
+	// description: |
+	//   CVE ID for the template
+	// examples:
+	//   - value: "\"CVE-2020-14420\""
+	CVEID stringslice.StringSlice `json:"cve-id,omitempty" yaml:"cve-id,omitempty" jsonschema:"title=cve ids for the template,description=CVE IDs for the template,example=CVE-2020-14420"`
+	// description: |
+	//   CWE ID for the template.
+	// examples:
+	//   - value: "\"CWE-22\""
+	CWEID stringslice.StringSlice `json:"cwe-id,omitempty" yaml:"cwe-id,omitempty" jsonschema:"title=cwe ids for the template,description=CWE IDs for the template,example=CWE-22"`
+	// description: |
+	//   CVSS Metrics for the template.
+	// examples:
+	//   - value: "\"3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\""
+	CVSSMetrics string `json:"cvss-metrics,omitempty" yaml:"cvss-metrics,omitempty" jsonschema:"title=cvss metrics for the template,description=CVSS Metrics for the template,example=3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"`
+	// description: |
+	//   CVSS Score for the template.
+	// examples:
+	//   - value: "\"9.8\""
+	CVSSScore float64 `json:"cvss-score,omitempty" yaml:"cvss-score,omitempty" jsonschema:"title=cvss score for the template,description=CVSS Score for the template,example=9.8"`
+}

v2/pkg/model/model_test.go

@@ -0,0 +1,117 @@
+package model
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice"
+
+	"gopkg.in/yaml.v2"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestInfoJsonMarshal(t *testing.T) {
+	info := Info{
+		Name:           "Test Template Name",
+		Authors:        stringslice.StringSlice{Value: []string{"forgedhallpass", "ice3man"}},
+		Description:    "Test description",
+		SeverityHolder: severity.Holder{Severity: severity.High},
+		Tags:           stringslice.StringSlice{Value: []string{"cve", "misc"}},
+		Reference:      stringslice.StringSlice{Value: "reference1"},
+	}
+
+	result, err := json.Marshal(&info)
+	assert.Nil(t, err)
+
+	expected := `{"name":"Test Template Name","author":["forgedhallpass","ice3man"],"tags":["cve","misc"],"description":"Test description","reference":"reference1","severity":"high"}`
+	assert.Equal(t, expected, string(result))
+}
+
+func TestInfoYamlMarshal(t *testing.T) {
+	info := Info{
+		Name:           "Test Template Name",
+		Authors:        stringslice.StringSlice{Value: []string{"forgedhallpass", "ice3man"}},
+		Description:    "Test description",
+		SeverityHolder: severity.Holder{Severity: severity.High},
+		Tags:           stringslice.StringSlice{Value: []string{"cve", "misc"}},
+		Reference:      stringslice.StringSlice{Value: "reference1"},
+	}
+
+	result, err := yaml.Marshal(&info)
+	assert.Nil(t, err)
+
+	expected := `name: Test Template Name
+author:
+- forgedhallpass
+- ice3man
+tags:
+- cve
+- misc
+description: Test description
+reference: reference1
+severity: high
+`
+	assert.Equal(t, expected, string(result))
+}
+
+func TestUnmarshal(t *testing.T) {
+	templateName := "Test Template"
+	authors := []string{"forgedhallpass", "ice3man"}
+	tags := []string{"cve", "misc"}
+	references := []string{"http://test.com", "http://domain.com"}
+
+	dynamicKey1 := "customDynamicKey1"
+	dynamicKey2 := "customDynamicKey2"
+
+	dynamicKeysMap := map[string]string{
+		dynamicKey1: "customDynamicValue1",
+		dynamicKey2: "customDynamicValue2",
+	}
+
+	assertUnmarshalledTemplateInfo := func(t *testing.T, yamlPayload string) Info {
+		info := Info{}
+		err := yaml.Unmarshal([]byte(yamlPayload), &info)
+		assert.Nil(t, err)
+		assert.Equal(t, info.Name, templateName)
+		assert.Equal(t, info.Authors.ToSlice(), authors)
+		assert.Equal(t, info.Tags.ToSlice(), tags)
+		assert.Equal(t, info.SeverityHolder.Severity, severity.Critical)
+		assert.Equal(t, info.Reference.ToSlice(), references)
+		assert.Equal(t, info.Metadata, dynamicKeysMap)
+		return info
+	}
+
+	yamlPayload1 := `
+  name: ` + templateName + `
+  author: ` + strings.Join(authors, ", ") + `
+  tags: ` + strings.Join(tags, ", ") + `
+  severity: critical
+  reference: ` + strings.Join(references, ", ") + `
+  metadata:
+     ` + dynamicKey1 + `: ` + dynamicKeysMap[dynamicKey1] + `
+     ` + dynamicKey2 + `: ` + dynamicKeysMap[dynamicKey2] + `
+`
+	yamlPayload2 := `
+  name: ` + templateName + `
+  author:
+    - ` + authors[0] + `
+    - ` + authors[1] + `
+  tags:
+    - ` + tags[0] + `
+    - ` + tags[1] + `
+  severity: critical
+  reference:
+    - ` + references[0] + ` # comments are not unmarshalled
+    - ` + references[1] + `
+  metadata:
+     ` + dynamicKey1 + `: ` + dynamicKeysMap[dynamicKey1] + `
+     ` + dynamicKey2 + `: ` + dynamicKeysMap[dynamicKey2] + `
+`
+
+	info1 := assertUnmarshalledTemplateInfo(t, yamlPayload1)
+	info2 := assertUnmarshalledTemplateInfo(t, yamlPayload2)
+	assert.Equal(t, info1, info2)
+}

v2/pkg/model/types/severity/severities.go

@@ -0,0 +1,62 @@
+package severity
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/projectdiscovery/goflags"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice"
+)
+
+// Severities used by the goflags library for parsing an array of Severity types, passed as CLI arguments from the user
+type Severities []Severity
+
+func (severities *Severities) Set(values string) error {
+	inputSeverities, err := goflags.ToNormalizedStringSlice(values)
+	if err != nil {
+		return err
+	}
+
+	for _, inputSeverity := range inputSeverities {
+		if err := setSeverity(severities, inputSeverity); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (severities *Severities) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var stringSliceValue stringslice.StringSlice
+	if err := unmarshal(&stringSliceValue); err != nil {
+		return err
+	}
+
+	stringSLice := stringSliceValue.ToSlice()
+	var result = make(Severities, 0, len(stringSLice))
+	for _, severityString := range stringSLice {
+		if err := setSeverity(&result, severityString); err != nil {
+			return err
+		}
+	}
+	*severities = result
+	return nil
+}
+
+func (severities Severities) String() string {
+	var stringSeverities []string
+	for _, severity := range severities {
+		stringSeverities = append(stringSeverities, severity.String())
+	}
+	return strings.Join(stringSeverities, ", ")
+}
+
+func setSeverity(severities *Severities, value string) error {
+	computedSeverity, err := toSeverity(value)
+	if err != nil {
+		return fmt.Errorf("'%s' is not a valid severity", value)
+	}
+
+	// TODO change the Severities type to map[Severity]interface{}, where the values are struct{}{}, to "simulates" a "set" data structure
+	*severities = append(*severities, computedSeverity)
+	return nil
+}

v2/pkg/model/types/severity/severity.go

@@ -0,0 +1,53 @@
+package severity
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+type Severity int
+
+const (
+	Undefined Severity = iota
+	Info
+	Low
+	Medium
+	High
+	Critical
+	limit
+)
+
+var severityMappings = map[Severity]string{
+	Info:     "info",
+	Low:      "low",
+	Medium:   "medium",
+	High:     "high",
+	Critical: "critical",
+}
+
+func GetSupportedSeverities() Severities {
+	var result []Severity
+	for index := Severity(1); index < limit; index++ {
+		result = append(result, index)
+	}
+	return result
+}
+
+func toSeverity(valueToMap string) (Severity, error) {
+	normalizedValue := normalizeValue(valueToMap)
+	for key, currentValue := range severityMappings {
+		if normalizedValue == currentValue {
+			return key, nil
+		}
+	}
+	return -1, errors.New("Invalid severity: " + valueToMap)
+}
+
+func normalizeValue(value string) string {
+	return strings.TrimSpace(strings.ToLower(value))
+}
+
+func (severity Severity) String() string {
+	return severityMappings[severity]
+}

v2/pkg/model/types/severity/severity_holder.go

@@ -0,0 +1,48 @@
+package severity
+
+import (
+	"encoding/json"
+
+	"github.com/alecthomas/jsonschema"
+)
+
+//nolint:exported,revive //prefer to be explicit about the name, and make it refactor-safe
+// Holder holds a Severity type. Required for un/marshalling purposes
+type Holder struct {
+	Severity Severity
+}
+
+func (severityHolder Holder) JSONSchemaType() *jsonschema.Type {
+	gotType := &jsonschema.Type{
+		Type:        "string",
+		Title:       "severity of the template",
+		Description: "Seriousness of the implications of the template",
+	}
+	for _, severity := range GetSupportedSeverities() {
+		gotType.Enum = append(gotType.Enum, severity.String())
+	}
+	return gotType
+}
+
+func (severityHolder *Holder) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var marshalledSeverity string
+	if err := unmarshal(&marshalledSeverity); err != nil {
+		return err
+	}
+
+	computedSeverity, err := toSeverity(marshalledSeverity)
+	if err != nil {
+		return err
+	}
+
+	severityHolder.Severity = computedSeverity
+	return nil
+}
+
+func (severityHolder *Holder) MarshalJSON() ([]byte, error) {
+	return json.Marshal(severityHolder.Severity.String())
+}
+
+func (severityHolder Holder) MarshalYAML() (interface{}, error) {
+	return severityHolder.Severity.String(), nil
+}

v2/pkg/model/types/severity/severity_test.go

@@ -0,0 +1,65 @@
+package severity
+
+import (
+	"testing"
+
+	"gopkg.in/yaml.v2"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestYamlUnmarshal(t *testing.T) {
+	testUnmarshal(t, yaml.Unmarshal, func(value string) string { return value })
+}
+
+func TestYamlMarshal(t *testing.T) {
+	severity := Holder{Severity: High}
+
+	marshalled, err := severity.MarshalYAML()
+	assert.Nil(t, err, "could not marshal yaml")
+	assert.Equal(t, "high", marshalled, "could not marshal severity correctly")
+}
+
+func TestYamlUnmarshalFail(t *testing.T) {
+	testUnmarshalFail(t, yaml.Unmarshal, createYAML)
+}
+
+func TestGetSupportedSeverities(t *testing.T) {
+	severities := GetSupportedSeverities()
+	assert.Equal(t, severities, Severities{Info, Low, Medium, High, Critical})
+}
+
+func testUnmarshal(t *testing.T, unmarshaller func(data []byte, v interface{}) error, payloadCreator func(value string) string) {
+	payloads := [...]string{
+		payloadCreator("Info"),
+		payloadCreator("info"),
+		payloadCreator("inFo "),
+		payloadCreator("infO "),
+		payloadCreator(" INFO "),
+	}
+
+	for _, payload := range payloads { // nolint:scopelint // false-positive
+		t.Run(payload, func(t *testing.T) {
+			result := unmarshal(payload, unmarshaller)
+			assert.Equal(t, result.Severity, Info)
+			assert.Equal(t, result.Severity.String(), "info")
+		})
+	}
+}
+
+func testUnmarshalFail(t *testing.T, unmarshaller func(data []byte, v interface{}) error, payloadCreator func(value string) string) {
+	assert.Panics(t, func() { unmarshal(payloadCreator("invalid"), unmarshaller) })
+}
+
+func unmarshal(value string, unmarshaller func(data []byte, v interface{}) error) Holder {
+	severityStruct := Holder{}
+	var err = unmarshaller([]byte(value), &severityStruct)
+	if err != nil {
+		panic(err)
+	}
+	return severityStruct
+}
+
+func createYAML(value string) string {
+	return "severity: " + value + "\n"
+}

v2/pkg/model/types/stringslice/stringslice.go

@@ -0,0 +1,92 @@
+package stringslice
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/alecthomas/jsonschema"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
+)
+
+// StringSlice represents a single (in-lined) or multiple string value(s).
+// The unmarshaller does not automatically convert in-lined strings to []string, hence the interface{} type is required.
+type StringSlice struct {
+	Value interface{}
+}
+
+func (stringSlice StringSlice) JSONSchemaType() *jsonschema.Type {
+	gotType := &jsonschema.Type{
+		OneOf: []*jsonschema.Type{{Type: "string"}, {Type: "array"}},
+	}
+	return gotType
+}
+
+func (stringSlice *StringSlice) IsEmpty() bool {
+	return len(stringSlice.ToSlice()) == 0
+}
+
+func (stringSlice StringSlice) ToSlice() []string {
+	switch value := stringSlice.Value.(type) {
+	case string:
+		return []string{value}
+	case []string:
+		return value
+	case nil:
+		return []string{}
+	default:
+		panic(fmt.Sprintf("Unexpected StringSlice type: '%T'", value))
+	}
+}
+
+func (stringSlice StringSlice) String() string {
+	return strings.Join(stringSlice.ToSlice(), ", ")
+}
+
+func (stringSlice *StringSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	marshalledSlice, err := marshalStringToSlice(unmarshal)
+	if err != nil {
+		return err
+	}
+
+	result := make([]string, 0, len(marshalledSlice))
+	//nolint:gosimple,nolintlint //cannot be replaced with result = append(result, slices...) because the values are being normalized
+	for _, value := range marshalledSlice {
+		result = append(result, strings.ToLower(strings.TrimSpace(value))) // TODO do we need to introduce RawStringSlice and/or NormalizedStringSlices?
+	}
+	stringSlice.Value = result
+	return nil
+}
+
+func (stringSlice StringSlice) MarshalYAML() (interface{}, error) {
+	return stringSlice.Value, nil
+}
+
+func (stringSlice StringSlice) MarshalJSON() ([]byte, error) {
+	return json.Marshal(stringSlice.Value)
+}
+
+func marshalStringToSlice(unmarshal func(interface{}) error) ([]string, error) {
+	var marshalledValueAsString string
+	var marshalledValuesAsSlice []string
+
+	sliceMarshalError := unmarshal(&marshalledValuesAsSlice)
+	if sliceMarshalError != nil {
+		stringMarshalError := unmarshal(&marshalledValueAsString)
+		if stringMarshalError != nil {
+			return nil, stringMarshalError
+		}
+	}
+
+	var result []string
+	if len(marshalledValuesAsSlice) > 0 {
+		result = marshalledValuesAsSlice
+	} else if utils.IsNotBlank(marshalledValueAsString) {
+		result = strings.Split(marshalledValueAsString, ",")
+	} else {
+		result = []string{}
+	}
+
+	return result, nil
+}

v2/pkg/model/worflow_loader.go

@@ -0,0 +1,12 @@
+package model
+
+// TODO shouldn't this rather be TemplateLoader?
+
+// WorkflowLoader is a loader interface required for workflow initialization.
+type WorkflowLoader interface {
+	// GetTemplatePathsByTags returns a list of template paths based on the provided tags from the templates directory
+	GetTemplatePathsByTags(tags []string) []string
+
+	// GetTemplatePaths takes a list of templates and returns paths for them
+	GetTemplatePaths(templatesList []string, noValidate bool) []string
+}

v2/pkg/operators/common/dsl/dsl.go

@@ -1,12 +1,12 @@
 package dsl
 
 import (
-	"bytes"
 	"crypto/md5"
 	"crypto/sha1"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"html"
 	"math"
@@ -17,7 +17,8 @@ import (
 	"time"
 
 	"github.com/Knetic/govaluate"
-	"github.com/projectdiscovery/nuclei/v2/internal/collaborator"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/deserialization"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/spaolacci/murmur3"
 )
@@ -30,151 +31,117 @@ const (
 	withMaxRandArgsSize  = withCutSetArgsSize
 )
 
-// HelperFunctions contains the dsl helper functions
-func HelperFunctions() map[string]govaluate.ExpressionFunction {
-	functions := make(map[string]govaluate.ExpressionFunction)
-
-	functions["len"] = func(args ...interface{}) (interface{}, error) {
+var functions = map[string]govaluate.ExpressionFunction{
+	"len": func(args ...interface{}) (interface{}, error) {
 		length := len(types.ToString(args[0]))
 		return float64(length), nil
-	}
-
-	functions["toupper"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"toupper": func(args ...interface{}) (interface{}, error) {
 		return strings.ToUpper(types.ToString(args[0])), nil
-	}
-
-	functions["tolower"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"tolower": func(args ...interface{}) (interface{}, error) {
 		return strings.ToLower(types.ToString(args[0])), nil
-	}
-
-	functions["replace"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"replace": func(args ...interface{}) (interface{}, error) {
 		return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil
-	}
-
-	functions["replace_regex"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"replace_regex": func(args ...interface{}) (interface{}, error) {
 		compiled, err := regexp.Compile(types.ToString(args[1]))
 		if err != nil {
 			return nil, err
 		}
 		return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil
-	}
-
-	functions["trim"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Trim(types.ToString(args[0]), types.ToString(args[2])), nil
-	}
-
-	functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"trim": func(args ...interface{}) (interface{}, error) {
+		return strings.Trim(types.ToString(args[0]), types.ToString(args[1])), nil
+	},
+	"trimleft": func(args ...interface{}) (interface{}, error) {
 		return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil
-	}
-
-	functions["trimright"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"trimright": func(args ...interface{}) (interface{}, error) {
 		return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil
-	}
-
-	functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"trimspace": func(args ...interface{}) (interface{}, error) {
 		return strings.TrimSpace(types.ToString(args[0])), nil
-	}
-
-	functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"trimprefix": func(args ...interface{}) (interface{}, error) {
 		return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil
-	}
-
-	functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"trimsuffix": func(args ...interface{}) (interface{}, error) {
 		return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil
-	}
-
-	functions["reverse"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"reverse": func(args ...interface{}) (interface{}, error) {
 		return reverseString(types.ToString(args[0])), nil
-	}
-
+	},
 	// encoding
-	functions["base64"] = func(args ...interface{}) (interface{}, error) {
+	"base64": func(args ...interface{}) (interface{}, error) {
 		sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
 
 		return sEnc, nil
-	}
-
+	},
 	// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
-	functions["base64_py"] = func(args ...interface{}) (interface{}, error) {
+	"base64_py": func(args ...interface{}) (interface{}, error) {
 		sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
-		return insertInto(sEnc, 76, '\n'), nil
-	}
-
-	functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
+		return deserialization.InsertInto(sEnc, 76, '\n'), nil
+	},
+	"base64_decode": func(args ...interface{}) (interface{}, error) {
 		return base64.StdEncoding.DecodeString(types.ToString(args[0]))
-	}
-
-	functions["url_encode"] = func(args ...interface{}) (interface{}, error) {
-		return url.PathEscape(types.ToString(args[0])), nil
-	}
-
-	functions["url_decode"] = func(args ...interface{}) (interface{}, error) {
-		return url.PathUnescape(types.ToString(args[0]))
-	}
-
-	functions["hex_encode"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"url_encode": func(args ...interface{}) (interface{}, error) {
+		return url.QueryEscape(types.ToString(args[0])), nil
+	},
+	"url_decode": func(args ...interface{}) (interface{}, error) {
+		return url.QueryUnescape(types.ToString(args[0]))
+	},
+	"hex_encode": func(args ...interface{}) (interface{}, error) {
 		return hex.EncodeToString([]byte(types.ToString(args[0]))), nil
-	}
-
-	functions["hex_decode"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"hex_decode": func(args ...interface{}) (interface{}, error) {
 		hx, _ := hex.DecodeString(types.ToString(args[0]))
 		return string(hx), nil
-	}
-
-	functions["html_escape"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"html_escape": func(args ...interface{}) (interface{}, error) {
 		return html.EscapeString(types.ToString(args[0])), nil
-	}
-
-	functions["html_unescape"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"html_unescape": func(args ...interface{}) (interface{}, error) {
 		return html.UnescapeString(types.ToString(args[0])), nil
-	}
-
+	},
 	// hashing
-	functions["md5"] = func(args ...interface{}) (interface{}, error) {
+	"md5": func(args ...interface{}) (interface{}, error) {
 		hash := md5.Sum([]byte(types.ToString(args[0])))
 
 		return hex.EncodeToString(hash[:]), nil
-	}
-
-	functions["sha256"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"sha256": func(args ...interface{}) (interface{}, error) {
 		h := sha256.New()
-		_, err := h.Write([]byte(types.ToString(args[0])))
-
-		if err != nil {
+		if _, err := h.Write([]byte(types.ToString(args[0]))); err != nil {
 			return nil, err
 		}
 		return hex.EncodeToString(h.Sum(nil)), nil
-	}
-
-	functions["sha1"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"sha1": func(args ...interface{}) (interface{}, error) {
 		h := sha1.New()
-		_, err := h.Write([]byte(types.ToString(args[0])))
-
-		if err != nil {
+		if _, err := h.Write([]byte(types.ToString(args[0]))); err != nil {
 			return nil, err
 		}
 		return hex.EncodeToString(h.Sum(nil)), nil
-	}
-
-	functions["mmh3"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"mmh3": func(args ...interface{}) (interface{}, error) {
 		return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil
-	}
-
+	},
 	// search
-	functions["contains"] = func(args ...interface{}) (interface{}, error) {
+	"contains": func(args ...interface{}) (interface{}, error) {
 		return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil
-	}
-
-	functions["regex"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"regex": func(args ...interface{}) (interface{}, error) {
 		compiled, err := regexp.Compile(types.ToString(args[0]))
 		if err != nil {
 			return nil, err
 		}
 		return compiled.MatchString(types.ToString(args[1])), nil
-	}
-
+	},
 	// random generators
-	functions["rand_char"] = func(args ...interface{}) (interface{}, error) {
+	"rand_char": func(args ...interface{}) (interface{}, error) {
 		chars := letters + numbers
 		bad := ""
 		if len(args) >= 1 {
@@ -185,15 +152,14 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 		}
 		chars = trimAll(chars, bad)
 		return chars[rand.Intn(len(chars))], nil
-	}
-
-	functions["rand_base"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"rand_base": func(args ...interface{}) (interface{}, error) {
 		l := 0
 		bad := ""
 		base := letters + numbers
 
 		if len(args) >= 1 {
-			l = args[0].(int)
+			l = int(args[0].(float64))
 		}
 		if len(args) >= withCutSetArgsSize {
 			bad = types.ToString(args[1])
@@ -203,81 +169,100 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 		}
 		base = trimAll(base, bad)
 		return randSeq(base, l), nil
-	}
-
-	functions["rand_text_alphanumeric"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"rand_text_alphanumeric": func(args ...interface{}) (interface{}, error) {
 		l := 0
 		bad := ""
 		chars := letters + numbers
 
 		if len(args) >= 1 {
-			l = args[0].(int)
+			l = int(args[0].(float64))
 		}
 		if len(args) >= withCutSetArgsSize {
 			bad = types.ToString(args[1])
 		}
 		chars = trimAll(chars, bad)
 		return randSeq(chars, l), nil
-	}
-
-	functions["rand_text_alpha"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"rand_text_alpha": func(args ...interface{}) (interface{}, error) {
 		l := 0
 		bad := ""
 		chars := letters
 
 		if len(args) >= 1 {
-			l = args[0].(int)
+			l = int(args[0].(float64))
 		}
 		if len(args) >= withCutSetArgsSize {
 			bad = types.ToString(args[1])
 		}
 		chars = trimAll(chars, bad)
 		return randSeq(chars, l), nil
-	}
-
-	functions["rand_text_numeric"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"rand_text_numeric": func(args ...interface{}) (interface{}, error) {
 		l := 0
 		bad := ""
 		chars := numbers
 
 		if len(args) >= 1 {
-			l = args[0].(int)
+			l = int(args[0].(float64))
 		}
 		if len(args) >= withCutSetArgsSize {
 			bad = types.ToString(args[1])
 		}
 		chars = trimAll(chars, bad)
 		return randSeq(chars, l), nil
-	}
-
-	functions["rand_int"] = func(args ...interface{}) (interface{}, error) {
+	},
+	"rand_int": func(args ...interface{}) (interface{}, error) {
 		min := 0
 		max := math.MaxInt32
 
 		if len(args) >= 1 {
-			min = args[0].(int)
+			min = int(args[0].(float64))
 		}
 		if len(args) >= withMaxRandArgsSize {
-			max = args[1].(int)
+			max = int(args[1].(float64))
 		}
 		return rand.Intn(max-min) + min, nil
-	}
-
+	},
 	// Time Functions
-	functions["waitfor"] = func(args ...interface{}) (interface{}, error) {
+	"waitfor": func(args ...interface{}) (interface{}, error) {
 		seconds := args[0].(float64)
 		time.Sleep(time.Duration(seconds) * time.Second)
 		return true, nil
-	}
+	},
+	// deserialization Functions
+	"generate_java_gadget": func(args ...interface{}) (interface{}, error) {
+		gadget := args[0].(string)
+		cmd := args[1].(string)
 
-	// Collaborator
-	functions["collab"] = func(args ...interface{}) (interface{}, error) {
-		// check if collaborator contains a specific pattern
-		return collaborator.DefaultCollaborator.Has(types.ToString(args[0])), nil
-	}
+		var encoding string
+		if len(args) > 2 {
+			encoding = args[2].(string)
+		}
+		data := deserialization.GenerateJavaGadget(gadget, cmd, encoding)
+		return data, nil
+	},
+	// for debug purposes
+	"print_debug": func(args ...interface{}) (interface{}, error) {
+		gologger.Info().Msgf("print_debug value: %s", fmt.Sprint(args))
+		return true, nil
+	},
+}
+
+// HelperFunctions returns the dsl helper functions
+func HelperFunctions() map[string]govaluate.ExpressionFunction {
 	return functions
 }
 
+// AddHelperFunction allows creation of additional helper functions to be supported with templates
+func AddHelperFunction(key string, value func(args ...interface{}) (interface{}, error)) error {
+	if _, ok := functions[key]; !ok {
+		functions[key] = value
+		return nil
+	}
+	return errors.New("duplicate helper function key defined")
+}
+
 func reverseString(s string) string {
 	runes := []rune(s)
 	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
@@ -300,17 +285,3 @@ func randSeq(base string, n int) string {
 	}
 	return string(b)
 }
-
-func insertInto(s string, interval int, sep rune) string {
-	var buffer bytes.Buffer
-	before := interval - 1
-	last := len(s) - 1
-	for i, char := range s {
-		buffer.WriteRune(char)
-		if i%interval == before && i != last {
-			buffer.WriteRune(sep)
-		}
-	}
-	buffer.WriteRune(sep)
-	return buffer.String()
-}

v2/pkg/operators/common/dsl/dsl_test.go

@@ -0,0 +1,19 @@
+package dsl
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDSLURLEncodeDecode(t *testing.T) {
+	functions := HelperFunctions()
+
+	encoded, err := functions["url_encode"]("&test\"")
+	require.Nil(t, err, "could not url encode")
+	require.Equal(t, "%26test%22", encoded, "could not get url encoded data")
+
+	decoded, err := functions["url_decode"]("%26test%22")
+	require.Nil(t, err, "could not url encode")
+	require.Equal(t, "&test\"", decoded, "could not get url decoded data")
+}

v2/pkg/operators/extractors/compile.go

@@ -4,12 +4,14 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
+
+	"github.com/itchyny/gojq"
 )
 
-// CompileExtractors performs the initial setup operation on a extractor
+// CompileExtractors performs the initial setup operation on an extractor
 func (e *Extractor) CompileExtractors() error {
 	var ok bool
-	// Setup the extractor type
+	// Set up the extractor type
 	e.extractorType, ok = ExtractorTypes[e.Type]
 	if !ok {
 		return fmt.Errorf("unknown extractor type specified: %s", e.Type)
@@ -28,7 +30,19 @@ func (e *Extractor) CompileExtractors() error {
 		e.KVal[i] = strings.ToLower(kval)
 	}
 
-	// Setup the part of the request to match, if any.
+	for _, query := range e.JSON {
+		query, err := gojq.Parse(query)
+		if err != nil {
+			return fmt.Errorf("could not parse json: %s", query)
+		}
+		compiled, err := gojq.Compile(query)
+		if err != nil {
+			return fmt.Errorf("could not compile json: %s", query)
+		}
+		e.jsonCompiled = append(e.jsonCompiled, compiled)
+	}
+
+	// Set up the part of the request to match, if any.
 	if e.Part == "" {
 		e.Part = "body"
 	}

v2/pkg/operators/extractors/extract.go

@@ -1,6 +1,12 @@
 package extractors
 
 import (
+	"strings"
+
+	"encoding/json"
+
+	"github.com/antchfx/htmlquery"
+
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
@@ -42,3 +48,68 @@ func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{}
 	}
 	return results
 }
+
+// ExtractHTML extracts items from text using XPath selectors
+func (e *Extractor) ExtractHTML(corpus string) map[string]struct{} {
+	results := make(map[string]struct{})
+
+	doc, err := htmlquery.Parse(strings.NewReader(corpus))
+	if err != nil {
+		return results
+	}
+	for _, k := range e.XPath {
+		nodes, err := htmlquery.QueryAll(doc, k)
+		if err != nil {
+			continue
+		}
+		for _, node := range nodes {
+			var value string
+
+			if e.Attribute != "" {
+				value = htmlquery.SelectAttr(node, e.Attribute)
+			} else {
+				value = htmlquery.InnerText(node)
+			}
+			if _, ok := results[value]; !ok {
+				results[value] = struct{}{}
+			}
+		}
+	}
+	return results
+}
+
+// ExtractJSON extracts text from a corpus using JQ queries and returns it
+func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} {
+	results := make(map[string]struct{})
+
+	var jsonObj interface{}
+
+	if err := json.Unmarshal([]byte(corpus), &jsonObj); err != nil {
+		return results
+	}
+
+	for _, k := range e.jsonCompiled {
+		iter := k.Run(jsonObj)
+		for {
+			v, ok := iter.Next()
+			if !ok {
+				break
+			}
+			if _, ok := v.(error); ok {
+				break
+			}
+			var result string
+			if res, err := types.JSONScalarToString(v); err == nil {
+				result = res
+			} else if res, err := json.Marshal(v); err == nil {
+				result = string(res)
+			} else {
+				result = types.ToString(v)
+			}
+			if _, ok := results[result]; !ok {
+				results[result] = struct{}{}
+			}
+		}
+	}
+	return results
+}

v2/pkg/operators/extractors/extractors.go

@@ -1,32 +1,110 @@
 package extractors
 
-import "regexp"
+import (
+	"regexp"
+
+	"github.com/itchyny/gojq"
+)
 
 // Extractor is used to extract part of response using a regex.
 type Extractor struct {
-	// Name is the extractor's name
-	Name string `yaml:"name,omitempty"`
-	// Type is the type of the extractor
-	Type string `yaml:"type"`
+	// description: |
+	//   Name of the extractor. Name should be lowercase and must not contain
+	//   spaces or underscores (_).
+	// examples:
+	//   - value: "\"cookie-extractor\""
+	Name string `yaml:"name,omitempty" jsonschema:"title=name of the extractor,description=Name of the extractor"`
+	// description: |
+	//   Type is the type of the extractor.
+	// values:
+	//   - "regex"
+	//   - "kval"
+	//   - "json"
+	//   - "xpath"
+	Type string `yaml:"type" jsonschema:"title=type of the extractor,description=Type of the extractor,enum=regex,enum=kval,enum=json,enum=xpath"`
 	// extractorType is the internal type of the extractor
 	extractorType ExtractorType
 
-	// Regex are the regex pattern required to be present in the response
-	Regex []string `yaml:"regex"`
-	// RegexGroup specifies a group to extract from the regex
-	RegexGroup int `yaml:"group"`
+	// description: |
+	//   Regex contains the regular expression patterns to extract from a part.
+	//
+	//   Go regex engine does not support lookaheads or lookbehinds, so as a result
+	//   they are also not supported in nuclei.
+	// examples:
+	//   - name: Braintree Access Token Regex
+	//     value: >
+	//       []string{"access_token\\$production\\$[0-9a-z]{16}\\$[0-9a-f]{32}"}
+	//   - name: Wordpress Author Extraction regex
+	//     value: >
+	//       []string{"Author:(?:[A-Za-z0-9 -\\_=\"]+)?<span(?:[A-Za-z0-9 -\\_=\"]+)?>([A-Za-z0-9]+)<\\/span>"}
+	Regex []string `yaml:"regex,omitempty" jsonschema:"title=regex to extract from part,description=Regex to extract from part"`
+	// description: |
+	//   Group specifies a numbered group to extract from the regex.
+	// examples:
+	//   - name: Example Regex Group
+	//     value: "1"
+	RegexGroup int `yaml:"group,omitempty" jsonschema:"title=group to extract from regex,description=Group to extract from regex"`
 	// regexCompiled is the compiled variant
 	regexCompiled []*regexp.Regexp
 
-	// KVal are the kval to be present in the response headers/cookies
-	KVal []string `yaml:"kval,omitempty"`
-
-	// Part is the part of the request to match
+	// description: |
+	//   kval contains the key-value pairs present in the HTTP response header.
+	//   kval extractor can be used to extract HTTP response header and cookie key-value pairs.
+	//   kval extractor inputs are case-insensitive, and does not support dash (-) in input which can replaced with underscores (_)
+	// 	 For example, Content-Type should be replaced with content_type
 	//
-	// By default, matching is performed in request body.
-	Part string `yaml:"part,omitempty"`
-	// Internal defines if this is used internally
-	Internal bool `yaml:"internal,omitempty"`
+	//   A list of supported parts is available in docs for request types.
+	// examples:
+	//   - name: Extract Server Header From HTTP Response
+	//     value: >
+	//       []string{"server"}
+	//   - name: Extracting value of PHPSESSID Cookie
+	//     value: >
+	//       []string{"phpsessid"}
+	//   - name: Extracting value of Content-Type Cookie
+	//     value: >
+	//       []string{"content_type"}
+	KVal []string `yaml:"kval,omitempty" jsonschema:"title=kval pairs to extract from response,description=Kval pairs to extract from response"`
+
+	// description: |
+	//   JSON allows using jq-style syntax to extract items from json response
+	//
+	// examples:
+	//   - value: >
+	//       []string{".[] | .id"}
+	//   - value: >
+	//       []string{".batters | .batter | .[] | .id"}
+	JSON []string `yaml:"json,omitempty" jsonschema:"title=json jq expressions to extract data,description=JSON JQ expressions to evaluate from response part"`
+	// description: |
+	//   XPath allows using xpath expressions to extract items from html response
+	//
+	// examples:
+	//   - value: >
+	//       []string{"/html/body/div/p[2]/a"}
+	XPath []string `yaml:"xpath,omitempty" jsonschema:"title=html xpath expressions to extract data,description=XPath allows using xpath expressions to extract items from html response"`
+	// description: |
+	//   Attribute is an optional attribute to extract from response XPath.
+	//
+	// examples:
+	//   - value: "\"href\""
+	Attribute string `yaml:"attribute,omitempty" jsonschema:"title=optional attribute to extract from xpath,description=Optional attribute to extract from response XPath"`
+
+	// jsonCompiled is the compiled variant
+	jsonCompiled []*gojq.Code
+
+	// description: |
+	//   Part is the part of the request response to extract data from.
+	//
+	//   Each protocol exposes a lot of different parts which are well
+	//   documented in docs for each request type.
+	// examples:
+	//   - value: "\"body\""
+	//   - value: "\"raw\""
+	Part string `yaml:"part,omitempty" jsonschema:"title=part of response to extract data from,description=Part of the request response to extract data from"`
+	// description: |
+	//   Internal, when set to true will allow using the value extracted
+	//   in the next request for some protocols (like HTTP).
+	Internal bool `yaml:"internal,omitempty" jsonschema:"title=mark extracted value for internal variable use,description=Internal when set to true will allow using the value extracted in the next request for some protocols"`
 }
 
 // ExtractorType is the type of the extractor specified
@@ -37,12 +115,18 @@ const (
 	RegexExtractor ExtractorType = iota + 1
 	// KValExtractor extracts responses with key:value
 	KValExtractor
+	// XPathExtractor extracts responses with Xpath selectors
+	XPathExtractor
+	// JSONExtractor extracts responses with json
+	JSONExtractor
 )
 
-// ExtractorTypes is an table for conversion of extractor type from string.
+// ExtractorTypes is a table for conversion of extractor type from string.
 var ExtractorTypes = map[string]ExtractorType{
 	"regex": RegexExtractor,
 	"kval":  KValExtractor,
+	"xpath": XPathExtractor,
+	"json":  JSONExtractor,
 }
 
 // GetType returns the type of the matcher

v2/pkg/operators/matchers/compile.go

@@ -6,6 +6,7 @@ import (
 	"regexp"
 
 	"github.com/Knetic/govaluate"
+
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
 )
 
@@ -22,7 +23,7 @@ func (m *Matcher) CompileMatchers() error {
 		}
 	}
 
-	// Setup the matcher type
+	// Set up the matcher type
 	m.matcherType, ok = MatcherTypes[m.Type]
 	if !ok {
 		return fmt.Errorf("unknown matcher type specified: %s", m.Type)
@@ -50,7 +51,7 @@ func (m *Matcher) CompileMatchers() error {
 		m.dslCompiled = append(m.dslCompiled, compiled)
 	}
 
-	// Setup the condition type, if any.
+	// Set up the condition type, if any.
 	if m.Condition != "" {
 		m.condition, ok = ConditionTypes[m.Condition]
 		if !ok {

v2/pkg/operators/matchers/match.go

@@ -48,7 +48,7 @@ func (m *Matcher) MatchWords(corpus string) bool {
 			if m.condition == ANDCondition {
 				return false
 			}
-			// Continue with the flow since its an OR Condition.
+			// Continue with the flow since it's an OR Condition.
 			continue
 		}
 
@@ -76,7 +76,7 @@ func (m *Matcher) MatchRegex(corpus string) bool {
 			if m.condition == ANDCondition {
 				return false
 			}
-			// Continue with the flow since its an OR Condition.
+			// Continue with the flow since it's an OR Condition.
 			continue
 		}
 
@@ -105,7 +105,7 @@ func (m *Matcher) MatchBinary(corpus string) bool {
 			if m.condition == ANDCondition {
 				return false
 			}
-			// Continue with the flow since its an OR Condition.
+			// Continue with the flow since it's an OR Condition.
 			continue
 		}
 
@@ -141,7 +141,7 @@ func (m *Matcher) MatchDSL(data map[string]interface{}) bool {
 			if m.condition == ANDCondition {
 				return false
 			}
-			// Continue with the flow since its an OR Condition.
+			// Continue with the flow since it's an OR Condition.
 			continue
 		}
 

v2/pkg/operators/matchers/matchers.go

@@ -8,36 +8,103 @@ import (
 
 // Matcher is used to match a part in the output from a protocol.
 type Matcher struct {
-	// Type is the type of the matcher
-	Type string `yaml:"type"`
-	// Condition is the optional condition between two matcher variables
+	// description: |
+	//   Type is the type of the matcher.
+	// values:
+	//   - "status"
+	//   - "size"
+	//   - "word"
+	//   - "regex"
+	//   - "binary"
+	//   - "dsl"
+	Type string `yaml:"type" jsonschema:"title=type of matcher,description=Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl"`
+	// description: |
+	//   Condition is the optional condition between two matcher variables. By default,
+	//   the condition is assumed to be OR.
+	// values:
+	//   - "and"
+	//   - "or"
+	Condition string `yaml:"condition,omitempty" jsonschema:"title=condition between matcher variables,description=Condition between the matcher variables,enum=and,enum=or"`
+
+	// description: |
+	//   Part is the part of the request response to match data from.
 	//
-	// By default, the condition is assumed to be OR.
-	Condition string `yaml:"condition,omitempty"`
+	//   Each protocol exposes a lot of different parts which are well
+	//   documented in docs for each request type.
+	// examples:
+	//   - value: "\"body\""
+	//   - value: "\"raw\""
+	Part string `yaml:"part,omitempty" jsonschema:"title=part of response to match,description=Part of response to match data from"`
 
-	// Part is the part of the data to match
-	Part string `yaml:"part,omitempty"`
+	// description: |
+	//   Negative specifies if the match should be reversed
+	//   It will only match if the condition is not true.
+	Negative bool `yaml:"negative,omitempty" jsonschema:"title=negative specifies if match reversed,description=Negative specifies if the match should be reversed. It will only match if the condition is not true"`
 
-	// Negative specifies if the match should be reversed
-	// It will only match if the condition is not true.
-	Negative bool `yaml:"negative,omitempty"`
-
-	// Name is matcher Name
-	Name string `yaml:"name,omitempty"`
-	// Status are the acceptable status codes for the response
-	Status []int `yaml:"status,omitempty"`
-	// Size is the acceptable size for the response
-	Size []int `yaml:"size,omitempty"`
-	// Words are the words required to be present in the response
-	Words []string `yaml:"words,omitempty"`
-	// Regex are the regex pattern required to be present in the response
-	Regex []string `yaml:"regex,omitempty"`
-	// Binary are the binary characters required to be present in the response
-	Binary []string `yaml:"binary,omitempty"`
-	// DSL are the dsl queries
-	DSL []string `yaml:"dsl,omitempty"`
-	// Encoding specifies the encoding for the word content if any.
-	Encoding string `yaml:"encoding,omitempty"`
+	// description: |
+	//   Name of the matcher. Name should be lowercase and must not contain
+	//   spaces or underscores (_).
+	// examples:
+	//   - value: "\"cookie-matcher\""
+	Name string `yaml:"name,omitempty" jsonschema:"title=name of the matcher,description=Name of the matcher"`
+	// description: |
+	//   Status are the acceptable status codes for the response.
+	// examples:
+	//   - value: >
+	//       []int{200, 302}
+	Status []int `yaml:"status,omitempty" jsonschema:"title=status to match,description=Status to match for the response"`
+	// description: |
+	//   Size is the acceptable size for the response
+	// examples:
+	//   - value: >
+	//       []int{3029, 2042}
+	Size []int `yaml:"size,omitempty" jsonschema:"title=acceptable size for response,description=Size is the acceptable size for the response"`
+	// description: |
+	//   Words contains word patterns required to be present in the response part.
+	// examples:
+	//   - name: Match for outlook mail protection domain
+	//     value: >
+	//       []string{"mail.protection.outlook.com"}
+	//   - name: Match for application/json in response headers
+	//     value: >
+	//       []string{"application/json"}
+	Words []string `yaml:"words,omitempty" jsonschema:"title=words to match in response,description= Words contains word patterns required to be present in the response part"`
+	// description: |
+	//   Regex contains Regular Expression patterns required to be present in the response part.
+	// examples:
+	//   - name: Match for Linkerd Service via Regex
+	//     value: >
+	//       []string{`(?mi)^Via\\s*?:.*?linkerd.*$`}
+	//   - name: Match for Open Redirect via Location header
+	//     value: >
+	//       []string{`(?m)^(?:Location\\s*?:\\s*?)(?:https?://|//)?(?:[a-zA-Z0-9\\-_\\.@]*)example\\.com.*$`}
+	Regex []string `yaml:"regex,omitempty" jsonschema:"title=regex to match in response,description=Regex contains regex patterns required to be present in the response part"`
+	// description: |
+	//   Binary are the binary patterns required to be present in the response part.
+	// examples:
+	//   - name: Match for Springboot Heapdump Actuator "JAVA PROFILE", "HPROF", "Gunzip magic byte"
+	//     value: >
+	//       []string{"4a4156412050524f46494c45", "4850524f46", "1f8b080000000000"}
+	//   - name: Match for 7zip files
+	//     value: >
+	//       []string{"377ABCAF271C"}
+	Binary []string `yaml:"binary,omitempty" jsonschema:"title=binary patterns to match in response,description=Binary are the binary patterns required to be present in the response part"`
+	// description: |
+	//   DSL are the dsl expressions that will be evaluated as part of nuclei matching rules.
+	//   A list of these helper functions are available [here](https://nuclei.projectdiscovery.io/templating-guide/helper-functions/).
+	// examples:
+	//   - name: DSL Matcher for package.json file
+	//     value: >
+	//       []string{"contains(body, 'packages') && contains(tolower(all_headers), 'application/octet-stream') && status_code == 200"}
+	//   - name: DSL Matcher for missing strict transport security header
+	//     value: >
+	//       []string{"!contains(tolower(all_headers), ''strict-transport-security'')"}
+	DSL []string `yaml:"dsl,omitempty" jsonschema:"title=dsl expressions to match in response,description=DSL are the dsl expressions that will be evaluated as part of nuclei matching rules"`
+	// description: |
+	//   Encoding specifies the encoding for the words field if any.
+	// values:
+	//   - "hex"
+	Encoding string `yaml:"encoding,omitempty" jsonschema:"title=encoding for word field,description=Optional encoding for the word fields,enum=hex"`
 
 	// cached data for the compiled matcher
 	condition     ConditionType
@@ -64,7 +131,7 @@ const (
 	DSLMatcher
 )
 
-// MatcherTypes is an table for conversion of matcher type from string.
+// MatcherTypes is a table for conversion of matcher type from string.
 var MatcherTypes = map[string]MatcherType{
 	"status": StatusMatcher,
 	"size":   SizeMatcher,
@@ -84,7 +151,7 @@ const (
 	ORCondition
 )
 
-// ConditionTypes is an table for conversion of condition type from string.
+// ConditionTypes is a table for conversion of condition type from string.
 var ConditionTypes = map[string]ConditionType{
 	"and": ANDCondition,
 	"or":  ORCondition,

v2/pkg/operators/operators.go

@@ -2,21 +2,31 @@ package operators
 
 import (
 	"github.com/pkg/errors"
+
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 )
 
 // Operators contains the operators that can be applied on protocols
 type Operators struct {
-	// Matchers contains the detection mechanism for the request to identify
-	// whether the request was successful
-	Matchers []*matchers.Matcher `yaml:"matchers,omitempty"`
-	// Extractors contains the extraction mechanism for the request to identify
-	// and extract parts of the response.
-	Extractors []*extractors.Extractor `yaml:"extractors,omitempty"`
-	// MatchersCondition is the condition of the matchers
-	// whether to use AND or OR. Default is OR.
-	MatchersCondition string `yaml:"matchers-condition,omitempty"`
+	// description: |
+	//   Matchers contains the detection mechanism for the request to identify
+	//   whether the request was successful by doing pattern matching
+	//   on request/responses.
+	//
+	//   Multiple matchers can be combined with `matcher-condition` flag
+	//   which accepts either `and` or `or` as argument.
+	Matchers []*matchers.Matcher `yaml:"matchers,omitempty" jsonschema:"title=matchers to run on response,description=Detection mechanism to identify whether the request was successful by doing pattern matching"`
+	// description: |
+	//   Extractors contains the extraction mechanism for the request to identify
+	//   and extract parts of the response.
+	Extractors []*extractors.Extractor `yaml:"extractors,omitempty" jsonschema:"title=extractors to run on response,description=Extractors contains the extraction mechanism for the request to identify and extract parts of the response"`
+	// description: |
+	//   MatchersCondition is the condition between the matchers. Default is OR.
+	// values:
+	//   - "and"
+	//   - "or"
+	MatchersCondition string `yaml:"matchers-condition,omitempty" jsonschema:"title=condition between the matchers,description=Conditions between the matchers,enum=and,enum=or"`
 	// cached variables that may be used along with request.
 	matchersCondition matchers.ConditionType
 }
@@ -65,10 +75,34 @@ type Result struct {
 	PayloadValues map[string]interface{}
 }
 
+// Merge merges a result structure into the other.
+func (r *Result) Merge(result *Result) {
+	if !r.Matched && result.Matched {
+		r.Matched = result.Matched
+	}
+	if !r.Extracted && result.Extracted {
+		r.Extracted = result.Extracted
+	}
+
+	for k, v := range result.Matches {
+		r.Matches[k] = v
+	}
+	for k, v := range result.Extracts {
+		r.Extracts[k] = v
+	}
+	r.OutputExtracts = append(r.OutputExtracts, result.OutputExtracts...)
+	for k, v := range result.DynamicValues {
+		r.DynamicValues[k] = v
+	}
+	for k, v := range result.PayloadValues {
+		r.PayloadValues[k] = v
+	}
+}
+
 // MatchFunc performs matching operation for a matcher on model and returns true or false.
 type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) bool
 
-// ExtractFunc performs extracting operation for a extractor on model and returns true or false.
+// ExtractFunc performs extracting operation for an extractor on model and returns true or false.
 type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
 
 // Execute executes the operators on data and returns a result structure
@@ -81,6 +115,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac
 		Extracts:      make(map[string][]string),
 		DynamicValues: make(map[string]interface{}),
 	}
+
 	// Start with the extractors first and evaluate them.
 	for _, extractor := range r.Extractors {
 		var extractorResults []string
@@ -112,7 +147,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac
 				return nil, false
 			}
 		} else {
-			// If the matcher has matched, and its an OR
+			// If the matcher has matched, and it's an OR
 			// write the first output then move to next matcher.
 			if matcherCondition == matchers.ORCondition && matcher.Name != "" {
 				result.Matches[matcher.Name] = struct{}{}
@@ -126,7 +161,8 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac
 	if len(result.DynamicValues) > 0 {
 		return result, true
 	}
-	// Don't print if we have matchers and they have not matched, irregardless of extractor
+
+	// Don't print if we have matchers and they have not matched, regardless of extractor
 	if len(r.Matchers) > 0 && !matches {
 		return nil, false
 	}
@@ -137,3 +173,21 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac
 	}
 	return nil, false
 }
+
+// ExecuteInternalExtractors executes internal dynamic extractors
+func (r *Operators) ExecuteInternalExtractors(data map[string]interface{}, extract ExtractFunc) map[string]interface{} {
+	dynamicValues := make(map[string]interface{})
+
+	// Start with the extractors first and evaluate them.
+	for _, extractor := range r.Extractors {
+		if !extractor.Internal {
+			continue
+		}
+		for match := range extract(data, extractor) {
+			if _, ok := dynamicValues[extractor.Name]; !ok {
+				dynamicValues[extractor.Name] = match
+			}
+		}
+	}
+	return dynamicValues
+}

v2/pkg/output/file_output_writer.go

@@ -20,11 +20,10 @@ func newFileOutputWriter(file string) (*fileWriter, error) {
 
 // WriteString writes an output to the underlying file
 func (w *fileWriter) Write(data []byte) error {
-	_, err := w.file.Write(data)
-	if err != nil {
+	if _, err := w.file.Write(data); err != nil {
 		return err
 	}
-	_, err = w.file.Write([]byte("\n"))
+	_, err := w.file.Write([]byte("\n"))
 	return err
 }
 

v2/pkg/output/format_json.go

@@ -6,5 +6,9 @@ import (
 
 // formatJSON formats the output for json based formatting
 func (w *StandardWriter) formatJSON(output *ResultEvent) ([]byte, error) {
+	if !w.jsonReqResp { // don't show request-response in json if not asked
+		output.Request = ""
+		output.Response = ""
+	}
 	return jsoniter.Marshal(output)
 }

v2/pkg/output/format_screen.go

@@ -11,10 +11,11 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
 	builder := &bytes.Buffer{}
 
 	if !w.noMetadata {
-		builder.WriteRune('[')
-		builder.WriteString(w.aurora.Cyan(output.Timestamp.Format("2006-01-02 15:04:05")).String())
-		builder.WriteString("] ")
-
+		if !w.noTimestamp {
+			builder.WriteRune('[')
+			builder.WriteString(w.aurora.Cyan(output.Timestamp.Format("2006-01-02 15:04:05")).String())
+			builder.WriteString("] ")
+		}
 		builder.WriteRune('[')
 		builder.WriteString(w.aurora.BrightGreen(output.TemplateID).String())
 
@@ -31,7 +32,7 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
 		builder.WriteString("] ")
 
 		builder.WriteString("[")
-		builder.WriteString(w.severityColors.Data[types.ToString(output.Info["severity"])])
+		builder.WriteString(w.severityColors(output.Info.SeverityHolder.Severity))
 		builder.WriteString("] ")
 	}
 	builder.WriteString(output.Matched)
@@ -54,7 +55,7 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
 	if len(output.Metadata) > 0 {
 		builder.WriteString(" [")
 
-		var first bool = true
+		first := true
 		for name, value := range output.Metadata {
 			if !first {
 				builder.WriteRune(',')

v2/pkg/output/output.go

@@ -6,10 +6,15 @@ import (
 	"sync"
 	"time"
 
+	"github.com/pkg/errors"
+
 	jsoniter "github.com/json-iterator/go"
 	"github.com/logrusorgru/aurora"
-	"github.com/pkg/errors"
+
+	"github.com/projectdiscovery/interactsh/pkg/server"
 	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 )
 
@@ -28,13 +33,15 @@ type Writer interface {
 // StandardWriter is a writer writing output to file and screen for results.
 type StandardWriter struct {
 	json           bool
+	jsonReqResp    bool
+	noTimestamp    bool
 	noMetadata     bool
 	aurora         aurora.Aurora
 	outputFile     *fileWriter
 	outputMutex    *sync.Mutex
 	traceFile      *fileWriter
 	traceMutex     *sync.Mutex
-	severityColors *colorizer.Colorizer
+	severityColors func(severity.Severity) string
 }
 
 var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
@@ -53,8 +60,10 @@ type InternalWrappedEvent struct {
 type ResultEvent struct {
 	// TemplateID is the ID of the template for the result.
 	TemplateID string `json:"templateID"`
+	// TemplatePath is the path of template
+	TemplatePath string `json:"-"`
 	// Info contains information block of the template for the result.
-	Info map[string]interface{} `json:"info,inline"`
+	Info model.Info `json:"info,inline"`
 	// MatcherName is the name of the matcher matched if any.
 	MatcherName string `json:"matcher_name,omitempty"`
 	// ExtractorName is the name of the extractor matched if any.
@@ -69,9 +78,9 @@ type ResultEvent struct {
 	Matched string `json:"matched,omitempty"`
 	// ExtractedResults contains the extraction result from the inputs.
 	ExtractedResults []string `json:"extracted_results,omitempty"`
-	// Request is the optional dumped request for the match.
+	// Request is the optional, dumped request for the match.
 	Request string `json:"request,omitempty"`
-	// Response is the optional dumped response for the match.
+	// Response is the optional, dumped response for the match.
 	Response string `json:"response,omitempty"`
 	// Metadata contains any optional metadata for the event
 	Metadata map[string]interface{} `json:"meta,omitempty"`
@@ -79,10 +88,14 @@ type ResultEvent struct {
 	IP string `json:"ip,omitempty"`
 	// Timestamp is the time the result was found at.
 	Timestamp time.Time `json:"timestamp"`
+	// Interaction is the full details of interactsh interaction.
+	Interaction *server.Interaction `json:"interaction,omitempty"`
+
+	FileToIndexPosition map[string]int `json:"-"`
 }
 
 // NewStandardWriter creates a new output writer based on user configurations
-func NewStandardWriter(colors, noMetadata, json bool, file, traceFile string) (*StandardWriter, error) {
+func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, file, traceFile string) (*StandardWriter, error) {
 	auroraColorizer := aurora.NewAurora(colors)
 
 	var outputFile *fileWriter
@@ -103,7 +116,9 @@ func NewStandardWriter(colors, noMetadata, json bool, file, traceFile string) (*
 	}
 	writer := &StandardWriter{
 		json:           json,
+		jsonReqResp:    jsonReqResp,
 		noMetadata:     noMetadata,
+		noTimestamp:    noTimestamp,
 		aurora:         auroraColorizer,
 		outputFile:     outputFile,
 		outputMutex:    &sync.Mutex{},

v2/pkg/parsers/parser.go

@@ -0,0 +1,141 @@
+package parsers
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+
+	"gopkg.in/yaml.v2"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates/cache"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
+)
+
+const mandatoryFieldMissingTemplate = "mandatory '%s' field is missing"
+
+// LoadTemplate returns true if the template is valid and matches the filtering criteria.
+func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string) (bool, error) {
+	template, templateParseError := ParseTemplate(templatePath)
+	if templateParseError != nil {
+		return false, templateParseError
+	}
+
+	if len(template.Workflows) > 0 {
+		return false, nil
+	}
+
+	templateInfo := template.Info
+	if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
+		return false, validationError
+	}
+
+	return isTemplateInfoMetadataMatch(tagFilter, &templateInfo, extraTags)
+}
+
+// LoadWorkflow returns true if the workflow is valid and matches the filtering criteria.
+func LoadWorkflow(templatePath string) (bool, error) {
+	template, templateParseError := ParseTemplate(templatePath)
+	if templateParseError != nil {
+		return false, templateParseError
+	}
+
+	templateInfo := template.Info
+
+	if len(template.Workflows) > 0 {
+		if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
+			return false, validationError
+		}
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string) (bool, error) {
+	templateTags := templateInfo.Tags.ToSlice()
+	templateAuthors := templateInfo.Authors.ToSlice()
+	templateSeverity := templateInfo.SeverityHolder.Severity
+
+	match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags)
+
+	if err == filter.ErrExcluded {
+		return false, filter.ErrExcluded
+	}
+
+	return match, err
+}
+
+func validateMandatoryInfoFields(info *model.Info) error {
+	if info == nil {
+		return fmt.Errorf(mandatoryFieldMissingTemplate, "info")
+	}
+
+	if utils.IsBlank(info.Name) {
+		return fmt.Errorf(mandatoryFieldMissingTemplate, "name")
+	}
+
+	if info.Authors.IsEmpty() {
+		return fmt.Errorf(mandatoryFieldMissingTemplate, "author")
+	}
+	return nil
+}
+
+var (
+	parsedTemplatesCache *cache.Templates
+	ShouldValidate       bool
+	fieldErrorRegexp     = regexp.MustCompile(`not found in`)
+)
+
+const (
+	SyntaxWarningStats = "syntax-warnings"
+	SyntaxErrorStats   = "syntax-errors"
+)
+
+func init() {
+
+	parsedTemplatesCache = cache.New()
+
+	stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)")
+	stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)")
+}
+
+// ParseTemplate parses a template and returns a *templates.Template structure
+func ParseTemplate(templatePath string) (*templates.Template, error) {
+	if value, err := parsedTemplatesCache.Has(templatePath); value != nil {
+		return value.(*templates.Template), err
+	}
+
+	f, err := os.Open(templatePath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+
+	template := &templates.Template{}
+	if err := yaml.UnmarshalStrict(data, template); err != nil {
+		errString := err.Error()
+		if !fieldErrorRegexp.MatchString(errString) {
+			stats.Increment(SyntaxErrorStats)
+			return nil, err
+		}
+		stats.Increment(SyntaxWarningStats)
+		if ShouldValidate {
+			gologger.Error().Msgf("Syntax warnings for template %s: %s", templatePath, err)
+		} else {
+			gologger.Warning().Msgf("Syntax warnings for template %s: %s", templatePath, err)
+		}
+	}
+	parsedTemplatesCache.Store(templatePath, template, nil)
+	return template, nil
+}

v2/pkg/parsers/workflow_loader.go

@@ -0,0 +1,62 @@
+package parsers
+
+import (
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+)
+
+type workflowLoader struct {
+	pathFilter *filter.PathFilter
+	tagFilter  *filter.TagFilter
+	options    *protocols.ExecuterOptions
+}
+
+// NewLoader returns a new workflow loader structure
+func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error) {
+	tagFilter := filter.New(&filter.Config{
+		Tags:        options.Options.Tags,
+		ExcludeTags: options.Options.ExcludeTags,
+		Authors:     options.Options.Author,
+		Severities:  options.Options.Severities,
+		IncludeTags: options.Options.IncludeTags,
+	})
+	pathFilter := filter.NewPathFilter(&filter.PathFilterConfig{
+		IncludedTemplates: options.Options.IncludeTemplates,
+		ExcludedTemplates: options.Options.ExcludedTemplates,
+	}, options.Catalog)
+	return &workflowLoader{pathFilter: pathFilter, tagFilter: tagFilter, options: options}, nil
+}
+
+func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string {
+	includedTemplates := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory})
+	templatePathMap := w.pathFilter.Match(includedTemplates)
+
+	loadedTemplates := make([]string, 0, len(templatePathMap))
+	for templatePath := range templatePathMap {
+		loaded, err := LoadTemplate(templatePath, w.tagFilter, templateTags)
+		if err != nil {
+			gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
+		} else if loaded {
+			loadedTemplates = append(loadedTemplates, templatePath)
+		}
+	}
+	return loadedTemplates
+}
+
+func (w *workflowLoader) GetTemplatePaths(templatesList []string, noValidate bool) []string {
+	includedTemplates := w.options.Catalog.GetTemplatesPath(templatesList)
+	templatesPathMap := w.pathFilter.Match(includedTemplates)
+
+	loadedTemplates := make([]string, 0, len(templatesPathMap))
+	for templatePath := range templatesPathMap {
+		matched, err := LoadTemplate(templatePath, w.tagFilter, nil)
+		if err != nil {
+			gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
+		} else if matched || noValidate {
+			loadedTemplates = append(loadedTemplates, templatePath)
+		}
+	}
+	return loadedTemplates
+}

v2/pkg/progress/progress.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -39,13 +40,14 @@ var _ Progress = &StatsTicker{}
 // StatsTicker is a progress instance for showing program stats
 type StatsTicker struct {
 	active       bool
-	tickDuration time.Duration
-	stats        clistats.StatisticsClient
+	outputJSON   bool
 	server       *http.Server
+	stats        clistats.StatisticsClient
+	tickDuration time.Duration
 }
 
 // NewStatsTicker creates and returns a new progress tracking object.
-func NewStatsTicker(duration int, active, metrics bool, port int) (Progress, error) {
+func NewStatsTicker(duration int, active, outputJSON, metrics bool, port int) (Progress, error) {
 	var tickDuration time.Duration
 	if active {
 		tickDuration = time.Duration(duration) * time.Second
@@ -62,6 +64,7 @@ func NewStatsTicker(duration int, active, metrics bool, port int) (Progress, err
 	progress.active = active
 	progress.stats = stats
 	progress.tickDuration = tickDuration
+	progress.outputJSON = outputJSON
 
 	if metrics {
 		http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
@@ -92,7 +95,13 @@ func (p *StatsTicker) Init(hostCount int64, rulesCount int, requestCount int64)
 	p.stats.AddCounter("total", uint64(requestCount))
 
 	if p.active {
-		if err := p.stats.Start(printCallback, p.tickDuration); err != nil {
+		var printCallbackFunc clistats.PrintCallback
+		if p.outputJSON {
+			printCallbackFunc = printCallbackJSON
+		} else {
+			printCallbackFunc = printCallback
+		}
+		if err := p.stats.Start(printCallbackFunc, p.tickDuration); err != nil {
 			gologger.Warning().Msgf("Couldn't start statistics: %s", err)
 		}
 	}
@@ -167,30 +176,35 @@ func printCallback(stats clistats.StatisticsClient) {
 	builder.WriteRune(')')
 	builder.WriteRune('\n')
 
-	gologger.Print().Msgf("%s", builder.String())
+	fmt.Fprintf(os.Stderr, "%s", builder.String())
 }
 
-// getMetrics returns a map of important metrics for client
-func (p *StatsTicker) getMetrics() map[string]interface{} {
+func printCallbackJSON(stats clistats.StatisticsClient) {
+	builder := &strings.Builder{}
+	_ = json.NewEncoder(builder).Encode(metricsMap(stats))
+	fmt.Fprintf(os.Stderr, "%s", builder.String())
+}
+
+func metricsMap(stats clistats.StatisticsClient) map[string]interface{} {
 	results := make(map[string]interface{})
 
-	startedAt, _ := p.stats.GetStatic("startedAt")
+	startedAt, _ := stats.GetStatic("startedAt")
 	duration := time.Since(startedAt.(time.Time))
 
 	results["startedAt"] = startedAt.(time.Time)
 	results["duration"] = fmtDuration(duration)
-	templates, _ := p.stats.GetStatic("templates")
+	templates, _ := stats.GetStatic("templates")
 	results["templates"] = clistats.String(templates)
-	hosts, _ := p.stats.GetStatic("hosts")
+	hosts, _ := stats.GetStatic("hosts")
 	results["hosts"] = clistats.String(hosts)
-	matched, _ := p.stats.GetCounter("matched")
+	matched, _ := stats.GetCounter("matched")
 	results["matched"] = clistats.String(matched)
-	requests, _ := p.stats.GetCounter("requests")
+	requests, _ := stats.GetCounter("requests")
 	results["requests"] = clistats.String(requests)
-	total, _ := p.stats.GetCounter("total")
+	total, _ := stats.GetCounter("total")
 	results["total"] = clistats.String(total)
 	results["rps"] = clistats.String(uint64(float64(requests) / duration.Seconds()))
-	errors, _ := p.stats.GetCounter("errors")
+	errors, _ := stats.GetCounter("errors")
 	results["errors"] = clistats.String(errors)
 
 	//nolint:gomnd // this is not a magic number
@@ -200,6 +214,11 @@ func (p *StatsTicker) getMetrics() map[string]interface{} {
 	return results
 }
 
+// getMetrics returns a map of important metrics for client
+func (p *StatsTicker) getMetrics() map[string]interface{} {
+	return metricsMap(p.stats)
+}
+
 // fmtDuration formats the duration for the time elapsed
 func fmtDuration(d time.Duration) string {
 	d = d.Round(time.Second)
@@ -215,7 +234,11 @@ func fmtDuration(d time.Duration) string {
 func (p *StatsTicker) Stop() {
 	if p.active {
 		// Print one final summary
-		printCallback(p.stats)
+		if p.outputJSON {
+			printCallbackJSON(p.stats)
+		} else {
+			printCallback(p.stats)
+		}
 		if err := p.stats.Stop(); err != nil {
 			gologger.Warning().Msgf("Couldn't stop statistics: %s", err)
 		}

v2/pkg/projectfile/httputil.go

@@ -18,8 +18,7 @@ func hash(v interface{}) (string, error) {
 
 	sh := sha256.New()
 
-	_, err = io.WriteString(sh, string(data))
-	if err != nil {
+	if _, err = io.WriteString(sh, string(data)); err != nil {
 		return "", err
 	}
 	return hex.EncodeToString(sh.Sum(nil)), nil
@@ -28,8 +27,7 @@ func hash(v interface{}) (string, error) {
 func marshal(data interface{}) ([]byte, error) {
 	var b bytes.Buffer
 	enc := gob.NewEncoder(&b)
-	err := enc.Encode(data)
-	if err != nil {
+	if err := enc.Encode(data); err != nil {
 		return nil, err
 	}
 
@@ -38,8 +36,7 @@ func marshal(data interface{}) ([]byte, error) {
 
 func unmarshal(data []byte, obj interface{}) error {
 	dec := gob.NewDecoder(bytes.NewBuffer(data))
-	err := dec.Decode(obj)
-	if err != nil {
+	if err := dec.Decode(obj); err != nil {
 		return err
 	}
 

v2/pkg/projectfile/project.go

@@ -44,8 +44,7 @@ func (pf *ProjectFile) Get(req []byte) (*http.Response, error) {
 
 	var httprecord HTTPRecord
 	httprecord.Response = newInternalResponse()
-	err = unmarshal(data, &httprecord)
-	if err != nil {
+	if err := unmarshal(data, &httprecord); err != nil {
 		return nil, err
 	}
 

v2/pkg/protocols/common/clusterer/executer.go

@@ -2,6 +2,7 @@ package clusterer
 
 import (
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
@@ -22,7 +23,8 @@ type Executer struct {
 
 type clusteredOperator struct {
 	templateID   string
-	templateInfo map[string]interface{}
+	templatePath string
+	templateInfo model.Info
 	operator     *operators.Operators
 }
 
@@ -38,6 +40,7 @@ func NewExecuter(requests []*templates.Template, options *protocols.ExecuterOpti
 		executer.operators = append(executer.operators, &clusteredOperator{
 			templateID:   req.ID,
 			templateInfo: req.Info,
+			templatePath: req.Path,
 			operator:     req.RequestsHTTP[0].CompiledOperators,
 		})
 	}
@@ -60,13 +63,15 @@ func (e *Executer) Requests() int {
 func (e *Executer) Execute(input string) (bool, error) {
 	var results bool
 
+	previous := make(map[string]interface{})
 	dynamicValues := make(map[string]interface{})
-	err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) {
+	err := e.requests.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
 		for _, operator := range e.operators {
 			result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract)
 			if matched && result != nil {
 				event.OperatorsResult = result
 				event.InternalEvent["template-id"] = operator.templateID
+				event.InternalEvent["template-path"] = operator.templatePath
 				event.InternalEvent["template-info"] = operator.templateInfo
 				event.Results = e.requests.MakeResultEvent(event)
 				results = true
@@ -82,6 +87,9 @@ func (e *Executer) Execute(input string) (bool, error) {
 			}
 		}
 	})
+	if err != nil && e.options.HostErrorsCache != nil && e.options.HostErrorsCache.CheckError(err) {
+		e.options.HostErrorsCache.MarkFailed(input)
+	}
 	return results, err
 }
 
@@ -94,11 +102,15 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve
 			if matched && result != nil {
 				event.OperatorsResult = result
 				event.InternalEvent["template-id"] = operator.templateID
+				event.InternalEvent["template-path"] = operator.templatePath
 				event.InternalEvent["template-info"] = operator.templateInfo
 				event.Results = e.requests.MakeResultEvent(event)
 				callback(event)
 			}
 		}
 	})
+	if err != nil && e.options.HostErrorsCache != nil && e.options.HostErrorsCache.CheckError(err) {
+		e.options.HostErrorsCache.MarkFailed(input)
+	}
 	return err
 }

v2/pkg/protocols/common/executer/executer.go

@@ -24,8 +24,7 @@ func NewExecuter(requests []protocols.Request, options *protocols.ExecuterOption
 // Compile compiles the execution generators preparing any requests possible.
 func (e *Executer) Compile() error {
 	for _, request := range e.requests {
-		err := request.Compile(e.options)
-		if err != nil {
+		if err := request.Compile(e.options); err != nil {
 			return err
 		}
 	}
@@ -77,6 +76,11 @@ func (e *Executer) Execute(input string) (bool, error) {
 			}
 		})
 		if err != nil {
+			if e.options.HostErrorsCache != nil {
+				if e.options.HostErrorsCache.CheckError(err) {
+					e.options.HostErrorsCache.MarkFailed(input)
+				}
+			}
 			gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err)
 		}
 	}
@@ -109,6 +113,11 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve
 			callback(event)
 		})
 		if err != nil {
+			if e.options.HostErrorsCache != nil {
+				if e.options.HostErrorsCache.CheckError(err) {
+					e.options.HostErrorsCache.MarkFailed(input)
+				}
+			}
 			gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err)
 		}
 	}

v2/pkg/protocols/common/expressions/expressions.go

@@ -37,3 +37,30 @@ func Evaluate(data string, base map[string]interface{}) (string, error) {
 	// Replacer dynamic values if any in raw request and parse  it
 	return replacer.Replace(data, dynamicValues), nil
 }
+
+// EvaluateByte checks if the match contains a dynamic variable, for each
+// found one we will check if it's an expression and can
+// be compiled, it will be evaluated and the results will be returned.
+//
+// The provided keys from finalValues will be used as variable names
+// for substitution inside the expression.
+func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) {
+	final := replacer.Replace(string(data), base)
+
+	dynamicValues := make(map[string]interface{})
+	for _, match := range templateExpressionRegex.FindAllString(final, -1) {
+		expr := generators.TrimDelimiters(match)
+
+		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
+		if err != nil {
+			continue
+		}
+		result, err := compiled.Evaluate(base)
+		if err != nil {
+			continue
+		}
+		dynamicValues[expr] = result
+	}
+	// Replacer dynamic values if any in raw request and parse  it
+	return []byte(replacer.Replace(final, dynamicValues)), nil
+}

v2/pkg/protocols/common/generators/env.go

@@ -0,0 +1,28 @@
+package generators
+
+import (
+	"os"
+
+	"github.com/projectdiscovery/stringsutil"
+)
+
+var envVars map[string]interface{}
+
+func parseEnvVars() map[string]interface{} {
+	sliceEnvVars := os.Environ()
+	parsedEnvVars := make(map[string]interface{}, len(sliceEnvVars))
+	for _, envVar := range sliceEnvVars {
+		key, val := stringsutil.Before(envVar, "="), stringsutil.After(envVar, "=")
+		parsedEnvVars[key] = val
+	}
+	return parsedEnvVars
+}
+
+// EnvVars returns a map with all environment variables into a map
+func EnvVars() map[string]interface{} {
+	if envVars == nil {
+		envVars = parseEnvVars()
+	}
+
+	return envVars
+}

v2/pkg/protocols/common/generators/generators.go

@@ -2,10 +2,6 @@
 
 package generators
 
-import (
-	"errors"
-)
-
 // Generator is the generator struct for generating payloads
 type Generator struct {
 	Type     Type
@@ -16,7 +12,7 @@ type Generator struct {
 type Type int
 
 const (
-	// Sniper replaces each variables with values at a time.
+	// Sniper replaces each variable with values at a time.
 	Sniper Type = iota + 1
 	// PitchFork replaces variables with positional value from multiple wordlists
 	PitchFork
@@ -24,7 +20,7 @@ const (
 	ClusterBomb
 )
 
-// StringToType is an table for conversion of attack type from string.
+// StringToType is a table for conversion of attack type from string.
 var StringToType = map[string]Type{
 	"sniper":      Sniper,
 	"pitchfork":   PitchFork,
@@ -45,16 +41,6 @@ func New(payloads map[string]interface{}, payloadType Type, templatePath string)
 	generator.Type = payloadType
 	generator.payloads = compiled
 
-	// Validate the payload types
-	if payloadType == PitchFork {
-		var totalLength int
-		for v := range compiled {
-			if totalLength != 0 && totalLength != len(v) {
-				return nil, errors.New("pitchfork payloads must be of equal number")
-			}
-			totalLength = len(v)
-		}
-	}
 	return generator, nil
 }
 
@@ -107,6 +93,11 @@ func (i *Iterator) Total() int {
 		}
 	case PitchFork:
 		count = len(i.payloads[0].values)
+		for _, p := range i.payloads {
+			if count > len(p.values) {
+				count = len(p.values)
+			}
+		}
 	case ClusterBomb:
 		count = 1
 		for _, p := range i.payloads {

v2/pkg/protocols/common/generators/generators_test.go

@@ -27,7 +27,7 @@ func TestSniperGenerator(t *testing.T) {
 
 func TestPitchforkGenerator(t *testing.T) {
 	usernames := []string{"admin", "token"}
-	passwords := []string{"admin", "password"}
+	passwords := []string{"password1", "password2", "password3"}
 
 	generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchFork, "")
 	require.Nil(t, err, "could not create generator")
@@ -43,7 +43,7 @@ func TestPitchforkGenerator(t *testing.T) {
 		require.Contains(t, usernames, value["username"], "Could not get correct pitchfork username")
 		require.Contains(t, passwords, value["password"], "Could not get correct pitchfork password")
 	}
-	require.Equal(t, len(passwords), count, "could not get correct pitchfork counts")
+	require.Equal(t, len(usernames), count, "could not get correct pitchfork counts")
 }
 
 func TestClusterbombGenerator(t *testing.T) {

v2/pkg/protocols/common/generators/maps.go

@@ -1,6 +1,8 @@
 package generators
 
-import "strings"
+import (
+	"strings"
+)
 
 // MergeMaps merges two maps into a new map
 func MergeMaps(m1, m2 map[string]interface{}) map[string]interface{} {
@@ -14,7 +16,7 @@ func MergeMaps(m1, m2 map[string]interface{}) map[string]interface{} {
 	return m
 }
 
-// ExpandMapValues converts values from flat string to strings slice
+// ExpandMapValues converts values from flat string to string slice
 func ExpandMapValues(m map[string]string) map[string][]string {
 	m1 := make(map[string][]string, len(m))
 	for k, v := range m {

v2/pkg/protocols/common/generators/slice.go

@@ -0,0 +1,16 @@
+package generators
+
+import "github.com/projectdiscovery/stringsutil"
+
+// SliceToMap converts a slice of strings to map of string splitting each item at sep as "key sep value"
+func SliceToMap(s []string, sep string) map[string]interface{} {
+	m := make(map[string]interface{})
+	for _, sliceItem := range s {
+		key := stringsutil.Before(sliceItem, sep)
+		value := stringsutil.After(sliceItem, sep)
+		if key != "" {
+			m[key] = value
+		}
+	}
+	return m
+}

v2/pkg/protocols/common/generators/validate.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
@@ -26,10 +26,10 @@ func (g *Generator) validate(payloads map[string]interface{}, templatePath strin
 			}
 
 			changed := false
-			pathTokens := strings.Split(templatePath, "/")
+			pathTokens := strings.Split(templatePath, string(os.PathSeparator))
 
 			for i := range pathTokens {
-				tpath := path.Join(strings.Join(pathTokens[:i], "/"), pt)
+				tpath := filepath.Join(filepath.Join(pathTokens[:i]...), pt)
 				if fileExists(tpath) {
 					payloads[name] = tpath
 					changed = true