Merge pull request #1038 from projectdiscovery/dev

v2.5.2 Bugfix release

.github/workflows/build-test.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.15
+          go-version: 1.17
 
       - name: Check out code
         uses: actions/checkout@v2

.github/workflows/functional-test.yml

@@ -0,0 +1,25 @@
+name: 🧪 Functional Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+
+jobs:  
+  functional:
+    name: Functional Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Functional Tests
+        run: |
+          chmod +x run.sh
+          bash run.sh
+        working-directory: v2/cmd/functional-test

.github/workflows/publish-docs.yaml

@@ -0,0 +1,51 @@
+name: ⏰ Publish Docs
+
+on:
+  push:
+    branches:
+      - dev
+  workflow_dispatch:
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@master
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+ 
+      - name: "Set up Go"
+        uses: actions/setup-go@v2
+        with: 
+          go-version: 1.17
+
+      - name: Generate YAML Syntax Documentation
+        id: generate-docs
+        run: |
+          if ! which dstdocgen > /dev/null; then
+            echo -e "Command dstdocgen not found! Installing\c"
+            go install github.com/projectdiscovery/yamldoc-go/cmd/docgen/dstdocgen@main
+          fi
+          go generate pkg/templates/templates.go
+          go build -o "cmd/docgen/docgen" cmd/docgen/docgen.go 
+          ./cmd/docgen/docgen ../SYNTAX-REFERENCE.md ../nuclei-jsonschema.json
+          echo "::set-output name=changes::$(git status -s | wc -l)"
+        working-directory: v2
+
+      - name: Commit files
+        if: steps.generate-docs.outputs.changes > 0
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          git add SYNTAX-REFERENCE.md nuclei-jsonschema.json
+          git commit -m "Auto Generate Syntax Docs + JSONSchema [$(date)] :robot:" -a
+
+      - name: Push changes
+        if: steps.generate-docs.outputs.changes > 0
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch: ${{ github.ref }}

.github/workflows/release-binary.yml

@@ -18,7 +18,7 @@ jobs:
         name: "Set up Go"
         uses: actions/setup-go@v2
         with: 
-          go-version: 1.16
+          go-version: 1.17
       - 
         env: 
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.gitignore

@@ -1,10 +1,13 @@
-cmd/nuclei/nuclei*
-v2/cmd/nuclei/nuclei
 .idea
-integration_tests/integration-test
 integration_tests/nuclei
-v2/cmd/integration-test/integration-test
-bin
+integration_tests/integration-test
+v2/cmd/nuclei/main
+v2/cmd/nuclei/nuclei
+v2/cmd/integration-test/nuclei
+v2/cmd/functional-test/nuclei_dev
+v2/cmd/functional-test/nuclei_main
+v2/cmd/functional-test/functional-test
+v2/cmd/docgen/docgen
 v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class
 v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class
-v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser
+v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser

.golangci.yml

@@ -1,108 +0,0 @@
-linters-settings:
-  dupl:
-    threshold: 100
-  exhaustive:
-    default-signifies-exhaustive: false
-  # funlen:
-  #   lines: 100
-  #   statements: 50
-  #goconst:
-  #  min-len: 2
-  #  min-occurrences: 2
-  gocritic:
-    enabled-tags:
-      - diagnostic
-      - experimental
-      - opinionated
-      - performance
-      - style
-    disabled-checks:
-      - dupImport # https://github.com/go-critic/go-critic/issues/845
-      - ifElseChain
-  # gocyclo:
-  #   min-complexity: 15
-  goimports:
-    local-prefixes: github.com/golangci/golangci-lint
-  golint:
-    min-confidence: 0
-  gomnd:
-    settings:
-      mnd:
-        # don't include the "operation" and "assign"
-        checks: argument,case,condition,return
-  govet:
-    check-shadowing: true
-    settings:
-      printf:
-        funcs:
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
-  # lll:
-  #   line-length: 140
-  maligned:
-    suggest-new: true
-  misspell:
-    locale: US
-  nolintlint:
-    allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
-    allow-unused: false # report any unused nolint directives
-    require-explanation: false # don't require an explanation for nolint directives
-    require-specific: false # don't require nolint directives to be specific about which linter is being skipped
-
-linters:
-  # please, do not use `enable-all`: it's deprecated and will be removed soon.
-  # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
-  disable-all: true
-  enable:
-    #- bodyclose
-    - deadcode
-    - dogsled
-    - errcheck
-    - exhaustive
-    - gochecknoinits
-    #- goconst
-    - gocritic
-    - gofmt
-    - goimports
-    #- gomnd
-    - goprintffuncname
-    - gosimple
-    - govet
-    - ineffassign
-    - interfacer
-    - maligned
-    - misspell
-    - nakedret
-    - noctx
-    - nolintlint
-    - rowserrcheck
-    - scopelint
-    - staticcheck
-    - structcheck
-    - stylecheck
-    - typecheck
-    - unconvert
-    - unparam
-    - unused
-    - varcheck
-    - whitespace
-    - revive
-
-  # don't enable:
-  # - depguard
-  # - asciicheck
-  # - funlen
-  # - gochecknoglobals
-  # - gocognit
-  # - gocyclo
-  # - godot
-  # - godox
-  # - goerr113
-  # - gosec
-  # - lll
-  # - nestif
-  # - prealloc
-  # - testpackage
-  # - wsl

.run/IntegrationTests.run.xml

@@ -0,0 +1,19 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="IntegrationTests" type="ShConfigurationType">
+    <option name="SCRIPT_TEXT" value="" />
+    <option name="INDEPENDENT_SCRIPT_PATH" value="true" />
+    <option name="SCRIPT_PATH" value="$PROJECT_DIR$/integration_tests/run.sh" />
+    <option name="SCRIPT_OPTIONS" value="" />
+    <option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
+    <option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$/integration_tests/" />
+    <option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
+    <option name="INTERPRETER_PATH" value="/bin/zsh" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="EXECUTE_IN_TERMINAL" value="true" />
+    <option name="EXECUTE_SCRIPT_FILE" value="true" />
+    <envs>
+      <env name="DEBUG" value="true" />
+    </envs>
+    <method v="2" />
+  </configuration>
+</component>

.run/RegressionTests.run.xml

@@ -0,0 +1,19 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="RegressionTests" type="ShConfigurationType">
+    <option name="SCRIPT_TEXT" value="" />
+    <option name="INDEPENDENT_SCRIPT_PATH" value="true" />
+    <option name="SCRIPT_PATH" value="$PROJECT_DIR$/v2/cmd/functional-test/run.sh" />
+    <option name="SCRIPT_OPTIONS" value="" />
+    <option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
+    <option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$/v2/cmd/functional-test/" />
+    <option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
+    <option name="INTERPRETER_PATH" value="/bin/zsh" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="EXECUTE_IN_TERMINAL" value="true" />
+    <option name="EXECUTE_SCRIPT_FILE" value="true" />
+    <envs>
+      <env name="DEBUG" value="true" />
+    </envs>
+    <method v="2" />
+  </configuration>
+</component>

.run/UnitTests.run.xml

@@ -0,0 +1,12 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="UnitTests" type="GoTestRunConfiguration" factoryName="Go Test">
+    <module name="nuclei" />
+    <working_directory value="$PROJECT_DIR$/v2" />
+    <go_parameters value="-i" />
+    <kind value="DIRECTORY" />
+    <directory value="$PROJECT_DIR$/v2/" />
+    <filePath value="$PROJECT_DIR$" />
+    <framework value="gotest" />
+    <method v="2" />
+  </configuration>
+</component>

Dockerfile

@@ -1,7 +1,7 @@
-FROM golang:1.16.6-alpine as build-env
+FROM golang:1.17.1-alpine as build-env
 RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
 
-FROM alpine:latest
+FROM alpine:3.14
 RUN apk add --no-cache bind-tools ca-certificates
 COPY --from=build-env /go/bin/nuclei /usr/local/bin/nuclei
 ENTRYPOINT ["nuclei"]

README.md

@@ -30,7 +30,7 @@
 
 Nuclei is used to send requests across targets based on a template leading to zero false positives and providing fast scanning on large number of hosts. Nuclei offers scanning for a variety of protocols including TCP, DNS, HTTP, File, etc. With powerful and flexible templating, all kinds of security checks can be modelled with Nuclei.
 
-We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-templates) that houses various type of vulnerability templates contributed by **more than 100** security researchers and engineers. It is preloaded with ready to use templates using `-update-templates` flag.
+We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-templates) that houses various type of vulnerability templates contributed by **more than 200** security researchers and engineers.
 
 
 
@@ -56,9 +56,11 @@ GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
 
 ### Nuclei Templates
 
-Nuclei has had built-in support for automatic update/download templates since version [v2.4.0](https://github.com/projectdiscovery/nuclei/releases/tag/v2.4.0). [**Nuclei-Templates**](https://github.com/projectdiscovery/nuclei-templates) project provides a community-contributed list of ready-to-use templates that is constantly updated.
+Nuclei has had built-in support for automatic template download/update as default since version [v2.5.2](https://github.com/projectdiscovery/nuclei/releases/tag/v2.5.2). [**Nuclei-Templates**](https://github.com/projectdiscovery/nuclei-templates) project provides a community-contributed list of ready-to-use templates that is constantly updated.
 
-You may still use the `update-templates` flag to update the nuclei templates at any time; automatic updates happen every 24 hours. You can write your own checks for your individual workflow and needs following Nuclei's [templating guide](https://nuclei.projectdiscovery.io/templating-guide/).
+You may still use the `update-templates` flag to update the nuclei templates at any time; You can write your own checks for your individual workflow and needs following Nuclei's [templating guide](https://nuclei.projectdiscovery.io/templating-guide/).
+
+The YAML DSL reference syntax is available [here](SYNTAX-REFERENCE.md).
 
 </td>
 </tr>
@@ -78,12 +80,12 @@ Nuclei is a fast, template based vulnerability scanner focusing
 on extensive configurability, massive extensibility and ease of use.
 
 Usage:
-  ./nuclei [flags]
+  nuclei [flags]
 
 Flags:
 TARGET:
-   -u, -target string  target URL/host to scan
-   -l, -list string    path to file containing a list of target URLs/hosts to scan (one per line)
+   -u, -target string[]  target URLs/hosts to scan
+   -l, -list string      path to file containing a list of target URLs/hosts to scan (one per line)
 
 TEMPLATES:
    -tl                      list all available templates
@@ -98,7 +100,7 @@ FILTERING:
    -etags, -exclude-tags string[]         exclude templates with the provided tags
    -include-templates string[]            templates to be executed even if they are excluded either by default or configuration
    -exclude-templates, -exclude string[]  template or template directory paths to exclude
-   -severity, -impact string[]            execute templates that match the provided severities only
+   -severity, -impact value[]             Templates to run based on severity. Possible values: info, low, medium, high, critical
    -author string[]                       execute templates that are (co-)created by the specified authors
 
 OUTPUT:
@@ -109,7 +111,8 @@ OUTPUT:
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -json                         write output in JSONL(ines) format
    -irr, -include-rr             include request/response pairs in the JSONL output (for findings only)
-   -nm, -no-meta                 don't display match metadata
+   -nm, -no-meta                 don't display match metadata in CLI output
+   -nts, -no-timestamp           don't display timestamp metadata in CLI output
    -rdb, -report-db string       local nuclei reporting database (always use this to persist report data)
    -me, -markdown-export string  directory to export results in markdown format
    -se, -sarif-export string     file to export results in SARIF format
@@ -118,28 +121,33 @@ CONFIGURATIONS:
    -config string              path to the nuclei configuration file
    -rc, -report-config string  nuclei reporting module configuration file
    -H, -header string[]        custom headers in header:value format
+   -V, -var value              custom vars in var=value format
    -r, -resolvers string       file containing resolver list for nuclei
    -system-resolvers           use system DNS resolving as error fallback
    -passive                    enable passive HTTP response processing mode
+   -env-vars                   enable environment variables support
 
 INTERACTSH:
-   -no-interactsh                     do not use interactsh server for blind interaction polling
-   -interactsh-url string             self-hosted Interactsh Server URL (default "https://interact.sh")
+   -no-interactsh                     disable interactsh server for OOB testing
+   -interactsh-url string             interactsh server url for self-hosted instance (default "https://interact.sh")
+   -interactsh-token string           authentication token for self-hosted interactsh server
    -interactions-cache-size int       number of requests to keep in the interactions cache (default 5000)
    -interactions-eviction int         number of seconds to wait before evicting requests from cache (default 60)
    -interactions-poll-duration int    number of seconds to wait before each interaction poll request (default 5)
    -interactions-cooldown-period int  extra time for interaction polling before exiting (default 5)
 
 RATE-LIMIT:
-   -rl, -rate-limit int  maximum number of requests to send per second (default 150)
-   -bs, -bulk-size int   maximum number of hosts to be analyzed in parallel per template (default 25)
-   -c, -concurrency int  maximum number of templates to be executed in parallel (default 10)
+   -rl, -rate-limit int          maximum number of requests to send per second (default 150)
+   -rlm, -rate-limit-minute int  maximum number of requests to send per minute
+   -bs, -bulk-size int           maximum number of hosts to be analyzed in parallel per template (default 25)
+   -c, -concurrency int          maximum number of templates to be executed in parallel (default 10)
 
 OPTIMIZATIONS:
    -timeout int               time to wait in seconds before timeout (default 5)
    -retries int               number of times to retry a failed request (default 1)
+   -max-host-error int        max errors for a host before skipping from scan (default 30)
    -project                   use a project folder to avoid sending same request multiple times
-   -project-path string       set a specific project path (default "/var/folders/ml/m31ysb5x73l1s3kjlyn5g4180000gn/T/")
+   -project-path string       set a specific project path (default "$TMPDIR/")
    -spm, -stop-at-first-path  stop processing HTTP requests after the first match (may break template/workflow logic)
 
 HEADLESS:
@@ -160,6 +168,7 @@ DEBUG:
 UPDATE:
    -update                        update nuclei to the latest released version
    -ut, -update-templates         update the community templates to latest released version
+   -nut, -no-update-templates     do not check for nuclei-templates updates
    -ud, -update-directory string  overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates")
 
 STATISTICS:
@@ -211,7 +220,7 @@ Nuclei offers great number of features that are helpful for security engineers t
 <tr>
 <td>  
 
-**For bugbounty hunters:**
+**For Bug Bounty hunters:**
 
 Nuclei allows you to customise your testing approach with your own suite of checks and easily run across your bug bounty programs. Moreover, Nuclei can be easily integrated into any continuous scanning workflow.
 
@@ -229,9 +238,9 @@ Please check our other open-source projects that might fit into your bug bounty
 <tr>
 <td>
   
-**For pentesters:**
+**For Penetration Testers:**
 
-Nuclei immensely improve how you approach security assessment by augmenting the manual repetitve processes. Consultancies are already converting their manual assessment steps with Nuclei, it allows them to run set of their custom assessment approach across thousands of hosts in an automated manner. 
+Nuclei immensely improve how you approach security assessment by augmenting the manual, repetitive processes. Consultancies are already converting their manual assessment steps with Nuclei, it allows them to run set of their custom assessment approach across thousands of hosts in an automated manner. 
 
 Pen-testers get the full power of our public templates and customization capabilities to speed-up their assessment process, and specifically with the regression cycle where you can easily verify the fix.
 
@@ -246,7 +255,7 @@ Pen-testers get the full power of our public templates and customization capabil
 
 # For Developers and Organisations
 
-Nuclei is built with simplicity in mind, with the community backed templates by hundreds of security researchers, it allows you to stay updated with latest security threats using continuous Nuclei scanning on the hosts. It is designed to be easily integrated into regression tests cycle, to verify the fixes and  eliminate vulnerabilities from occuring in future.
+Nuclei is built with simplicity in mind, with the community backed templates by hundreds of security researchers, it allows you to stay updated with the latest security threats using continuous Nuclei scanning on the hosts. It is designed to be easily integrated into regression tests cycle, to verify the fixes and eliminate vulnerabilities from occurring in the future.
 
 - **CI/CD:** Engineers are already utilising Nuclei within their CI/CD pipeline, it allows them to constantly monitor their staging and production environments with customised templates.
 - **Continuous Regression Cycle:** With Nuclei, you can create your custom template on every new identified vulnerability and put into Nuclei engine to eliminate in the continuous regression cycle.

README_CN.md

@@ -97,7 +97,7 @@ nuclei -h
 |burp-collaborator-biid|使用burp-collaborator插件|nuclei -burp-collaborator-biid XXXX|
 |c|并行的最大模板数量(默认10)|nuclei -c 10|
 |l|对URL列表进行测试|nuclei -l urls.txt|
-|target|对目标进行测试|nuclei -target hxxps://example.com|
+|target|对目标进行测试|nuclei -target hxxps://example.com -target hxxps://example2.com|
 |t|要检测的模板种类|nuclei -t git-core.yaml -t cves/|
 |no-color|输出不显示颜色|nuclei -no-color|
 |no-meta|不显示匹配的元数据|nuclei -no-meta|
@@ -250,4 +250,4 @@ nano ~/nuclei-templates/.nuclei-ignore
 
 --------
 
-Nuclei是由[projectdiscovery](https://projectdiscovery.io)团队用🖤制作的，当然社区也贡献了很多，通过 **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)**文件以获取更多详细信息。
+Nuclei是由[projectdiscovery](https://projectdiscovery.io)团队用🖤制作的，当然社区也贡献了很多，通过 **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)**文件以获取更多详细信息。

SECURITY.md

@@ -2,4 +2,4 @@
 
 ## Reporting a Vulnerability
 
-DO NOT CREATE AN ISSUE to report a security problem. Instead, please send an email to security@projectdiscovery.io and we will acknowledge it within 3 working days.
+DO NOT CREATE AN ISSUE to report a security problem. Instead, please send an email to security@projectdiscovery.io, and we will acknowledge it within 3 working days.

integration_tests/run.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+rm integration-test nuclei 2>/dev/null
 cd ../v2/cmd/nuclei
 go build
 mv nuclei ../../../integration_tests/nuclei 

integration_tests/test-issue-tracker-config1.yaml

@@ -0,0 +1,68 @@
+allow-list:
+  severity: high, critical
+deny-list:
+  severity: low
+
+# github contains configuration options for github issue tracker
+github:
+  # base-url is the optional self-hosted github application url
+  base-url: https://localhost:8443/github
+  # username is the username of the github user
+  username: test-username
+  # owner is the owner name of the repository for issues.
+  owner: test-owner
+  # token is the token for github account.
+  token: test-token
+  # project-name is the name of the repository.
+  project-name: test-project
+  # issue-label is the label of the created issue type
+  issue-label: bug
+
+# gitlab contains configuration options for gitlab issue tracker
+gitlab:
+  # base-url is the optional self-hosted gitlab application url
+  base-url: https://localhost:8443/gitlab
+  # username is the username of the gitlab user
+  username: test-username
+  # token is the token for gitlab account.
+  token: test-token
+  # project-id is the ID of the repository.
+  project-id: 1234
+  # issue-label is the label of the created issue type
+  issue-label: bug
+
+# jira contains configuration options for jira issue tracker
+jira:
+  # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
+  cloud: true
+  # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created
+  update-existing: false
+  # URL is the jira application url
+  url: https://localhost/jira
+  # account-id is the account-id of the jira user or username in case of on-prem Jira
+  account-id: test-account-id
+  # email is the email of the user for jira instance
+  email: [email protected]
+  # token is the token for jira instance or password in case of on-prem Jira
+  token: test-token
+  # project-name is the name of the project.
+  project-name: test-project-name
+  # issue-type is the name of the created issue type
+  issue-type: bug
+
+# elasticsearch contains configuration options for elasticsearch exporter
+elasticsearch:
+  # IP for elasticsearch instance
+  ip: 127.0.0.1
+  # Port is the port of elasticsearch instance
+  port: 9200
+  # IndexName is the name of the elasticsearch index
+  index-name: nuclei
+  # SSL enables ssl for elasticsearch connection
+  ssl: false
+  # SSLVerification disables SSL verification for elasticsearch
+  ssl-verification: false
+  # Username for the elasticsearch instance
+  username: test
+  # Password is the password for elasticsearch instance
+  password: test

integration_tests/test-issue-tracker-config2.yaml

@@ -0,0 +1,70 @@
+allow-list:
+  severity:
+    - high
+    - critical
+deny-list:
+  severity: low
+
+# github contains configuration options for github issue tracker
+github:
+  # base-url is the optional self-hosted github application url
+  base-url: https://localhost:8443/github
+  # username is the username of the github user
+  username: test-username
+  # owner is the owner name of the repository for issues.
+  owner: test-owner
+  # token is the token for github account.
+  token: test-token
+  # project-name is the name of the repository.
+  project-name: test-project
+  # issue-label is the label of the created issue type
+  issue-label: bug
+
+# gitlab contains configuration options for gitlab issue tracker
+gitlab:
+  # base-url is the optional self-hosted gitlab application url
+  base-url: https://localhost:8443/gitlab
+  # username is the username of the gitlab user
+  username: test-username
+  # token is the token for gitlab account.
+  token: test-token
+  # project-id is the ID of the repository.
+  project-id: 1234
+  # issue-label is the label of the created issue type
+  issue-label: bug
+
+# jira contains configuration options for jira issue tracker
+jira:
+  # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
+  cloud: true
+  # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created
+  update-existing: false
+  # URL is the jira application url
+  url: https://localhost/jira
+  # account-id is the account-id of the jira user or username in case of on-prem Jira
+  account-id: test-account-id
+  # email is the email of the user for jira instance
+  email: [email protected]
+  # token is the token for jira instance or password in case of on-prem Jira
+  token: test-token
+  # project-name is the name of the project.
+  project-name: test-project-name
+  # issue-type is the name of the created issue type
+  issue-type: bug
+
+# elasticsearch contains configuration options for elasticsearch exporter
+elasticsearch:
+  # IP for elasticsearch instance
+  ip: 127.0.0.1
+  # Port is the port of elasticsearch instance
+  port: 9200
+  # IndexName is the name of the elasticsearch index
+  index-name: nuclei
+  # SSL enables ssl for elasticsearch connection
+  ssl: false
+  # SSLVerification disables SSL verification for elasticsearch
+  ssl-verification: false
+  # Username for the elasticsearch instance
+  username: test
+  # Password is the password for elasticsearch instance
+  password: test

nuclei-jsonschema.json

@@ -0,0 +1,959 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$ref": "#/definitions/templates.Template",
+  "definitions": {
+    "model.Classification": {
+      "properties": {
+        "cve-id": {
+          "$ref": "#/definitions/stringslice.StringSlice",
+          "title": "cve ids for the template",
+          "description": "CVE IDs for the template"
+        },
+        "cwe-id": {
+          "$ref": "#/definitions/stringslice.StringSlice",
+          "title": "cwe ids for the template",
+          "description": "CWE IDs for the template"
+        },
+        "cvss-metrics": {
+          "type": "string",
+          "title": "cvss metrics for the template",
+          "description": "CVSS Metrics for the template",
+          "examples": [
+            "3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"
+          ]
+        },
+        "cvss-score": {
+          "type": "number",
+          "title": "cvss score for the template",
+          "description": "CVSS Score for the template"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "model.Info": {
+      "properties": {
+        "name": {
+          "type": "string",
+          "title": "name of the template",
+          "description": "Name is a short summary of what the template does",
+          "examples": [
+            "Nagios Default Credentials Check"
+          ]
+        },
+        "author": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/stringslice.StringSlice",
+          "title": "author of the template",
+          "description": "Author is the author of the template"
+        },
+        "tags": {
+          "$ref": "#/definitions/stringslice.StringSlice",
+          "title": "tags of the template",
+          "description": "Any tags for the template"
+        },
+        "description": {
+          "type": "string",
+          "title": "description of the template",
+          "description": "In-depth explanation on what the template does",
+          "examples": [
+            "Bower is a package manager which stores package information in the bower.json file"
+          ]
+        },
+        "reference": {
+          "$ref": "#/definitions/stringslice.StringSlice",
+          "title": "references for the template",
+          "description": "Links relevant to the template"
+        },
+        "severity": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/severity.Holder"
+        },
+        "metadata": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object",
+          "title": "additional metadata for the template",
+          "description": "Additional metadata fields for the template"
+        },
+        "classification": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/model.Classification",
+          "title": "classification info for the template",
+          "description": "Classification information for the template"
+        },
+        "remediation": {
+          "type": "string",
+          "title": "remediation steps for the template",
+          "description": "In-depth explanation on how to fix the issues found by the template",
+          "examples": [
+            "Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties"
+          ]
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "severity.Holder": {
+      "enum": [
+        "info",
+        "low",
+        "medium",
+        "high",
+        "critical"
+      ],
+      "type": "string",
+      "title": "severity of the template",
+      "description": "Seriousness of the implications of the template"
+    },
+    "stringslice.StringSlice": {
+      "oneOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "array"
+        }
+      ]
+    },
+    "extractors.Extractor": {
+      "required": [
+        "type"
+      ],
+      "properties": {
+        "name": {
+          "type": "string",
+          "title": "name of the extractor",
+          "description": "Name of the extractor"
+        },
+        "type": {
+          "enum": [
+            "regex",
+            "kval",
+            "json",
+            "xpath"
+          ],
+          "type": "string",
+          "title": "type of the extractor",
+          "description": "Type of the extractor"
+        },
+        "regex": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "regex to extract from part",
+          "description": "Regex to extract from part"
+        },
+        "group": {
+          "type": "integer",
+          "title": "group to extract from regex",
+          "description": "Group to extract from regex"
+        },
+        "kval": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "kval pairs to extract from response",
+          "description": "Kval pairs to extract from response"
+        },
+        "json": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "json jq expressions to extract data",
+          "description": "JSON JQ expressions to evaluate from response part"
+        },
+        "xpath": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "html xpath expressions to extract data",
+          "description": "XPath allows using xpath expressions to extract items from html response"
+        },
+        "attribute": {
+          "type": "string",
+          "title": "optional attribute to extract from xpath",
+          "description": "Optional attribute to extract from response XPath"
+        },
+        "part": {
+          "type": "string",
+          "title": "part of response to extract data from",
+          "description": "Part of the request response to extract data from"
+        },
+        "internal": {
+          "type": "boolean",
+          "title": "mark extracted value for internal variable use",
+          "description": "Internal when set to true will allow using the value extracted in the next request for some protocols"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "matchers.Matcher": {
+      "required": [
+        "type"
+      ],
+      "properties": {
+        "type": {
+          "enum": [
+            "status",
+            "size",
+            "word",
+            "regex",
+            "binary",
+            "dsl"
+          ],
+          "type": "string",
+          "title": "type of matcher",
+          "description": "Type of the matcher"
+        },
+        "condition": {
+          "enum": [
+            "and",
+            "or"
+          ],
+          "type": "string",
+          "title": "condition between matcher variables",
+          "description": "Condition between the matcher variables"
+        },
+        "part": {
+          "type": "string",
+          "title": "part of response to match",
+          "description": "Part of response to match data from"
+        },
+        "negative": {
+          "type": "boolean",
+          "title": "negative specifies if match reversed",
+          "description": "Negative specifies if the match should be reversed. It will only match if the condition is not true"
+        },
+        "name": {
+          "type": "string",
+          "title": "name of the matcher",
+          "description": "Name of the matcher"
+        },
+        "status": {
+          "items": {
+            "type": "integer"
+          },
+          "type": "array",
+          "title": "status to match",
+          "description": "Status to match for the response"
+        },
+        "size": {
+          "items": {
+            "type": "integer"
+          },
+          "type": "array",
+          "title": "acceptable size for response",
+          "description": "Size is the acceptable size for the response"
+        },
+        "words": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "words to match in response",
+          "description": " Words contains word patterns required to be present in the response part"
+        },
+        "regex": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "regex to match in response",
+          "description": "Regex contains regex patterns required to be present in the response part"
+        },
+        "binary": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "binary patterns to match in response",
+          "description": "Binary are the binary patterns required to be present in the response part"
+        },
+        "dsl": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "dsl expressions to match in response",
+          "description": "DSL are the dsl expressions that will be evaluated as part of nuclei matching rules"
+        },
+        "encoding": {
+          "enum": [
+            "hex"
+          ],
+          "type": "string",
+          "title": "encoding for word field",
+          "description": "Optional encoding for the word fields"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "dns.Request": {
+      "properties": {
+        "matchers": {
+          "items": {
+            "$ref": "#/definitions/matchers.Matcher"
+          },
+          "type": "array",
+          "title": "matchers to run on response",
+          "description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
+        },
+        "extractors": {
+          "items": {
+            "$ref": "#/definitions/extractors.Extractor"
+          },
+          "type": "array",
+          "title": "extractors to run on response",
+          "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
+        },
+        "matchers-condition": {
+          "enum": [
+            "and",
+            "or"
+          ],
+          "type": "string",
+          "title": "condition between the matchers",
+          "description": "Conditions between the matchers"
+        },
+        "id": {
+          "type": "string",
+          "title": "id of the dns request",
+          "description": "ID is the optional ID of the DNS Request"
+        },
+        "name": {
+          "type": "string",
+          "title": "hostname to make dns request for",
+          "description": "Name is the Hostname to make DNS request for"
+        },
+        "type": {
+          "enum": [
+            "A",
+            "NS",
+            "DS",
+            "CNAME",
+            "SOA",
+            "PTR",
+            "MX",
+            "TXT",
+            "AAAA"
+          ],
+          "type": "string",
+          "title": "type of dns request to make",
+          "description": "Type is the type of DNS request to make"
+        },
+        "class": {
+          "enum": [
+            "inet",
+            "csnet",
+            "chaos",
+            "hesiod",
+            "none",
+            "any"
+          ],
+          "type": "string",
+          "title": "class of DNS request",
+          "description": "Class is the class of the DNS request"
+        },
+        "retries": {
+          "type": "integer",
+          "title": "retries for dns request",
+          "description": "Retries is the number of retries for the DNS request"
+        },
+        "recursion": {
+          "type": "boolean",
+          "title": "recurse all servers",
+          "description": "Recursion determines if resolver should recurse all records to get fresh results"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "file.Request": {
+      "properties": {
+        "matchers": {
+          "items": {
+            "$ref": "#/definitions/matchers.Matcher"
+          },
+          "type": "array",
+          "title": "matchers to run on response",
+          "description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
+        },
+        "extractors": {
+          "items": {
+            "$ref": "#/definitions/extractors.Extractor"
+          },
+          "type": "array",
+          "title": "extractors to run on response",
+          "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
+        },
+        "matchers-condition": {
+          "enum": [
+            "and",
+            "or"
+          ],
+          "type": "string",
+          "title": "condition between the matchers",
+          "description": "Conditions between the matchers"
+        },
+        "extensions": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "extensions to match",
+          "description": "List of extensions to perform matching on"
+        },
+        "denylist": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "extensions to deny match",
+          "description": "List of file extensions to deny during matching"
+        },
+        "id": {
+          "type": "string",
+          "title": "id of the request",
+          "description": "ID is the optional ID for the request"
+        },
+        "max-size": {
+          "type": "integer",
+          "title": "max size data to run request on",
+          "description": "Maximum size of the file to run request on"
+        },
+        "no-recursive": {
+          "type": "boolean",
+          "title": "do not perform recursion",
+          "description": "Specifies whether to not do recursive checks if folders are provided"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "headless.Request": {
+      "properties": {
+        "id": {
+          "type": "string",
+          "title": "id of the request",
+          "description": "Optional ID of the headless request"
+        },
+        "steps": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/engine.Action"
+          },
+          "type": "array",
+          "title": "list of actions for headless request",
+          "description": "List of actions to run for headless request"
+        },
+        "matchers": {
+          "items": {
+            "$ref": "#/definitions/matchers.Matcher"
+          },
+          "type": "array",
+          "title": "matchers to run on response",
+          "description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
+        },
+        "extractors": {
+          "items": {
+            "$ref": "#/definitions/extractors.Extractor"
+          },
+          "type": "array",
+          "title": "extractors to run on response",
+          "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
+        },
+        "matchers-condition": {
+          "enum": [
+            "and",
+            "or"
+          ],
+          "type": "string",
+          "title": "condition between the matchers",
+          "description": "Conditions between the matchers"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "engine.Action": {
+      "required": [
+        "action"
+      ],
+      "properties": {
+        "args": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object",
+          "title": "arguments for headless action",
+          "description": "Args contain arguments for the headless action"
+        },
+        "name": {
+          "type": "string",
+          "title": "name for headless action",
+          "description": "Name is the name assigned to the headless action"
+        },
+        "description": {
+          "type": "string",
+          "title": "description for headless action",
+          "description": "Description of the headless action"
+        },
+        "action": {
+          "enum": [
+            "navigate",
+            "script",
+            "click",
+            "rightclick",
+            "text",
+            "screenshot",
+            "time",
+            "select",
+            "files",
+            "waitload",
+            "getresource",
+            "extract",
+            "setmethod",
+            "addheader",
+            "setheader",
+            "deleteheader",
+            "setbody",
+            "waitevent",
+            "keyboard",
+            "debug",
+            "sleep"
+          ],
+          "type": "string",
+          "title": "action to perform",
+          "description": "Type of actions to perform"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "http.Request": {
+      "properties": {
+        "matchers": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/matchers.Matcher"
+          },
+          "type": "array",
+          "title": "matchers to run on response",
+          "description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
+        },
+        "extractors": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/extractors.Extractor"
+          },
+          "type": "array",
+          "title": "extractors to run on response",
+          "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
+        },
+        "matchers-condition": {
+          "enum": [
+            "and",
+            "or"
+          ],
+          "type": "string",
+          "title": "condition between the matchers",
+          "description": "Conditions between the matchers"
+        },
+        "path": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "path(s) for the http request",
+          "description": "Path(s) to send http requests to"
+        },
+        "raw": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "description": "HTTP Requests in Raw Format"
+        },
+        "id": {
+          "type": "string",
+          "title": "id for the http request",
+          "description": "ID for the HTTP Request"
+        },
+        "name": {
+          "type": "string",
+          "title": "name for the http request",
+          "description": "Optional name for the HTTP Request"
+        },
+        "attack": {
+          "enum": [
+            "sniper",
+            "pitchfork",
+            "clusterbomb"
+          ],
+          "type": "string",
+          "title": "attack is the payload combination",
+          "description": "Attack is the type of payload combinations to perform"
+        },
+        "method": {
+          "enum": [
+            "GET",
+            "HEAD",
+            "POST",
+            "PUT",
+            "DELETE",
+            "CONNECT",
+            "OPTIONS",
+            "TRACE",
+            "PATCH",
+            "PURGE"
+          ],
+          "type": "string",
+          "title": "method is the http request method",
+          "description": "Method is the HTTP Request Method"
+        },
+        "body": {
+          "type": "string",
+          "title": "body is the http request body",
+          "description": "Body is an optional parameter which contains HTTP Request body"
+        },
+        "payloads": {
+          "patternProperties": {
+            ".*": {
+              "additionalProperties": true
+            }
+          },
+          "type": "object",
+          "title": "payloads for the http request",
+          "description": "Payloads contains any payloads for the current request"
+        },
+        "headers": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object",
+          "title": "headers to send with the http request",
+          "description": "Headers contains HTTP Headers to send with the request"
+        },
+        "race_count": {
+          "type": "integer",
+          "title": "number of times to repeat request in race condition",
+          "description": "Number of times to send a request in Race Condition Attack"
+        },
+        "max-redirects": {
+          "type": "integer",
+          "title": "maximum number of redirects to follow",
+          "description": "Maximum number of redirects that should be followed"
+        },
+        "pipeline-concurrent-connections": {
+          "type": "integer",
+          "title": "number of pipelining connections",
+          "description": "Number of connections to create during pipelining"
+        },
+        "pipeline-requests-per-connection": {
+          "type": "integer",
+          "title": "number of requests to send per pipelining connections",
+          "description": "Number of requests to send per connection when pipelining"
+        },
+        "threads": {
+          "type": "integer",
+          "title": "threads for sending requests",
+          "description": "Threads specifies number of threads to use sending requests. This enables Connection Pooling"
+        },
+        "max-size": {
+          "type": "integer",
+          "title": "maximum http response body size",
+          "description": "Maximum size of http response body to read in bytes"
+        },
+        "cookie-reuse": {
+          "type": "boolean",
+          "title": "optional cookie reuse enable",
+          "description": "Optional setting that enables cookie reuse"
+        },
+        "redirects": {
+          "type": "boolean",
+          "title": "follow http redirects",
+          "description": "Specifies whether redirects should be followed by the HTTP Client"
+        },
+        "pipeline": {
+          "type": "boolean",
+          "title": "perform HTTP 1.1 pipelining",
+          "description": "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"
+        },
+        "unsafe": {
+          "type": "boolean",
+          "title": "use rawhttp non-strict-rfc client",
+          "description": "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests"
+        },
+        "race": {
+          "type": "boolean",
+          "title": "perform race-http request coordination attack",
+          "description": "Race determines if all the request have to be attempted at the same time (Race Condition)"
+        },
+        "req-condition": {
+          "type": "boolean",
+          "title": "preserve request history",
+          "description": "Automatically assigns numbers to requests and preserves their history"
+        },
+        "stop-at-first-match": {
+          "type": "boolean",
+          "title": "stop at first match",
+          "description": "Stop the execution after a match is found"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "network.Input": {
+      "properties": {
+        "data": {
+          "type": "string",
+          "title": "data to send as input",
+          "description": "Data is the data to send as the input"
+        },
+        "type": {
+          "enum": [
+            "hex",
+            "text"
+          ],
+          "type": "string",
+          "title": "type is the type of input data",
+          "description": "Type of input specified in data field"
+        },
+        "read": {
+          "type": "integer",
+          "title": "bytes to read from socket",
+          "description": "Number of bytes to read from socket"
+        },
+        "name": {
+          "type": "string",
+          "title": "optional name for data read",
+          "description": "Optional name of the data read to provide matching on"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "network.Request": {
+      "properties": {
+        "id": {
+          "type": "string",
+          "title": "id of the request",
+          "description": "ID of the network request"
+        },
+        "host": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "host to send requests to",
+          "description": "Host to send network requests to"
+        },
+        "attack": {
+          "enum": [
+            "sniper",
+            "pitchfork",
+            "clusterbomb"
+          ],
+          "type": "string",
+          "title": "attack is the payload combination",
+          "description": "Attack is the type of payload combinations to perform"
+        },
+        "payloads": {
+          "patternProperties": {
+            ".*": {
+              "additionalProperties": true
+            }
+          },
+          "type": "object",
+          "title": "payloads for the network request",
+          "description": "Payloads contains any payloads for the current request"
+        },
+        "inputs": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/network.Input"
+          },
+          "type": "array",
+          "title": "inputs for the network request",
+          "description": "Inputs contains any input/output for the current request"
+        },
+        "read-size": {
+          "type": "integer",
+          "title": "size of network response to read",
+          "description": "Size of response to read at the end. Default is 1024 bytes"
+        },
+        "matchers": {
+          "items": {
+            "$ref": "#/definitions/matchers.Matcher"
+          },
+          "type": "array",
+          "title": "matchers to run on response",
+          "description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
+        },
+        "extractors": {
+          "items": {
+            "$ref": "#/definitions/extractors.Extractor"
+          },
+          "type": "array",
+          "title": "extractors to run on response",
+          "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
+        },
+        "matchers-condition": {
+          "enum": [
+            "and",
+            "or"
+          ],
+          "type": "string",
+          "title": "condition between the matchers",
+          "description": "Conditions between the matchers"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "templates.Template": {
+      "required": [
+        "id",
+        "info"
+      ],
+      "properties": {
+        "id": {
+          "type": "string",
+          "title": "id of the template",
+          "description": "The Unique ID for the template",
+          "examples": [
+            "cve-2021-19520"
+          ]
+        },
+        "info": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/model.Info",
+          "title": "info for the template",
+          "description": "Info contains metadata for the template"
+        },
+        "requests": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/http.Request"
+          },
+          "type": "array",
+          "title": "http requests to make",
+          "description": "HTTP requests to make for the template"
+        },
+        "dns": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/dns.Request"
+          },
+          "type": "array",
+          "title": "dns requests to make",
+          "description": "DNS requests to make for the template"
+        },
+        "file": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/file.Request"
+          },
+          "type": "array",
+          "title": "file requests to make",
+          "description": "File requests to make for the template"
+        },
+        "network": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/network.Request"
+          },
+          "type": "array",
+          "title": "network requests to make",
+          "description": "Network requests to make for the template"
+        },
+        "headless": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/headless.Request"
+          },
+          "type": "array",
+          "title": "headless requests to make",
+          "description": "Headless requests to make for the template"
+        },
+        "workflows": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/workflows.WorkflowTemplate"
+          },
+          "type": "array",
+          "title": "list of workflows to execute",
+          "description": "List of workflows to execute for template"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "workflows.Matcher": {
+      "properties": {
+        "name": {
+          "type": "string",
+          "title": "name of item to match",
+          "description": "Name of item to match"
+        },
+        "subtemplates": {
+          "items": {
+            "$ref": "#/definitions/workflows.WorkflowTemplate"
+          },
+          "type": "array",
+          "title": "templates to run after match",
+          "description": "Templates to run after match"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "workflows.WorkflowTemplate": {
+      "properties": {
+        "template": {
+          "type": "string",
+          "title": "template/directory to execute",
+          "description": "Template or directory to execute as part of workflow"
+        },
+        "tags": {
+          "$ref": "#/definitions/stringslice.StringSlice",
+          "title": "tags to execute",
+          "description": "Tags to run template based on"
+        },
+        "matchers": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/workflows.Matcher"
+          },
+          "type": "array",
+          "title": "name based template result matchers",
+          "description": "Matchers perform name based matching to run subtemplates for a workflow"
+        },
+        "subtemplates": {
+          "items": {
+            "$ref": "#/definitions/workflows.WorkflowTemplate"
+          },
+          "type": "array",
+          "title": "subtemplate based result matchers",
+          "description": "Subtemplates are ran if the template field Template matches"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    }
+  }
+}

v2/.goreleaser.yml

@@ -20,6 +20,8 @@ builds:
       goarch: '386'
     - goos: windows
       goarch: 'arm'
+    - goos: windows
+      goarch: 'arm64'
 
   binary: '{{ .ProjectName }}'
   main: cmd/nuclei/main.go
@@ -30,4 +32,4 @@ archives:
       darwin: macOS
 
 checksum:
-  algorithm: sha256
+  algorithm: sha256

v2/Makefile

@@ -7,8 +7,16 @@ GOGET=$(GOCMD) get
     
 all: build
 build:
-		$(GOBUILD) -v -ldflags="-extldflags=-static" -o "nuclei" cmd/nuclei/main.go
-test: 
-		$(GOTEST) -v ./...
+	$(GOBUILD) -v -ldflags="-extldflags=-static" -o "nuclei" cmd/nuclei/main.go
+docs:
+	if ! which dstdocgen > /dev/null; then
+		echo -e "Command not found! Install? (y/n) \c"
+		go get -v github.com/projectdiscovery/yamldoc-go/cmd/docgen/dstdocgen
+	fi
+	$(GOCMD) generate pkg/templates/templates.go
+	$(GOBUILD) -o "cmd/docgen/docgen" cmd/docgen/docgen.go
+	./cmd/docgen/docgen docs.md nuclei-jsonschema.json
+test:
+	$(GOTEST) -v ./...
 tidy:
-		$(GOMOD) tidy
+	$(GOMOD) tidy

v2/cmd/cve-annotate/main.go

@@ -0,0 +1,162 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/Ice3man543/nvd"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+)
+
+var (
+	input       = flag.String("i", "", "Templates to annotate")
+	templateDir = flag.String("d", "", "Custom template directory for update")
+)
+
+func main() {
+	flag.Parse()
+
+	if *input == "" || *templateDir == "" {
+		log.Fatalf("invalid input, see -h\n")
+	}
+
+	if err := process(); err != nil {
+		log.Fatalf("could not process: %s\n", err)
+	}
+}
+
+func process() error {
+	tempDir, err := ioutil.TempDir("", "nuclei-nvd-%s")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tempDir)
+
+	client, err := nvd.NewClient(tempDir)
+	if err != nil {
+		return err
+	}
+	catalog := catalog.New(*templateDir)
+
+	paths, err := catalog.GetTemplatePath(*input)
+	if err != nil {
+		return err
+	}
+	for _, path := range paths {
+		data, err := ioutil.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		getCVEData(client, path, string(data))
+	}
+	return nil
+}
+
+var (
+	idRegex       = regexp.MustCompile("id: ([C|c][V|v][E|e]-[0-9]+-[0-9]+)")
+	severityRegex = regexp.MustCompile(`severity: ([a-z]+)`)
+)
+
+func getCVEData(client *nvd.Client, filePath, data string) {
+	matches := idRegex.FindAllStringSubmatch(data, 1)
+	if len(matches) == 0 {
+		return
+	}
+	cveName := matches[0][1]
+
+	severityMatches := severityRegex.FindAllStringSubmatch(data, 1)
+	if len(matches) == 0 {
+		return
+	}
+	severityValue := severityMatches[0][1]
+
+	// Skip if there's classification data already
+	if strings.Contains(data, "classification:") {
+		return
+	}
+	cveItem, err := client.FetchCVE(cveName)
+	if err != nil {
+		log.Printf("Could not fetch cve %s: %s\n", cveName, err)
+		return
+	}
+	var cweID []string
+	for _, problemData := range cveItem.CVE.Problemtype.ProblemtypeData {
+		for _, description := range problemData.Description {
+			cweID = append(cweID, description.Value)
+		}
+	}
+	cvssScore := cveItem.Impact.BaseMetricV3.CvssV3.BaseScore
+	cvssMetrics := cveItem.Impact.BaseMetricV3.CvssV3.VectorString
+
+	// Perform some hacky string replacement to place the metadata in templates
+	infoBlockIndexData := data[strings.Index(data, "info:"):]
+	requestsIndex := strings.Index(infoBlockIndexData, "requests:")
+	networkIndex := strings.Index(infoBlockIndexData, "network:")
+	if requestsIndex == -1 && networkIndex == -1 {
+		return
+	}
+	if networkIndex != -1 {
+		requestsIndex = networkIndex
+	}
+	infoBlockData := infoBlockIndexData[:requestsIndex]
+	infoBlockClean := strings.TrimRight(infoBlockData, "\n")
+
+	newInfoBlock := infoBlockClean
+	var changed bool
+
+	if newSeverity := isSeverityMatchingCvssScore(severityValue, cvssScore); newSeverity != "" {
+		changed = true
+		newInfoBlock = strings.ReplaceAll(newInfoBlock, severityMatches[0][0], "severity: "+newSeverity)
+		fmt.Printf("Adjusting severity for %s from %s=>%s (%.2f)\n", filePath, severityValue, newSeverity, cvssScore)
+	}
+	if !strings.Contains(infoBlockClean, "classification") && (cvssScore != 0 && cvssMetrics != "") {
+		changed = true
+		newInfoBlock = newInfoBlock + fmt.Sprintf("\n  classification:\n    cvss-metrics: %s\n    cvss-score: %.2f\n    cve-id: %s", cvssMetrics, cvssScore, cveName)
+		if len(cweID) > 0 && (cweID[0] != "NVD-CWE-Other" && cweID[0] != "NVD-CWE-noinfo") {
+			newInfoBlock = newInfoBlock + fmt.Sprintf("\n    cwe-id: %s", strings.Join(cweID, ","))
+		}
+	}
+	// If there is no description field, fill the description from CVE information
+	if !strings.Contains(infoBlockClean, "description:") && len(cveItem.CVE.Description.DescriptionData) > 0 {
+		changed = true
+		newInfoBlock = newInfoBlock + fmt.Sprintf("\n  description: %s", fmt.Sprintf("%q", cveItem.CVE.Description.DescriptionData[0].Value))
+	}
+	if !strings.Contains(infoBlockClean, "reference:") && len(cveItem.CVE.References.ReferenceData) > 0 {
+		changed = true
+		newInfoBlock = newInfoBlock + "\n  reference:"
+		for _, reference := range cveItem.CVE.References.ReferenceData {
+			newInfoBlock = newInfoBlock + fmt.Sprintf("\n    - %s", reference.URL)
+		}
+	}
+	newTemplate := strings.ReplaceAll(data, infoBlockClean, newInfoBlock)
+	if changed {
+		_ = ioutil.WriteFile(filePath, []byte(newTemplate), 0777)
+		fmt.Printf("Wrote updated template to %s\n", filePath)
+	}
+}
+
+func isSeverityMatchingCvssScore(severity string, score float64) string {
+	if score == 0.0 {
+		return ""
+	}
+	var expected string
+
+	if score >= 0.1 && score <= 3.9 {
+		expected = "low"
+	} else if score >= 4.0 && score <= 6.9 {
+		expected = "medium"
+	} else if score >= 7.0 && score <= 8.9 {
+		expected = "high"
+	} else if score >= 9.0 && score <= 10.0 {
+		expected = "critical"
+	}
+	if expected != "" && expected != severity {
+		return expected
+	}
+	return ""
+}

v2/cmd/docgen/docgen.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/alecthomas/jsonschema"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+)
+
+var pathRegex = regexp.MustCompile(`github.com/projectdiscovery/nuclei/v2/(?:internal|pkg)/(?:.*/)?([A-Za-z\.]+)`)
+
+func main() {
+	// Generate yaml syntax documentation
+	data, err := templates.GetTemplateDoc().Encode()
+	if err != nil {
+		log.Fatalf("Could not encode docs: %s\n", err)
+	}
+	err = ioutil.WriteFile(os.Args[1], data, 0777)
+	if err != nil {
+		log.Fatalf("Could not write docs: %s\n", err)
+	}
+
+	// Generate jsonschema
+	r := &jsonschema.Reflector{
+		PreferYAMLSchema:      true,
+		YAMLEmbeddedStructs:   true,
+		FullyQualifyTypeNames: true,
+	}
+	jsonschemaData := r.Reflect(&templates.Template{})
+
+	var buf bytes.Buffer
+	encoder := json.NewEncoder(&buf)
+	encoder.SetIndent("", "  ")
+	_ = encoder.Encode(jsonschemaData)
+
+	schema := buf.String()
+	for _, match := range pathRegex.FindAllStringSubmatch(schema, -1) {
+		schema = strings.ReplaceAll(schema, match[0], match[1])
+	}
+	err = ioutil.WriteFile(os.Args[2], []byte(schema), 0777)
+	if err != nil {
+		log.Fatalf("Could not write jsonschema: %s\n", err)
+	}
+}

v2/cmd/functional-test/main.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/pkg/errors"
+
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var (
+	success = aurora.Green("[✓]").String()
+	failed  = aurora.Red("[✘]").String()
+	errored = false
+
+	mainNucleiBinary = flag.String("main", "", "Main Branch Nuclei Binary")
+	devNucleiBinary  = flag.String("dev", "", "Dev Branch Nuclei Binary")
+	testcases        = flag.String("testcases", "", "Test cases file for nuclei functional tests")
+)
+
+func main() {
+	flag.Parse()
+
+	if err := runFunctionalTests(); err != nil {
+		log.Fatalf("Could not run functional tests: %s\n", err)
+	}
+	if errored {
+		os.Exit(1)
+	}
+}
+
+func runFunctionalTests() error {
+	file, err := os.Open(*testcases)
+	if err != nil {
+		return errors.Wrap(err, "could not open test cases")
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := strings.TrimSpace(scanner.Text())
+		if text == "" {
+			continue
+		}
+		if err := runIndividualTestCase(text); err != nil {
+			errored = true
+			fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, text, err)
+		} else {
+			fmt.Printf("%s Test \"%s\" passed!\n", success, text)
+		}
+	}
+	return nil
+}
+
+func runIndividualTestCase(testcase string) error {
+	parts := strings.Fields(testcase)
+
+	var finalArgs []string
+	if len(parts) > 1 {
+		finalArgs = parts[1:]
+	}
+	mainOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*mainNucleiBinary, finalArgs)
+	if err != nil {
+		return errors.Wrap(err, "could not run nuclei main test")
+	}
+	devOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*devNucleiBinary, finalArgs)
+	if err != nil {
+		return errors.Wrap(err, "could not run nuclei dev test")
+	}
+	if mainOutput == devOutput {
+		return nil
+	}
+	return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput)
+}

v2/cmd/functional-test/run.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo 'Building functional-test binary'
+go build
+
+echo 'Building Nuclei binary from current branch'
+go build -o nuclei_dev ../nuclei
+
+echo 'Installing latest release of nuclei'
+GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+
+echo 'Starting Nuclei functional test'
+./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt

v2/cmd/functional-test/testcases.txt

@@ -0,0 +1,51 @@
+{{binary}}
+{{binary}} -tags cve
+{{binary}} -tags cve,exposure
+{{binary}} -tags cve,exposure -tags token
+{{binary}} -tags cve,exposure -tags token,logs
+{{binary}} -tags "cve","exposure" -tags "token","logs"
+{{binary}} -tags 'cve','exposure' -tags 'token','logs'
+{{binary}} -tags cve -severity high
+{{binary}} -tags cve,exposure -severity high,critical
+{{binary}} -tags cve,exposure -severity high,critical,medium
+{{binary}} -tags cve -author geeknik
+{{binary}} -tags cve -author geeknik,pdteam
+{{binary}} -tags cve -author geeknik -severity high
+{{binary}} -tags cve
+{{binary}} -tags cve,exposure
+{{binary}} -tags cve,exposure -tags token
+{{binary}} -tags cve,exposure -tags token,logs
+{{binary}} -tags "cve","exposure" -tags "token","logs"
+{{binary}} -tags 'cve','exposure' -tags 'token','logs'
+{{binary}} -tags cve -severity high
+{{binary}} -tags cve,exposure -severity high,critical
+{{binary}} -tags cve,exposure -severity high,critical,medium
+{{binary}} -tags cve -author geeknik
+{{binary}} -tags cve -author geeknik,pdteam
+{{binary}} -tags cve -author geeknik -severity high
+{{binary}} -tags cve,exposure -author geeknik,pdteam -severity high,critical
+{{binary}} -tags "cve,exposure" -author "geeknik,pdteam" -severity high,critical
+{{binary}} -tags cve -etags ssrf
+{{binary}} -tags cve,exposure -etags ssrf,config
+{{binary}} -tags cve,exposure -etags ssrf,config -severity high
+{{binary}} -tags cve,exposure -etags ssrf,config -severity high -author geeknik
+{{binary}} -tags cve,dos,fuzz
+{{binary}} -tags cve -include-tags dos,fuzz
+{{binary}} -tags cve -exclude-tags cve2020
+{{binary}} -tags cve -exclude-templates cves/2020/
+{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml
+{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml -exclude-templates cves/2021/
+{{binary}} -t cves/
+{{binary}} -t cves/ -t exposures/
+{{binary}} -t cves/ -t exposures/ -tags config
+{{binary}} -t cves/ -t exposures/ -tags config,ssrf
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2021/
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2017/CVE-2017-7269.yaml
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -include-templates cves/2017/CVE-2017-7269.yaml
+{{binary}} -w workflows
+{{binary}} -w workflows -author geeknik,pdteam
+{{binary}} -w workflows -severity high,critical
+{{binary}} -w workflows -author geeknik,pdteam  -severity high,critical

v2/cmd/integration-test/dns.go

@@ -10,7 +10,7 @@ var dnsTestCases = map[string]testutils.TestCase{
 
 type dnsBasic struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *dnsBasic) Execute(filePath string) error {
 	var routerErr error
 

v2/cmd/integration-test/http.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/julienschmidt/httprouter"
+
 	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
 )
 
@@ -43,7 +44,7 @@ func httpDebugRequestDump(r *http.Request) {
 
 type httpGetHeaders struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpGetHeaders) Execute(filePath string) error {
 	router := httprouter.New()
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
@@ -67,7 +68,7 @@ func (h *httpGetHeaders) Execute(filePath string) error {
 
 type httpGetQueryString struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpGetQueryString) Execute(filePath string) error {
 	router := httprouter.New()
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
@@ -91,7 +92,7 @@ func (h *httpGetQueryString) Execute(filePath string) error {
 
 type httpGetRedirects struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpGetRedirects) Execute(filePath string) error {
 	router := httprouter.New()
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
@@ -117,7 +118,7 @@ func (h *httpGetRedirects) Execute(filePath string) error {
 
 type httpGet struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpGet) Execute(filePath string) error {
 	router := httprouter.New()
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
@@ -139,7 +140,7 @@ func (h *httpGet) Execute(filePath string) error {
 
 type httpPostBody struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpPostBody) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -172,7 +173,7 @@ func (h *httpPostBody) Execute(filePath string) error {
 
 type httpPostJSONBody struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpPostJSONBody) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -211,7 +212,7 @@ func (h *httpPostJSONBody) Execute(filePath string) error {
 
 type httpPostMultipartBody struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpPostMultipartBody) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -254,7 +255,7 @@ func (h *httpPostMultipartBody) Execute(filePath string) error {
 
 type httpRawDynamicExtractor struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRawDynamicExtractor) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -293,7 +294,7 @@ func (h *httpRawDynamicExtractor) Execute(filePath string) error {
 
 type httpRawGetQuery struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRawGetQuery) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -322,7 +323,7 @@ func (h *httpRawGetQuery) Execute(filePath string) error {
 
 type httpRawGet struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRawGet) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -350,7 +351,7 @@ func (h *httpRawGet) Execute(filePath string) error {
 
 type httpRawPayload struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRawPayload) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -386,7 +387,7 @@ func (h *httpRawPayload) Execute(filePath string) error {
 
 type httpRawPostBody struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRawPostBody) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -419,7 +420,7 @@ func (h *httpRawPostBody) Execute(filePath string) error {
 
 type httpRawCookieReuse struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRawCookieReuse) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error
@@ -468,7 +469,7 @@ func (h *httpRawCookieReuse) Execute(filePath string) error {
 
 type httpRawUnsafeRequest struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRawUnsafeRequest) Execute(filePath string) error {
 	var routerErr error
 
@@ -493,7 +494,7 @@ func (h *httpRawUnsafeRequest) Execute(filePath string) error {
 
 type httpRequestCondition struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *httpRequestCondition) Execute(filePath string) error {
 	router := httprouter.New()
 	var routerErr error

v2/cmd/integration-test/integration-test.go

@@ -13,6 +13,8 @@ var (
 	debug      = os.Getenv("DEBUG") == "true"
 	customTest = os.Getenv("TEST")
 	protocol   = os.Getenv("PROTO")
+
+	errored = false
 )
 
 func main() {
@@ -33,16 +35,18 @@ func main() {
 				if customTest != "" && !strings.Contains(file, customTest) {
 					continue // only run tests user asked
 				}
-				err := test.Execute(file)
-				if err != nil {
+				if err := test.Execute(file); err != nil {
 					fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, file, err)
-					os.Exit(1)
+					errored = true
 				} else {
 					fmt.Printf("%s Test \"%s\" passed!\n", success, file)
 				}
 			}
 		}
 	}
+	if errored {
+		os.Exit(1)
+	}
 }
 
 func errIncorrectResultsCount(results []string) error {

v2/cmd/integration-test/network.go

@@ -14,7 +14,7 @@ var networkTestcases = map[string]testutils.TestCase{
 
 type networkBasic struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *networkBasic) Execute(filePath string) error {
 	var routerErr error
 
@@ -47,7 +47,7 @@ func (h *networkBasic) Execute(filePath string) error {
 
 type networkMultiStep struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *networkMultiStep) Execute(filePath string) error {
 	var routerErr error
 

v2/cmd/integration-test/workflow.go

@@ -6,6 +6,7 @@ import (
 	"net/http/httptest"
 
 	"github.com/julienschmidt/httprouter"
+
 	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
 )
 
@@ -18,13 +19,13 @@ var workflowTestcases = map[string]testutils.TestCase{
 
 type workflowBasic struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *workflowBasic) Execute(filePath string) error {
 	router := httprouter.New()
-	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		httpDebugRequestDump(r)
 		fmt.Fprintf(w, "This is test matcher text")
-	}))
+	})
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
@@ -40,13 +41,13 @@ func (h *workflowBasic) Execute(filePath string) error {
 
 type workflowConditionMatched struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *workflowConditionMatched) Execute(filePath string) error {
 	router := httprouter.New()
-	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		httpDebugRequestDump(r)
 		fmt.Fprintf(w, "This is test matcher text")
-	}))
+	})
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
@@ -62,13 +63,13 @@ func (h *workflowConditionMatched) Execute(filePath string) error {
 
 type workflowConditionUnmatch struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *workflowConditionUnmatch) Execute(filePath string) error {
 	router := httprouter.New()
-	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		httpDebugRequestDump(r)
 		fmt.Fprintf(w, "This is test matcher text")
-	}))
+	})
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
@@ -84,13 +85,13 @@ func (h *workflowConditionUnmatch) Execute(filePath string) error {
 
 type workflowMatcherName struct{}
 
-// Executes executes a test case and returns an error if occurred
+// Execute executes a test case and returns an error if occurred
 func (h *workflowMatcherName) Execute(filePath string) error {
 	router := httprouter.New()
-	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		httpDebugRequestDump(r)
 		fmt.Fprintf(w, "This is test matcher text")
-	}))
+	})
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 

v2/cmd/nuclei/issue-tracker-config.yaml

@@ -1,7 +1,9 @@
+# to specify which severities should be reported
 #allow-list:
-#  severity: high,critical
+#  severity: critical, high
+# to specify which severities should be excluded from reporting
 #deny-list:
-#  severity: low
+#  severity: info, low, medium
 
 # github contains configuration options for github issue tracker
 #github: 
@@ -33,8 +35,10 @@
 
 # jira contains configuration options for jira issue tracker
 #jira:
-#  # Cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
+#  # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
 #  cloud: true
+#  # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created
+#  update-existing: false
 #  # URL is the jira application url
 #  url: ""
 #  # account-id is the account-id of the jira user or username in case of on-prem Jira
@@ -47,3 +51,20 @@
 #  project-name: ""
 #  # issue-type is the name of the created issue type
 #  issue-type: ""
+
+# elasticsearch contains configuration options for elasticsearch exporter
+#elasticsearch:
+#  # IP for elasticsearch instance
+#  ip: 127.0.0.1
+#  # Port is the port of elasticsearch instance
+#  port: 9200
+#  # IndexName is the name of the elasticsearch index
+#  index-name: nuclei
+#  # SSL enables ssl for elasticsearch connection
+#  ssl: false
+#  # SSLVerification disables SSL verification for elasticsearch
+#  ssl-verification: false
+#  # Username for the elasticsearch instance
+#  username: test
+#  # Password is the password for elasticsearch instance
+#  password: test

v2/cmd/nuclei/main.go

@@ -1,12 +1,14 @@
 package main
 
 import (
+	"fmt"
 	"os"
-	"path"
+	"path/filepath"
 
 	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/internal/runner"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
@@ -35,15 +37,19 @@ func main() {
 
 func readConfig() {
 	home, _ := os.UserHomeDir()
-	templatesDirectory := path.Join(home, "nuclei-templates")
+	templatesDirectory := filepath.Join(home, "nuclei-templates")
 
 	flagSet := goflags.NewFlagSet()
 	flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing
 on extensive configurability, massive extensibility and ease of use.`)
 
+	/* TODO Important: The defined default values, especially for slice/array types are NOT DEFAULT VALUES, but rather implicit values to which the user input is appended.
+	This can be very confusing and should be addressed
+	*/
+
 	createGroup(flagSet, "input", "Target",
-		flagSet.StringVarP(&options.Target, "target", "u", "", "target URL/host to scan"),
-		flagSet.StringVarP(&options.Targets, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
+		flagSet.StringSliceVarP(&options.Targets, "target", "u", []string{}, "target URLs/hosts to scan"),
+		flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
 	)
 
 	createGroup(flagSet, "templates", "Templates",
@@ -64,7 +70,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "templates to be executed even if they are excluded either by default or configuration"),
 		flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"),
 
-		flagSet.NormalizedStringSliceVarP(&options.Severity, "impact", "severity", []string{}, "execute templates that match the provided severities only"),
+		flagSet.VarP(&options.Severities, "impact", "severity", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
 		flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"),
 	)
 
@@ -80,6 +86,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"),
 
 		flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "don't display match metadata"),
+		flagSet.BoolVarP(&options.NoTimestamp, "no-timestamp", "nts", false, "don't display timestamp metadata in CLI output"),
 		flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "local nuclei reporting database (always use this to persist report data)"),
 
 		flagSet.StringVarP(&options.DiskExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"),
@@ -92,15 +99,18 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 		flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "custom headers in header:value format"),
 
+		flagSet.RuntimeMapVarP(&options.Vars, "var", "V", []string{}, "custom vars in var=value format"),
+
 		flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"),
 		flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"),
 		flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"),
+		flagSet.BoolVar(&options.EnvironmentVariables, "env-vars", false, "enable environment variables support"),
 	)
 
 	createGroup(flagSet, "interactsh", "interactsh",
-		flagSet.BoolVar(&options.NoInteractsh, "no-interactsh", false, "do not use interactsh server for blind interaction polling"),
-		flagSet.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "self-hosted Interactsh Server URL"),
-
+		flagSet.BoolVar(&options.NoInteractsh, "no-interactsh", false, "disable interactsh server for OOB testing"),
+		flagSet.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "interactsh server url for self-hosted instance"),
+		flagSet.StringVar(&options.InteractshToken, "interactsh-token", "", "authentication token for self-hosted interactsh server"),
 		flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"),
 		flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"),
 		flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"),
@@ -109,6 +119,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	createGroup(flagSet, "rate-limit", "Rate-Limit",
 		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
+		flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"),
 		flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
 		flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "maximum number of templates to be executed in parallel"),
 	)
@@ -116,6 +127,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	createGroup(flagSet, "optimization", "Optimizations",
 		flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"),
 		flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"),
+		flagSet.IntVar(&options.MaxHostError, "max-host-error", 30, "max errors for a host before skipping from scan"),
 
 		flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"),
 		flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"),
@@ -148,7 +160,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	createGroup(flagSet, "update", "Update",
 		flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei to the latest released version"),
 		flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update the community templates to latest released version"),
-		flagSet.BoolVarP(&options.NoUpdateTemplates, "no-update-templates", "nut", false, "Do not check for nuclei-templates updates"),
+		flagSet.BoolVarP(&options.NoUpdateTemplates, "no-update-templates", "nut", false, "do not check for nuclei-templates updates"),
 		flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default nuclei-templates directory"),
 	)
 

v2/go.mod

@@ -1,63 +1,119 @@
 module github.com/projectdiscovery/nuclei/v2
 
-go 1.15
+go 1.17
 
 require (
-	github.com/Knetic/govaluate v3.0.0+incompatible
-	github.com/andygrunwald/go-jira v1.13.0
+	github.com/Ice3man543/nvd v1.0.8
+	github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
+	github.com/akrylysov/pogreb v0.10.1 // indirect
+	github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c
+	github.com/andygrunwald/go-jira v1.14.0
+	github.com/antchfx/htmlquery v1.2.3
 	github.com/apex/log v1.9.0
 	github.com/blang/semver v3.5.1+incompatible
+	github.com/bluele/gcache v0.0.2
 	github.com/c4milo/unpackit v0.1.0 // indirect
 	github.com/corpix/uarand v0.1.1
-	github.com/fatih/structs v1.1.0 // indirect
-	github.com/go-rod/rod v0.91.1
-	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/go-rod/rod v0.101.7
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/gosuri/uilive v0.0.4 // indirect
 	github.com/gosuri/uiprogress v0.0.1 // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
-	github.com/json-iterator/go v1.1.10
+	github.com/itchyny/gojq v0.12.4
+	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/karlseguin/ccache v2.0.3+incompatible
 	github.com/karrick/godirwalk v1.16.1
 	github.com/logrusorgru/aurora v2.0.3+incompatible
-	github.com/mattn/go-runewidth v0.0.10 // indirect
-	github.com/miekg/dns v1.1.38
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/miekg/dns v1.1.43
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/owenrumney/go-sarif v1.0.4
+	github.com/owenrumney/go-sarif v1.0.11
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.8
-	github.com/projectdiscovery/fastdialer v0.0.8
-	github.com/projectdiscovery/goflags v0.0.6
+	github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e
+	github.com/projectdiscovery/goflags v0.0.7
 	github.com/projectdiscovery/gologger v1.1.4
-	github.com/projectdiscovery/hmap v0.0.1
-	github.com/projectdiscovery/interactsh v0.0.3
+	github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa
+	github.com/projectdiscovery/interactsh v0.0.4
+	github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77
 	github.com/projectdiscovery/rawhttp v0.0.7
-	github.com/projectdiscovery/retryabledns v1.0.10
-	github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727
-	github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d
+	github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a
+	github.com/projectdiscovery/retryablehttp-go v1.0.2
+	github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9
+	github.com/projectdiscovery/yamldoc-go v1.0.2
 	github.com/remeh/sizedwaitgroup v1.0.0
-	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/rs/xid v1.2.1
-	github.com/segmentio/ksuid v1.0.3
-	github.com/shirou/gopsutil/v3 v3.21.5
+	github.com/rs/xid v1.3.0
+	github.com/segmentio/ksuid v1.0.4
+	github.com/shirou/gopsutil/v3 v3.21.7
 	github.com/spaolacci/murmur3 v1.1.0
-	github.com/spf13/cast v1.3.1
+	github.com/spf13/cast v1.4.1
 	github.com/stretchr/testify v1.7.0
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible
-	github.com/trivago/tgo v1.0.7 // indirect
 	github.com/valyala/fasttemplate v1.2.1
-	github.com/xanzy/go-gitlab v0.44.0
-	go.uber.org/atomic v1.7.0
-	go.uber.org/multierr v1.6.0
-	go.uber.org/ratelimit v0.1.0
-	golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect
-	golang.org/x/net v0.0.0-20210521195947-fe42d452be8f
-	golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
-	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
-	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
+	github.com/xanzy/go-gitlab v0.50.3
+	github.com/ysmood/gson v0.6.4 // indirect
+	github.com/ysmood/leakless v0.7.0 // indirect
+	go.uber.org/atomic v1.9.0
+	go.uber.org/multierr v1.7.0
+	go.uber.org/ratelimit v0.2.0
+	golang.org/x/net v0.0.0-20210916014120-12bc252f5db8
+	golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab
+	golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect
+	golang.org/x/text v0.3.7
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 )
+
+require (
+	git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect
+	github.com/PuerkitoBio/goquery v1.6.0 // indirect
+	github.com/StackExchange/wmi v1.2.1 // indirect
+	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
+	github.com/andybalholm/cascadia v1.1.0 // indirect
+	github.com/antchfx/xpath v1.1.6 // indirect
+	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/dsnet/compress v0.0.1 // indirect
+	github.com/eggsampler/acme/v3 v3.2.1 // indirect
+	github.com/fatih/structs v1.1.0 // indirect
+	github.com/go-ole/go-ole v1.2.5 // indirect
+	github.com/golang-jwt/jwt v3.2.1+incompatible // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/go-querystring v1.0.0 // indirect
+	github.com/google/uuid v1.2.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
+	github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
+	github.com/itchyny/timefmt-go v0.1.3 // indirect
+	github.com/jasonlvhit/gocron v0.0.1 // indirect
+	github.com/karlseguin/ccache/v2 v2.0.8 // indirect
+	github.com/klauspost/compress v1.13.4 // indirect
+	github.com/klauspost/pgzip v1.2.5 // indirect
+	github.com/mattn/go-isatty v0.0.13 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect
+	github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect
+	github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect
+	github.com/projectdiscovery/mapcidr v0.0.8 // indirect
+	github.com/projectdiscovery/networkpolicy v0.0.1 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.7 // indirect
+	github.com/tklauser/numcpus v0.2.3 // indirect
+	github.com/trivago/tgo v1.0.7 // indirect
+	github.com/ulikunitz/xz v0.5.10 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/yl2chen/cidranger v1.0.2 // indirect
+	github.com/ysmood/goob v0.3.0 // indirect
+	github.com/zclconf/go-cty v1.8.4 // indirect
+	go.etcd.io/bbolt v1.3.6 // indirect
+	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)

v2/go.sum

@@ -31,32 +31,61 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8=
+git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs=
+github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
+github.com/Ice3man543/nvd v1.0.8 h1:2CBEgOxyWAkQocnnmEMmRtVPWooPRvcuHFLWj48EM4c=
+github.com/Ice3man543/nvd v1.0.8/go.mod h1:0DxLJk6revOcJKiZxa2K+rNF/HO1zJO97lqQtXhXfSc=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
+github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw=
+github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
-github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
-github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
-github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE=
-github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
+github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94=
+github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
+github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
+github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
+github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
+github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
+github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c h1:oJsq4z4xKgZWWOhrSZuLZ5KyYfRFytddLL1E5+psfIY=
+github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
+github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
+github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/andygrunwald/go-jira v1.14.0 h1:7GT/3qhar2dGJ0kq8w0d63liNyHOnxZsUZ9Pe4+AKBI=
+github.com/andygrunwald/go-jira v1.14.0/go.mod h1:KMo2f4DgMZA1C9FdImuLc04x4WQhn5derQpnsuBFgqE=
+github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M=
+github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
+github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0=
+github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
 github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
 github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA=
 github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
 github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
 github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
 github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
+github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og=
 github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I=
 github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r7ZRSxEo=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -65,19 +94,33 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
 github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4=
 github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
 github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU=
+github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ=
+github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
+github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8=
+github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE=
+github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
+github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
+github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
 github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
 github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eggsampler/acme/v3 v3.2.1 h1:Lfsrg3M2zt00QRnizOFzdpSfsS9oDvPsGrodXS/w1KI=
 github.com/eggsampler/acme/v3 v3.2.1/go.mod h1:/qh0rKC/Dh7Jj+p4So7DbWmFNzC4dpcpK53r226Fhuo=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -85,23 +128,28 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
-github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=
+github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
-github.com/go-rod/rod v0.91.1 h1:7xIlC/bXCXosZqZUl2x6GVB8tv4yMQ4W/ZVdGVa1qYI=
 github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI=
+github.com/go-rod/rod v0.101.7 h1:kbI5CNvcRhf7feybBln4xDutsM0mbsF0ENNZfKcF6WA=
+github.com/go-rod/rod v0.101.7/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY=
+github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
+github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -123,11 +171,14 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
 github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -136,8 +187,11 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@@ -146,6 +200,7 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@@ -163,51 +218,61 @@ github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY=
 github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI=
 github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw=
 github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0=
+github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
-github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
 github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
-github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
 github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs=
 github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk=
+github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA=
+github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o=
+github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg=
+github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
+github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
 github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
 github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
-github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU=
 github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w=
-github.com/karlseguin/ccache/v2 v2.0.7 h1:y5Pfi4eiyYCOD6LS/Kj+o6Nb4M5Ngpw9qFQs+v44ZYM=
-github.com/karlseguin/ccache/v2 v2.0.7/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ=
+github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA=
+github.com/karlseguin/ccache/v2 v2.0.8/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ=
 github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA=
 github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
 github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
+github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s=
+github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
 github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -215,95 +280,147 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA=
+github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
-github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
+github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw=
-github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
+github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
+github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4=
+github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/owenrumney/go-sarif v1.0.4 h1:0LFC5eHP6amc/9ajM1jDiE52UfXFcl/oozay+X3KgV4=
-github.com/owenrumney/go-sarif v1.0.4/go.mod h1:DXUGbHwQcCMvqcvZbxh8l/7diHsJVztOKZgmPt88RNI=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ=
+github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48=
+github.com/owenrumney/go-sarif v1.0.11 h1:7k4TLSi6h3vAozSECjO0arcQoeUNDMgvA7LDac95sJo=
+github.com/owenrumney/go-sarif v1.0.11/go.mod h1:hTBFbxU7GuVRUvwMx+eStp9M/Oun4xHCS3vqpPvket8=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e h1:7bwaFH1jvtOo5ndhTQgoA349ozhX+1dc4b6tbaPnBOA=
+github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs=
 github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20=
 github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
-github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0=
-github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4=
-github.com/projectdiscovery/goflags v0.0.6 h1:4ErduTfSC55cRR3TmUg+TQirBlCuBdBadrluAsy1pew=
-github.com/projectdiscovery/goflags v0.0.6/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
-github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E=
+github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U=
+github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc=
+github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0=
+github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges=
+github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ=
+github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk=
+github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
+github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
 github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
-github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
-github.com/projectdiscovery/interactsh v0.0.3 h1:PUkWk+NzSyd5glMqfORmuqizhsd7c3WdTYBOto/MQIU=
-github.com/projectdiscovery/interactsh v0.0.3/go.mod h1:dWnKO14d2FLP3kLhI9DecEsiAC/aZiJoUBGFjGhDskY=
+github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
+github.com/projectdiscovery/hmap v0.0.2-0.20210917073634-bfb0e9c03800/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
+github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0=
+github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo=
+github.com/projectdiscovery/interactsh v0.0.4 h1:3BtCZrrTovGYiqdFktXJ4NxKAQFvUvzcEI5pJIuShM8=
+github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE=
+github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE=
+github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
+github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
+github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 h1:VZ9H51A7tI7R/9I5W5l960Nkq7eMJqGd3y1wsuwzdjE=
+github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3/go.mod h1:blmYJkS8lSrrx3QcmcgS2tZIxlojeVmoGeA9twslCBU=
+github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
+github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM=
+github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00=
+github.com/projectdiscovery/mapcidr v0.0.8 h1:16U05F2x3o/jSTsxSCY2hCuCs9xOSwVxjo2zlsL4L4E=
+github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00=
+github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I=
+github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs=
+github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 h1:SNtAiRRrJtDJJDroaa/bFXt/Tix2LA6+rHRib0ORlJQ=
+github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M=
+github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY=
 github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg=
 github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
-github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
-github.com/projectdiscovery/retryabledns v1.0.10 h1:xJZ2aKoqrNg/OZEw1+4+QIOH40V/WkZDYY1ZZc+uphE=
-github.com/projectdiscovery/retryabledns v1.0.10/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
+github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
+github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
+github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a h1:WJQjr9qi/VjWhdNiGyNqcFi0967Gp0W3I769bCpHOJE=
+github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI=
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
-github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727 h1:CJHP3CLCc/eqdXQEvZy8KiiqtAk9kEsd1URtPyPAQ1s=
-github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
-github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nlOAex7twmrEqD5i6WLnugF9uO3DQ6jDEKN9gevrTAk=
-github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
-github.com/prologic/smtpd v0.0.0-20210126001904-0893ad18168e h1:ZT3wZ92sp/EHEE/HcFCWCsYS3ROLjHb6EqSX8qYrgXw=
-github.com/prologic/smtpd v0.0.0-20210126001904-0893ad18168e/go.mod h1:GkLsdH1RZj6RDKeI9A05NGZYmEZQ/PbQcZPnZoSZuYI=
+github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U=
+github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
+github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
+github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
+github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes=
+github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
+github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ=
+github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
-github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
-github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
-github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
+github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
+github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
+github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
+github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c=
+github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/shirou/gopsutil/v3 v3.21.5 h1:YUBf0w/KPLk7w1803AYBnH7BmA+1Z/Q5MEZxpREUaB4=
-github.com/shirou/gopsutil/v3 v3.21.5/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw=
+github.com/shirou/gopsutil/v3 v3.21.7 h1:PnTqQamUjwEDSgn+nBGu0qSDV/CfvyiR/gwTH3i7HTU=
+github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4=
 github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
 github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
-github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA=
+github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -320,13 +437,13 @@ github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPf
 github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible h1:guTq1YxwB8XSILkI9q4IrOmrCOS6Hc1L3hmOhi4Swcs=
 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible/go.mod h1:waFwwyiAhGey2e+dNoYQ/iLhIcFqhCW7zL/+vDU1WLo=
-github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M=
-github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek=
-github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc=
-github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8=
-github.com/trivago/tgo v1.0.1/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
+github.com/tklauser/go-sysconf v0.3.7 h1:HT7h4+536gjqeq1ZIJPgOl1rg1XFatQGVZWp7Py53eg=
+github.com/tklauser/go-sysconf v0.3.7/go.mod h1:JZIdXh4RmBvZDBZ41ld2bGxRV3n4daiiqA3skYhAoQ4=
+github.com/tklauser/numcpus v0.2.3 h1:nQ0QYpiritP6ViFhrKYsiv6VVxOpum2Gks5GhnJbS/8=
+github.com/tklauser/numcpus v0.2.3/go.mod h1:vpEPS/JC+oZGGQ/My/vJnNsvMDQL6PwOqt8dsCw5j+E=
 github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM=
 github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
@@ -338,45 +455,56 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+
 github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
-github.com/xanzy/go-gitlab v0.44.0 h1:cEiGhqu7EpFGuei2a2etAwB+x6403E5CvpLn35y+GPs=
-github.com/xanzy/go-gitlab v0.44.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
+github.com/xanzy/go-gitlab v0.50.3 h1:M7ncgNhCN4jaFNyXxarJhCLa9Qi6fdmCxFFhMTQPZiY=
+github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
+github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
 github.com/ysmood/goob v0.3.0 h1:XZ51cZJ4W3WCoCiUktixzMIQF86W7G5VFL4QQ/Q2uS0=
 github.com/ysmood/goob v0.3.0/go.mod h1:S3lq113Y91y1UBf1wj1pFOxeahvfKkCk6mTWTWbDdWs=
-github.com/ysmood/got v0.9.3 h1:qx51X49jL/WAiqZzPTkPZ0zp5pTmrWJa4zYFTYo0gHI=
 github.com/ysmood/got v0.9.3/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
-github.com/ysmood/gotrace v0.2.0 h1:IkTC6rJREwXSaG8yWK+NFwIJGIsxA1DjC6/gxYyQttE=
+github.com/ysmood/got v0.14.1/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
+github.com/ysmood/got v0.15.1 h1:X5jAbMyBf5yeezuFMp9HaMGXZWMSqIQcUlAHI+kJmUs=
+github.com/ysmood/got v0.15.1/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
 github.com/ysmood/gotrace v0.2.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
-github.com/ysmood/gson v0.6.3 h1:4cU+5oOdsyundXHy00t99H0rLXLthuseD3x6W+xmCiU=
+github.com/ysmood/gotrace v0.2.2 h1:006KHGRThSRf8lwh4EyhNmuuq/l+Ygs+JqojkhEG1/E=
+github.com/ysmood/gotrace v0.2.2/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
 github.com/ysmood/gson v0.6.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
-github.com/ysmood/leakless v0.6.12 h1:XxtRYl97bJklfv4BZVdyGnd/y42p6w8lu1hUzfCkT4M=
+github.com/ysmood/gson v0.6.4 h1:Yb6tosv6bk59HqjZu2/7o4BFherpYEMkDkXmlhgryZ4=
+github.com/ysmood/gson v0.6.4/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
 github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
+github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw=
+github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/zclconf/go-cty v1.8.2 h1:u+xZfBKgpycDnTNjPhGiTEYZS5qS/Sb5MqSfm7vzcjg=
-github.com/zclconf/go-cty v1.8.2/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/zclconf/go-cty v1.8.4 h1:pwhhz5P+Fjxse7S7UriBrMu6AUJSZM5pKqGem1PjGAs=
+github.com/zclconf/go-cty v1.8.4/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
+go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
-go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
+go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
+go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
+golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df h1:y7QZzfUiTwWam+xBn29Ulb8CBwVN5UdzmMDavl9Whlw=
-golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -407,10 +535,11 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -429,24 +558,32 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 h1:5vD4XjIc0X5+kHZjx4UecYdjA6mJo+XXNoaW0EjU5Os=
-golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab h1:llrcWN/wOwO+6gAyfBzxb5hZ+c3mriU/0+KNgYu6adA=
+golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -455,10 +592,13 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -467,13 +607,18 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -486,28 +631,37 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw=
+golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE=
-golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -543,12 +697,16 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs
 golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -629,8 +787,11 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
-google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -639,12 +800,14 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
 gopkg.in/corvus-ch/zbase32.v1 v1.0.0 h1:K4u1NprbDNvKPczKfHLbwdOWHTZ0zfv2ow71H1nRnFU=
 gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DSo0nK9U+c0Y=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -658,6 +821,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

v2/internal/colorizer/colorizer.go

@@ -1,24 +1,41 @@
 package colorizer
 
-import "github.com/logrusorgru/aurora"
+import (
+	"fmt"
 
-// Colorizer returns a colorized severity printer
-type Colorizer struct {
-	Data map[string]string
-}
+	"github.com/logrusorgru/aurora"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+)
 
 const (
 	fgOrange uint8 = 208
 )
 
-// New returns a new severity based colorizer
-func New(colorizer aurora.Aurora) *Colorizer {
-	severityMap := map[string]string{
-		"info":     colorizer.Blue("info").String(),
-		"low":      colorizer.Green("low").String(),
-		"medium":   colorizer.Yellow("medium").String(),
-		"high":     colorizer.Index(fgOrange, "high").String(),
-		"critical": colorizer.Red("critical").String(),
+func GetColor(colorizer aurora.Aurora, templateSeverity fmt.Stringer) string {
+	var method func(arg interface{}) aurora.Value
+	switch templateSeverity {
+	case severity.Info:
+		method = colorizer.Blue
+	case severity.Low:
+		method = colorizer.Green
+	case severity.Medium:
+		method = colorizer.Yellow
+	case severity.High:
+		method = func(stringValue interface{}) aurora.Value { return colorizer.Index(fgOrange, stringValue) }
+	case severity.Critical:
+		method = colorizer.Red
+	default:
+		gologger.Warning().Msgf("The '%s' severity does not have an color associated!", templateSeverity)
+		method = colorizer.White
+	}
+
+	return method(templateSeverity.String()).String()
+}
+
+func New(colorizer aurora.Aurora) func(severity.Severity) string {
+	return func(severity severity.Severity) string {
+		return GetColor(colorizer, severity)
 	}
-	return &Colorizer{Data: severityMap}
 }

v2/internal/runner/banner.go

@@ -20,6 +20,6 @@ func showBanner() {
 	gologger.Print().Msgf("%s\n", banner)
 	gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n")
 
-	gologger.Warning().Msgf("Use with caution. You are responsible for your actions\n")
-	gologger.Warning().Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
+	gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n")
+	gologger.Error().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
 }

v2/internal/runner/config.go

@@ -1 +0,0 @@
-package runner

v2/internal/runner/options.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/projectdiscovery/gologger"
@@ -20,6 +21,11 @@ func ParseOptions(options *types.Options) {
 	// Check if stdin pipe was given
 	options.Stdin = hasStdin()
 
+	// if VerboseVerbose is set, it implicitly enables the Verbose option as well
+	if options.VerboseVerbose {
+		options.Verbose = true
+	}
+
 	// Read the inputs and configure the logging
 	configureOutput(options)
 
@@ -35,7 +41,7 @@ func ParseOptions(options *types.Options) {
 		if err != nil {
 			gologger.Fatal().Msgf("Could not read template configuration: %s\n", err)
 		}
-		gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", configuration.CurrentVersion, configuration.TemplatesDirectory)
+		gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", configuration.TemplateVersion, configuration.TemplatesDirectory)
 		os.Exit(0)
 	}
 
@@ -76,21 +82,22 @@ func hasStdin() bool {
 
 // validateOptions validates the configuration options passed
 func validateOptions(options *types.Options) error {
-	// Both verbose and silent flags were used
 	if options.Verbose && options.Silent {
 		return errors.New("both verbose and silent mode specified")
 	}
 
-	// Validate proxy options if provided
-	err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)")
-	if err != nil {
+	if err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)"); err != nil {
 		return err
 	}
 
-	err = validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)")
-	if err != nil {
+	if err := validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)"); err != nil {
 		return err
 	}
+
+	if options.Validate {
+		validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows)
+	}
+
 	return nil
 }
 
@@ -107,10 +114,10 @@ func isValidURL(urlString string) bool {
 	return err == nil
 }
 
-// configureOutput configures the output on the screen
+// configureOutput configures the output logging levels to be displayed on the screen
 func configureOutput(options *types.Options) {
 	// If the user desires verbose output, show verbose output
-	if options.Verbose {
+	if options.Verbose || options.VerboseVerbose {
 		gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
 	}
 	if options.Debug {
@@ -149,3 +156,21 @@ func loadResolvers(options *types.Options) {
 		}
 	}
 }
+
+func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) {
+	allGivenTemplatePaths := append(templatePaths, workflowPaths...)
+
+	for _, templatePath := range allGivenTemplatePaths {
+		if templatesDirectory != templatePath && filepath.IsAbs(templatePath) {
+			fileInfo, err := os.Stat(templatePath)
+			if err == nil && fileInfo.IsDir() {
+				relativizedPath, err2 := filepath.Rel(templatesDirectory, templatePath)
+				if err2 != nil || (len(relativizedPath) >= 2 && relativizedPath[:2] == "..") {
+					gologger.Warning().Msgf("The given path (%s) is outside the default template directory path (%s)! "+
+						"Referenced sub-templates with relative paths in workflows will be resolved against the default template directory.", templatePath, templatesDirectory)
+					break
+				}
+			}
+		}
+	}
+}

v2/internal/runner/processor.go

@@ -7,13 +7,17 @@ import (
 	"go.uber.org/atomic"
 )
 
-// processTemplateWithList process a template on the URL list
+// processTemplateWithList execute a template against the list of user provided targets
 func (r *Runner) processTemplateWithList(template *templates.Template) bool {
 	results := &atomic.Bool{}
 	wg := sizedwaitgroup.New(r.options.BulkSize)
 	r.hostMap.Scan(func(k, _ []byte) error {
 		URL := string(k)
 
+		// Skip if the host has had errors
+		if r.hostErrors != nil && r.hostErrors.Check(URL) {
+			return nil
+		}
 		wg.Add()
 		go func(URL string) {
 			defer wg.Done()
@@ -37,6 +41,11 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool {
 
 	r.hostMap.Scan(func(k, _ []byte) error {
 		URL := string(k)
+
+		// Skip if the host has had errors
+		if r.hostErrors != nil && r.hostErrors.Check(URL) {
+			return nil
+		}
 		wg.Add()
 		go func(URL string) {
 			defer wg.Done()

v2/internal/runner/runner.go

@@ -4,23 +4,32 @@ import (
 	"bufio"
 	"fmt"
 	"os"
-	"path"
+	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
+	"github.com/remeh/sizedwaitgroup"
+	"github.com/rs/xid"
+	"go.uber.org/atomic"
+	"go.uber.org/ratelimit"
+	"gopkg.in/yaml.v2"
+
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/hmap/store/hybrid"
 	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
@@ -29,11 +38,8 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
-	"github.com/remeh/sizedwaitgroup"
-	"github.com/rs/xid"
-	"go.uber.org/atomic"
-	"go.uber.org/ratelimit"
-	"gopkg.in/yaml.v2"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
 )
 
 // Runner is a client for running the enumeration process.
@@ -49,9 +55,10 @@ type Runner struct {
 	progress        progress.Progress
 	colorizer       aurora.Aurora
 	issuesClient    *reporting.Client
-	severityColors  *colorizer.Colorizer
+	addColor        func(severity.Severity) string
 	browser         *engine.Browser
 	ratelimiter     ratelimit.Limiter
+	hostErrors      *hosterrorscache.Cache
 }
 
 // New creates a new client for running enumeration process.
@@ -65,6 +72,9 @@ func New(options *types.Options) (*Runner, error) {
 		}
 		return nil, nil
 	}
+	if options.Validate {
+		parsers.ShouldValidate = true
+	}
 	if err := runner.updateTemplates(); err != nil {
 		gologger.Warning().Msgf("Could not update templates: %s\n", err)
 	}
@@ -77,6 +87,167 @@ func New(options *types.Options) (*Runner, error) {
 	}
 
 	runner.catalog = catalog.New(runner.options.TemplatesDirectory)
+
+	reportingOptions, err := createReportingOptions(options)
+	if err != nil {
+		return nil, err
+	}
+	if reportingOptions != nil {
+		client, err := reporting.New(reportingOptions, options.ReportingDB)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create issue reporting client")
+		}
+		runner.issuesClient = client
+	}
+
+	// output coloring
+	useColor := !options.NoColor
+	runner.colorizer = aurora.NewAurora(useColor)
+	runner.addColor = colorizer.New(runner.colorizer)
+
+	if options.TemplateList {
+		runner.listAvailableTemplates()
+		os.Exit(0)
+	}
+
+	if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates {
+		os.Exit(0)
+	}
+	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create temporary input file")
+	}
+	runner.hostMap = hm
+
+	runner.inputCount = 0
+	dupeCount := 0
+
+	// Handle multiple targets
+	if len(options.Targets) != 0 {
+		for _, target := range options.Targets {
+			url := strings.TrimSpace(target)
+			if url == "" {
+				continue
+			}
+
+			if _, ok := runner.hostMap.Get(url); ok {
+				dupeCount++
+				continue
+			}
+
+			runner.inputCount++
+			// nolint:errcheck // ignoring error
+			runner.hostMap.Set(url, nil)
+		}
+	}
+
+	// Handle stdin
+	if options.Stdin {
+		scanner := bufio.NewScanner(os.Stdin)
+		for scanner.Scan() {
+			url := strings.TrimSpace(scanner.Text())
+			if url == "" {
+				continue
+			}
+
+			if _, ok := runner.hostMap.Get(url); ok {
+				dupeCount++
+				continue
+			}
+
+			runner.inputCount++
+			// nolint:errcheck // ignoring error
+			runner.hostMap.Set(url, nil)
+		}
+	}
+
+	// Handle target file
+	if options.TargetsFilePath != "" {
+		input, inputErr := os.Open(options.TargetsFilePath)
+		if inputErr != nil {
+			return nil, errors.Wrap(inputErr, "could not open targets file")
+		}
+		scanner := bufio.NewScanner(input)
+		for scanner.Scan() {
+			url := strings.TrimSpace(scanner.Text())
+			if url == "" {
+				continue
+			}
+			if _, ok := runner.hostMap.Get(url); ok {
+				dupeCount++
+				continue
+			}
+			runner.inputCount++
+			// nolint:errcheck // ignoring error
+			runner.hostMap.Set(url, nil)
+		}
+		input.Close()
+	}
+
+	if dupeCount > 0 {
+		gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", dupeCount)
+	}
+
+	// Create the output file if asked
+	outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.Output, options.TraceLogFile)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create output file")
+	}
+	runner.output = outputWriter
+
+	if options.JSON && options.EnableProgressBar {
+		options.StatsJSON = true
+	}
+	if options.StatsJSON {
+		options.EnableProgressBar = true
+	}
+	// Creates the progress tracking object
+	var progressErr error
+	runner.progress, progressErr = progress.NewStatsTicker(options.StatsInterval, options.EnableProgressBar, options.StatsJSON, options.Metrics, options.MetricsPort)
+	if progressErr != nil {
+		return nil, progressErr
+	}
+
+	// create project file if requested or load the existing one
+	if options.Project {
+		var projectFileErr error
+		runner.projectFile, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: utils.IsBlank(options.ProjectPath)})
+		if projectFileErr != nil {
+			return nil, projectFileErr
+		}
+	}
+
+	if !options.NoInteractsh {
+		interactshClient, err := interactsh.New(&interactsh.Options{
+			ServerURL:      options.InteractshURL,
+			Authorization:  options.InteractshToken,
+			CacheSize:      int64(options.InteractionsCacheSize),
+			Eviction:       time.Duration(options.InteractionsEviction) * time.Second,
+			ColldownPeriod: time.Duration(options.InteractionsColldownPeriod) * time.Second,
+			PollDuration:   time.Duration(options.InteractionsPollDuration) * time.Second,
+			Output:         runner.output,
+			IssuesClient:   runner.issuesClient,
+			Progress:       runner.progress,
+			Debug:          runner.options.Debug,
+		})
+		if err != nil {
+			gologger.Error().Msgf("Could not create interactsh client: %s", err)
+		} else {
+			runner.interactsh = interactshClient
+		}
+	}
+
+	if options.RateLimitMinute > 0 {
+		runner.ratelimiter = ratelimit.New(options.RateLimitMinute, ratelimit.Per(60*time.Second))
+	} else if options.RateLimit > 0 {
+		runner.ratelimiter = ratelimit.New(options.RateLimit)
+	} else {
+		runner.ratelimiter = ratelimit.NewUnlimited()
+	}
+	return runner, nil
+}
+
+func createReportingOptions(options *types.Options) (*reporting.Options, error) {
 	var reportingOptions *reporting.Options
 	if options.ReportingConfig != "" {
 		file, err := os.Open(options.ReportingConfig)
@@ -107,141 +278,7 @@ func New(options *types.Options) (*Runner, error) {
 			reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}
 		}
 	}
-	if reportingOptions != nil {
-		client, err := reporting.New(reportingOptions, options.ReportingDB)
-		if err != nil {
-			return nil, errors.Wrap(err, "could not create issue reporting client")
-		}
-		runner.issuesClient = client
-	}
-
-	// output coloring
-	useColor := !options.NoColor
-	runner.colorizer = aurora.NewAurora(useColor)
-	runner.severityColors = colorizer.New(runner.colorizer)
-
-	if options.TemplateList {
-		runner.listAvailableTemplates()
-		os.Exit(0)
-	}
-
-	if (len(options.Templates) == 0 || !options.NewTemplates || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
-		os.Exit(0)
-	}
-	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not create temporary input file")
-	}
-	runner.hostMap = hm
-
-	runner.inputCount = 0
-	dupeCount := 0
-
-	// Handle single target
-	if options.Target != "" {
-		runner.inputCount++
-		// nolint:errcheck // ignoring error
-		runner.hostMap.Set(options.Target, nil)
-	}
-
-	// Handle stdin
-	if options.Stdin {
-		scanner := bufio.NewScanner(os.Stdin)
-		for scanner.Scan() {
-			url := strings.TrimSpace(scanner.Text())
-			if url == "" {
-				continue
-			}
-			if _, ok := runner.hostMap.Get(url); ok {
-				dupeCount++
-				continue
-			}
-			runner.inputCount++
-			// nolint:errcheck // ignoring error
-			runner.hostMap.Set(url, nil)
-		}
-	}
-
-	// Handle taget file
-	if options.Targets != "" {
-		input, inputErr := os.Open(options.Targets)
-		if inputErr != nil {
-			return nil, errors.Wrap(inputErr, "could not open targets file")
-		}
-		scanner := bufio.NewScanner(input)
-		for scanner.Scan() {
-			url := strings.TrimSpace(scanner.Text())
-			if url == "" {
-				continue
-			}
-			if _, ok := runner.hostMap.Get(url); ok {
-				dupeCount++
-				continue
-			}
-			runner.inputCount++
-			// nolint:errcheck // ignoring error
-			runner.hostMap.Set(url, nil)
-		}
-		input.Close()
-	}
-
-	if dupeCount > 0 {
-		gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", dupeCount)
-	}
-
-	// Create the output file if asked
-	outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.JSON, options.Output, options.TraceLogFile)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not create output file")
-	}
-	runner.output = outputWriter
-
-	if options.JSON && options.EnableProgressBar {
-		options.StatsJSON = true
-	}
-	if options.StatsJSON {
-		options.EnableProgressBar = true
-	}
-	// Creates the progress tracking object
-	var progressErr error
-	runner.progress, progressErr = progress.NewStatsTicker(options.StatsInterval, options.EnableProgressBar, options.StatsJSON, options.Metrics, options.MetricsPort)
-	if progressErr != nil {
-		return nil, progressErr
-	}
-
-	// create project file if requested or load existing one
-	if options.Project {
-		var projectFileErr error
-		runner.projectFile, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""})
-		if projectFileErr != nil {
-			return nil, projectFileErr
-		}
-	}
-
-	if !options.NoInteractsh {
-		interactshClient, err := interactsh.New(&interactsh.Options{
-			ServerURL:      options.InteractshURL,
-			CacheSize:      int64(options.InteractionsCacheSize),
-			Eviction:       time.Duration(options.InteractionsEviction) * time.Second,
-			ColldownPeriod: time.Duration(options.InteractionsColldownPeriod) * time.Second,
-			PollDuration:   time.Duration(options.InteractionsPollDuration) * time.Second,
-			Output:         runner.output,
-			IssuesClient:   runner.issuesClient,
-			Progress:       runner.progress,
-		})
-		if err != nil {
-			gologger.Error().Msgf("Could not create interactsh client: %s", err)
-		} else {
-			runner.interactsh = interactshClient
-		}
-	}
-
-	if options.RateLimit > 0 {
-		runner.ratelimiter = ratelimit.New(options.RateLimit)
-	} else {
-		runner.ratelimiter = ratelimit.NewUnlimited()
-	}
-	return runner, nil
+	return reportingOptions, nil
 }
 
 // Close releases all the resources and cleans up
@@ -261,7 +298,7 @@ func (r *Runner) Close() {
 func (r *Runner) RunEnumeration() error {
 	defer r.Close()
 
-	// If user asked for new templates to be executed, collect the list from template directory.
+	// If user asked for new templates to be executed, collect the list from the templates' directory.
 	if r.options.NewTemplates {
 		templatesLoaded, err := r.readNewTemplatesFile()
 		if err != nil {
@@ -273,17 +310,31 @@ func (r *Runner) RunEnumeration() error {
 	r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
 	r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
 
-	executerOpts := protocols.ExecuterOptions{
-		Output:       r.output,
-		Options:      r.options,
-		Progress:     r.progress,
-		Catalog:      r.catalog,
-		IssuesClient: r.issuesClient,
-		RateLimiter:  r.ratelimiter,
-		Interactsh:   r.interactsh,
-		ProjectFile:  r.projectFile,
-		Browser:      r.browser,
+	var cache *hosterrorscache.Cache
+	if r.options.MaxHostError > 0 {
+		cache = hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount).SetVerbose(r.options.Verbose)
 	}
+	r.hostErrors = cache
+	executerOpts := protocols.ExecuterOptions{
+		Output:          r.output,
+		Options:         r.options,
+		Progress:        r.progress,
+		Catalog:         r.catalog,
+		IssuesClient:    r.issuesClient,
+		RateLimiter:     r.ratelimiter,
+		Interactsh:      r.interactsh,
+		ProjectFile:     r.projectFile,
+		Browser:         r.browser,
+		HostErrorsCache: cache,
+	}
+
+	workflowLoader, err := parsers.NewLoader(&executerOpts)
+	if err != nil {
+		return errors.Wrap(err, "Could not create loader.")
+	}
+
+	executerOpts.WorkflowLoader = workflowLoader
+
 	loaderConfig := loader.Config{
 		Templates:          r.options.Templates,
 		Workflows:          r.options.Workflows,
@@ -292,7 +343,7 @@ func (r *Runner) RunEnumeration() error {
 		ExcludeTags:        r.options.ExcludeTags,
 		IncludeTemplates:   r.options.IncludeTemplates,
 		Authors:            r.options.Author,
-		Severities:         r.options.Severity,
+		Severities:         r.options.Severities,
 		IncludeTags:        r.options.IncludeTags,
 		TemplatesDirectory: r.options.TemplatesDirectory,
 		Catalog:            r.catalog,
@@ -302,14 +353,23 @@ func (r *Runner) RunEnumeration() error {
 	if err != nil {
 		return errors.Wrap(err, "could not load templates from config")
 	}
+	store.Load()
+
 	if r.options.Validate {
-		if !store.ValidateTemplates(r.options.Templates, r.options.Workflows) {
-			return errors.New("an error occurred during templates validation")
+		if err := store.ValidateTemplates(r.options.Templates, r.options.Workflows); err != nil {
+			return err
+		}
+		if stats.GetValue(parsers.SyntaxErrorStats) == 0 && stats.GetValue(parsers.SyntaxWarningStats) == 0 {
+			gologger.Info().Msgf("All templates validated successfully\n")
+		} else {
+			return errors.New("encountered errors while performing template validation")
 		}
-		gologger.Info().Msgf("All templates validated successfully\n")
 		return nil // exit
 	}
-	store.Load()
+
+	// Display stats for any loaded templates' syntax warnings or errors
+	stats.Display(parsers.SyntaxWarningStats)
+	stats.Display(parsers.SyntaxErrorStats)
 
 	builder := &strings.Builder{}
 	if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" {
@@ -329,10 +389,10 @@ func (r *Runner) RunEnumeration() error {
 
 	gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr)
 
-	if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" {
+	if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { // TODO extract duplicated logic
 		builder.WriteString(" (")
 
-		if r.templatesConfig.CurrentVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
+		if r.templatesConfig.TemplateVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
 			builder.WriteString(r.colorizer.Green("latest").String())
 		} else {
 			builder.WriteString(r.colorizer.Red("outdated").String())
@@ -342,16 +402,18 @@ func (r *Runner) RunEnumeration() error {
 	messageStr = builder.String()
 	builder.Reset()
 
-	gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.CurrentVersion, messageStr)
-
+	if r.templatesConfig != nil {
+		gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.TemplateVersion, messageStr)
+	}
 	if r.interactsh != nil {
 		gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL)
 	}
 	if len(store.Templates()) > 0 {
-		gologger.Info().Msgf("Templates loaded: %d (New: %d)", len(store.Templates()), r.countNewTemplates())
+		gologger.Info().Msgf("Templates added in last update: %d", r.countNewTemplates())
+		gologger.Info().Msgf("Templates loaded for scan: %d", len(store.Templates()))
 	}
 	if len(store.Workflows()) > 0 {
-		gologger.Info().Msgf("Workflows loaded: %d", len(store.Workflows()))
+		gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows()))
 	}
 
 	// pre-parse all the templates, apply filters
@@ -360,7 +422,7 @@ func (r *Runner) RunEnumeration() error {
 	var unclusteredRequests int64
 	for _, template := range store.Templates() {
 		// workflows will dynamically adjust the totals while running, as
-		// it can't be know in advance which requests will be called
+		// it can't be known in advance which requests will be called
 		if len(template.Workflows) > 0 {
 			continue
 		}
@@ -377,7 +439,7 @@ func (r *Runner) RunEnumeration() error {
 	}
 	templatesMap := make(map[string]*templates.Template)
 	for _, v := range store.Templates() {
-		templatesMap[v.ID] = v
+		templatesMap[v.Path] = v
 	}
 	originalTemplatesCount := len(store.Templates())
 	clusterCount := 0
@@ -385,15 +447,16 @@ func (r *Runner) RunEnumeration() error {
 	for _, cluster := range clusters {
 		if len(cluster) > 1 && !r.options.OfflineHTTP {
 			executerOpts := protocols.ExecuterOptions{
-				Output:       r.output,
-				Options:      r.options,
-				Progress:     r.progress,
-				Catalog:      r.catalog,
-				RateLimiter:  r.ratelimiter,
-				IssuesClient: r.issuesClient,
-				Browser:      r.browser,
-				ProjectFile:  r.projectFile,
-				Interactsh:   r.interactsh,
+				Output:          r.output,
+				Options:         r.options,
+				Progress:        r.progress,
+				Catalog:         r.catalog,
+				RateLimiter:     r.ratelimiter,
+				IssuesClient:    r.issuesClient,
+				Browser:         r.browser,
+				ProjectFile:     r.projectFile,
+				Interactsh:      r.interactsh,
+				HostErrorsCache: cache,
 			}
 			clusterID := fmt.Sprintf("cluster-%s", xid.New().String())
 
@@ -408,6 +471,7 @@ func (r *Runner) RunEnumeration() error {
 			finalTemplates = append(finalTemplates, cluster...)
 		}
 	}
+
 	finalTemplates = append(finalTemplates, store.Workflows()...)
 
 	var totalRequests int64
@@ -425,9 +489,14 @@ func (r *Runner) RunEnumeration() error {
 
 	// 0 matches means no templates were found in directory
 	if templateCount == 0 {
-		return errors.New("no templates were found")
+		return errors.New("no valid templates were found")
 	}
 
+	/*
+		TODO does it make sense to run the logic below if there are no targets specified?
+		Can we safely assume the user is just experimenting with the template/workflow filters before running them?
+	*/
+
 	results := &atomic.Bool{}
 	wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
 
@@ -470,7 +539,10 @@ func (r *Runner) RunEnumeration() error {
 
 // readNewTemplatesFile reads newly added templates from directory if it exists
 func (r *Runner) readNewTemplatesFile() ([]string, error) {
-	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	if r.templatesConfig == nil {
+		return nil, nil
+	}
+	additionsFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
 	file, err := os.Open(additionsFile)
 	if err != nil {
 		return nil, err
@@ -489,9 +561,12 @@ func (r *Runner) readNewTemplatesFile() ([]string, error) {
 	return templatesList, nil
 }
 
-// readNewTemplatesFile reads newly added templates from directory if it exists
+// countNewTemplates returns the number of newly added templates
 func (r *Runner) countNewTemplates() int {
-	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	if r.templatesConfig == nil {
+		return 0
+	}
+	additionsFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
 	file, err := os.Open(additionsFile)
 	if err != nil {
 		return 0

v2/internal/runner/runner_test.go

@@ -0,0 +1,26 @@
+package runner
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+func Test_createReportingOptions(t *testing.T) {
+	var options types.Options
+	options.ReportingConfig = "../../../integration_tests/test-issue-tracker-config1.yaml"
+	resultOptions, err := createReportingOptions(&options)
+
+	assert.Nil(t, err)
+	assert.Equal(t, resultOptions.AllowList.Severities, severity.Severities{severity.High, severity.Critical})
+	assert.Equal(t, resultOptions.DenyList.Severities, severity.Severities{severity.Low})
+
+	options.ReportingConfig = "../../../integration_tests/test-issue-tracker-config2.yaml"
+	resultOptions2, err := createReportingOptions(&options)
+	assert.Nil(t, err)
+	assert.Equal(t, resultOptions2.AllowList.Severities, resultOptions.AllowList.Severities)
+	assert.Equal(t, resultOptions2.DenyList.Severities, resultOptions.DenyList.Severities)
+}

v2/internal/runner/templates.go

@@ -1,67 +1,36 @@
 package runner
 
 import (
-	"bytes"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strings"
 
 	"github.com/karrick/godirwalk"
+
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
-	"gopkg.in/yaml.v2"
 )
 
-// parseTemplateFile returns the parsed template file
-func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
-	f, err := os.Open(file)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	data, err := ioutil.ReadAll(f)
-	if err != nil {
-		return nil, err
-	}
-
-	template := &templates.Template{}
-	err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
-	if err != nil {
-		return nil, err
-	}
-	return template, nil
-}
-
-func (r *Runner) templateLogMsg(id, name, author, severity string) string {
+func (r *Runner) templateLogMsg(id, name string, authors []string, templateSeverity severity.Severity) string {
 	// Display the message for the template
-	message := fmt.Sprintf("[%s] %s (%s)",
+	return fmt.Sprintf("[%s] %s (%s) [%s]",
 		r.colorizer.BrightBlue(id).String(),
 		r.colorizer.Bold(name).String(),
-		r.colorizer.BrightYellow(appendAtSignToAuthors(author)).String())
-	if severity != "" {
-		message += " [" + r.severityColors.Data[severity] + "]"
-	}
-	return message
+		r.colorizer.BrightYellow(appendAtSignToAuthors(authors)).String(),
+		r.addColor(templateSeverity))
 }
 
-// appendAtSignToAuthors appends @ before each author and returns final string
-func appendAtSignToAuthors(author string) string {
-	authors := strings.Split(author, ",")
+// appendAtSignToAuthors appends @ before each author and returns the final string
+func appendAtSignToAuthors(authors []string) string {
 	if len(authors) == 0 {
 		return "@none"
 	}
-	if len(authors) == 1 {
-		if !strings.HasPrefix(authors[0], "@") {
-			return fmt.Sprintf("@%s", authors[0])
-		}
-		return authors[0]
-	}
+
 	values := make([]string, 0, len(authors))
 	for _, k := range authors {
-		if !strings.HasPrefix(authors[0], "@") {
+		if !strings.HasPrefix(k, "@") {
 			values = append(values, fmt.Sprintf("@%s", k))
 		} else {
 			values = append(values, k)
@@ -71,15 +40,18 @@ func appendAtSignToAuthors(author string) string {
 }
 
 func (r *Runner) logAvailableTemplate(tplPath string) {
-	t, err := r.parseTemplateFile(tplPath)
+	t, err := parsers.ParseTemplate(tplPath)
 	if err != nil {
 		gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
 	} else {
-		gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
+		gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID,
+			types.ToString(t.Info.Name),
+			t.Info.Authors.ToSlice(),
+			t.Info.SeverityHolder.Severity))
 	}
 }
 
-// ListAvailableTemplates prints available templates to stdout
+// listAvailableTemplates prints available templates to stdout
 func (r *Runner) listAvailableTemplates() {
 	if r.templatesConfig == nil {
 		return
@@ -92,7 +64,7 @@ func (r *Runner) listAvailableTemplates() {
 
 	gologger.Print().Msgf(
 		"\nListing available v.%s nuclei templates for %s",
-		r.templatesConfig.CurrentVersion,
+		r.templatesConfig.TemplateVersion,
 		r.templatesConfig.TemplatesDirectory,
 	)
 	err := directoryWalker(

v2/internal/runner/templates_test.go

@@ -0,0 +1,25 @@
+package runner
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_appendAtSignToAuthors(t *testing.T) {
+	result := appendAtSignToAuthors([]string{"user1", "user2", "user3"})
+	assert.Equal(t, result, "@user1,@user2,@user3")
+}
+
+func Test_appendAtSignToMissingAuthors(t *testing.T) {
+	result := appendAtSignToAuthors([]string{})
+	assert.Equal(t, result, "@none")
+
+	result = appendAtSignToAuthors(nil)
+	assert.Equal(t, result, "@none")
+}
+
+func Test_appendAtSignToOneAuthor(t *testing.T) {
+	result := appendAtSignToAuthors([]string{"user1"})
+	assert.Equal(t, result, "@user1")
+}

v2/internal/runner/update.go

@@ -7,26 +7,25 @@ import (
 	"context"
 	"crypto/md5"
 	"encoding/hex"
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
-	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/apex/log"
 	"github.com/blang/semver"
 	"github.com/google/go-github/github"
 	"github.com/olekukonko/tablewriter"
 	"github.com/pkg/errors"
+
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei-updatecheck-api/client"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 
 	"github.com/tj/go-update"
@@ -35,140 +34,95 @@ import (
 )
 
 const (
-	userName = "projectdiscovery"
-	repoName = "nuclei-templates"
+	userName             = "projectdiscovery"
+	repoName             = "nuclei-templates"
+	nucleiIgnoreFile     = ".nuclei-ignore"
+	nucleiConfigFilename = ".templates-config.json"
 )
 
-const nucleiIgnoreFile = ".nuclei-ignore"
-
-// nucleiConfigFilename is the filename of nuclei configuration file.
-const nucleiConfigFilename = ".templates-config.json"
-
 var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
 
 // updateTemplates checks if the default list of nuclei-templates
-// exist in the users home directory, if not the latest revision
-// is downloaded from github.
+// exist in the user's home directory, if not the latest revision
+// is downloaded from GitHub.
 //
-// If the path exists but is not latest, the new version is downloaded
-// from github and replaced with the templates directory.
+// If the path exists but does not contain the latest version of public templates,
+// the new version is downloaded from GitHub to the templates' directory, overwriting the old content.
 func (r *Runner) updateTemplates() error {
 	home, err := os.UserHomeDir()
 	if err != nil {
 		return err
 	}
-	configDir := path.Join(home, "/.config", "/nuclei")
+	configDir := filepath.Join(home, ".config", "nuclei")
 	_ = os.MkdirAll(configDir, os.ModePerm)
 
-	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
-	if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
-		configuration, readErr := config.ReadConfiguration()
-		if err != nil {
-			return readErr
-		}
-		r.templatesConfig = configuration
+	if err := r.readInternalConfigurationFile(home, configDir); err != nil {
+		return errors.Wrap(err, "could not read configuration file")
 	}
 
-	ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
+	// If the config doesn't exist, create it now.
 	if r.templatesConfig == nil {
 		currentConfig := &config.Config{
-			TemplatesDirectory: path.Join(home, "nuclei-templates"),
-			IgnoreURL:          ignoreURL,
+			TemplatesDirectory: filepath.Join(home, "nuclei-templates"),
 			NucleiVersion:      config.Version,
 		}
-		if writeErr := config.WriteConfiguration(currentConfig, false, false); writeErr != nil {
+		if writeErr := config.WriteConfiguration(currentConfig); writeErr != nil {
 			return errors.Wrap(writeErr, "could not write template configuration")
 		}
 		r.templatesConfig = currentConfig
 	}
 
-	if r.options.NoUpdateTemplates {
+	if r.options.NoUpdateTemplates && !r.options.UpdateTemplates {
 		return nil
 	}
-	// Check if last checked for nuclei-ignore is more than 1 hours.
-	// and if true, run the check.
-	//
-	// Also at the same time fetch latest version from github to do outdated nuclei
-	// and templates check.
-	checkedIgnore := false
-	if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
-		r.fetchLatestVersionsFromGithub()
-
-		if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" {
-			ignoreURL = r.templatesConfig.IgnoreURL
-		}
-		gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
-
-		checkedIgnore = true
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
-		if reqErr == nil {
-			resp, httpGet := http.DefaultClient.Do(req)
-			if httpGet != nil {
-				if resp != nil && resp.Body != nil {
-					resp.Body.Close()
-				}
-				gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
-			} else {
-				data, _ := ioutil.ReadAll(resp.Body)
-				resp.Body.Close()
-
-				if len(data) > 0 {
-					_ = ioutil.WriteFile(path.Join(configDir, nucleiIgnoreFile), data, 0644)
-				}
-				if r.templatesConfig != nil {
-					err = config.WriteConfiguration(r.templatesConfig, false, true)
-					if err != nil {
-						gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
-					}
-				}
-			}
-		}
-		cancel()
-	}
+	client.InitNucleiVersion(config.Version)
+	r.fetchLatestVersionsFromGithub(configDir) // also fetch the latest versions
 
 	ctx := context.Background()
-	if r.templatesConfig.CurrentVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
+
+	var noTemplatesFound bool
+	if _, err := os.Stat(r.templatesConfig.TemplatesDirectory); os.IsNotExist(err) {
+		noTemplatesFound = true
+	}
+
+	if r.templatesConfig.TemplateVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) || noTemplatesFound {
 		gologger.Info().Msgf("nuclei-templates are not installed, installing...\n")
 
-		// Use custom location if user has given a template directory
+		// Use the custom location if the user has given a template directory
 		r.templatesConfig = &config.Config{
-			TemplatesDirectory: path.Join(home, "nuclei-templates"),
+			TemplatesDirectory: filepath.Join(home, "nuclei-templates"),
 		}
-		if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") {
-			r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
+		if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != filepath.Join(home, "nuclei-templates") {
+			r.templatesConfig.TemplatesDirectory, _ = filepath.Abs(r.options.TemplatesDirectory)
+		}
+		r.fetchLatestVersionsFromGithub(configDir) // also fetch the latest versions
+
+		version, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion)
+		if err != nil {
+			return err
 		}
 
-		// Download the repository and also write the revision to a HEAD file.
-		version, asset, getErr := r.getLatestReleaseFromGithub()
+		// Download the repository and write the revision to a HEAD file.
+		asset, getErr := r.getLatestReleaseFromGithub(r.templatesConfig.NucleiTemplatesLatestVersion)
 		if getErr != nil {
 			return getErr
 		}
 		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
 
-		r.fetchLatestVersionsFromGithub() // also fetch latest versions
-		_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
-		if err != nil {
+		if _, err := r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()); err != nil {
 			return err
 		}
-		r.templatesConfig.CurrentVersion = version.String()
+		r.templatesConfig.TemplateVersion = version.String()
 
-		err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
-		if err != nil {
+		if err := config.WriteConfiguration(r.templatesConfig); err != nil {
 			return err
 		}
 		gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). GoodLuck!\n", version.String())
 		return nil
 	}
 
-	// Check if last checked is more than 24 hours and we don't have updateTemplates flag.
-	// If not, return since we don't want to do anything now.
-	if time.Since(r.templatesConfig.LastChecked) < 24*time.Hour && !r.options.UpdateTemplates {
-		return nil
-	}
-
 	// Get the configuration currently on disk.
-	verText := r.templatesConfig.CurrentVersion
+	verText := r.templatesConfig.TemplateVersion
 	indices := reVersion.FindStringIndex(verText)
 	if indices == nil {
 		return fmt.Errorf("invalid release found with tag %s", err)
@@ -182,13 +136,16 @@ func (r *Runner) updateTemplates() error {
 		return err
 	}
 
-	version, asset, err := r.getLatestReleaseFromGithub()
+	version, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion)
 	if err != nil {
 		return err
 	}
 
 	if version.EQ(oldVersion) {
-		return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore)
+		if r.options.UpdateTemplates {
+			gologger.Info().Msgf("No new updates found for nuclei templates")
+		}
+		return config.WriteConfiguration(r.templatesConfig)
 	}
 
 	if version.GT(oldVersion) {
@@ -198,16 +155,18 @@ func (r *Runner) updateTemplates() error {
 		if r.options.TemplatesDirectory != "" {
 			r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
 		}
-		r.templatesConfig.CurrentVersion = version.String()
+		r.templatesConfig.TemplateVersion = version.String()
 
 		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
-		r.fetchLatestVersionsFromGithub()
-		_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
+
+		asset, err := r.getLatestReleaseFromGithub(r.templatesConfig.NucleiTemplatesLatestVersion)
 		if err != nil {
 			return err
 		}
-		err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
-		if err != nil {
+		if _, err := r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()); err != nil {
+			return err
+		}
+		if err := config.WriteConfiguration(r.templatesConfig); err != nil {
 			return err
 		}
 		gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", version.String())
@@ -215,42 +174,52 @@ func (r *Runner) updateTemplates() error {
 	return nil
 }
 
-// getLatestReleaseFromGithub returns the latest release from github
-func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.RepositoryRelease, error) {
+// readInternalConfigurationFile reads the internal configuration file for nuclei
+func (r *Runner) readInternalConfigurationFile(home, configDir string) error {
+	templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename)
+	if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
+		configuration, readErr := config.ReadConfiguration()
+		if readErr != nil {
+			return readErr
+		}
+		r.templatesConfig = configuration
+
+		if configuration.TemplatesDirectory != "" && configuration.TemplatesDirectory != filepath.Join(home, "nuclei-templates") {
+			r.options.TemplatesDirectory = configuration.TemplatesDirectory
+		}
+	}
+	return nil
+}
+
+// checkNucleiIgnoreFileUpdates checks .nuclei-ignore file for updates from GitHub
+func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool {
+	data, err := client.GetLatestIgnoreFile()
+	if err != nil {
+		return false
+	}
+	if len(data) > 0 {
+		_ = ioutil.WriteFile(filepath.Join(configDir, nucleiIgnoreFile), data, 0644)
+	}
+	if r.templatesConfig != nil {
+		if err := config.WriteConfiguration(r.templatesConfig); err != nil {
+			gologger.Warning().Msgf("Could not get ignore-file from server: %s", err)
+		}
+	}
+	return true
+}
+
+// getLatestReleaseFromGithub returns the latest release from GitHub
+func (r *Runner) getLatestReleaseFromGithub(latestTag string) (*github.RepositoryRelease, error) {
 	client := github.NewClient(nil)
 
-	rels, _, err := client.Repositories.ListReleases(context.Background(), userName, repoName, nil)
+	release, _, err := client.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag)
 	if err != nil {
-		return semver.Version{}, nil, err
+		return nil, err
 	}
-
-	// Find the most recent version based on semantic versioning.
-	var latestRelease semver.Version
-	var latestPublish *github.RepositoryRelease
-	for _, release := range rels {
-		verText := release.GetTagName()
-		indices := reVersion.FindStringIndex(verText)
-		if indices == nil {
-			return semver.Version{}, nil, fmt.Errorf("invalid release found with tag %s", err)
-		}
-		if indices[0] > 0 {
-			verText = verText[indices[0]:]
-		}
-
-		ver, err := semver.Make(verText)
-		if err != nil {
-			return semver.Version{}, nil, err
-		}
-
-		if latestPublish == nil || ver.GTE(latestRelease) {
-			latestRelease = ver
-			latestPublish = release
-		}
+	if release == nil {
+		return nil, errors.New("no version found for the templates")
 	}
-	if latestPublish == nil {
-		return semver.Version{}, nil, errors.New("no version found for the templates")
-	}
-	return latestRelease, latestPublish, nil
+	return release, nil
 }
 
 // downloadReleaseAndUnzip downloads and unzips the release in a directory
@@ -275,18 +244,17 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU
 	}
 
 	reader := bytes.NewReader(buf)
-	z, err := zip.NewReader(reader, reader.Size())
+	zipReader, err := zip.NewReader(reader, reader.Size())
 	if err != nil {
 		return nil, fmt.Errorf("failed to uncompress zip file: %s", err)
 	}
 
-	// Create the template folder if it doesn't exists
-	err = os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm)
-	if err != nil {
+	// Create the template folder if it doesn't exist
+	if err := os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm); err != nil {
 		return nil, fmt.Errorf("failed to create template base folder: %s", err)
 	}
 
-	results, err := r.compareAndWriteTemplates(z)
+	results, err := r.compareAndWriteTemplates(zipReader)
 	if err != nil {
 		return nil, fmt.Errorf("failed to write templates: %s", err)
 	}
@@ -294,21 +262,20 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU
 	if r.options.Verbose {
 		r.printUpdateChangelog(results, version)
 	}
-	checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
-	err = writeTemplatesChecksum(checksumFile, results.checksums)
-	if err != nil {
+	checksumFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
+	if err := writeTemplatesChecksum(checksumFile, results.checksums); err != nil {
 		return nil, errors.Wrap(err, "could not write checksum")
 	}
 
 	// Write the additions to a cached file for new runs.
-	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	additionsFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
 	buffer := &bytes.Buffer{}
 	for _, addition := range results.additions {
 		buffer.WriteString(addition)
 		buffer.WriteString("\n")
 	}
-	err = ioutil.WriteFile(additionsFile, buffer.Bytes(), os.ModePerm)
-	if err != nil {
+
+	if err := ioutil.WriteFile(additionsFile, buffer.Bytes(), os.ModePerm); err != nil {
 		return nil, errors.Wrap(err, "could not write new additions file")
 	}
 	return results, err
@@ -322,9 +289,8 @@ type templateUpdateResults struct {
 	checksums     map[string]string
 }
 
-// compareAndWriteTemplates compares and returns the stats of a template
-// update operations.
-func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults, error) {
+// compareAndWriteTemplates compares and returns the stats of a template update operations.
+func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdateResults, error) {
 	results := &templateUpdateResults{
 		checksums: make(map[string]string),
 	}
@@ -334,115 +300,114 @@ func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults
 	// If the path isn't found in new update after being read from the previous checksum,
 	// it is removed. This allows us fine-grained control over the download process
 	// as well as solves a long problem with nuclei-template updates.
-	checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
-	previousChecksum, _ := readPreviousTemplatesChecksum(checksumFile)
-	for _, file := range z.File {
-		directory, name := filepath.Split(file.Name)
+	checksumFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
+	templateChecksumsMap, _ := createTemplateChecksumsMap(checksumFile)
+	for _, zipTemplateFile := range zipReader.File {
+		directory, name := filepath.Split(zipTemplateFile.Name)
 		if name == "" {
 			continue
 		}
-		paths := strings.Split(directory, "/")
-		finalPath := strings.Join(paths[1:], "/")
+		paths := strings.Split(directory, string(os.PathSeparator))
+		finalPath := filepath.Join(paths[1:]...)
 
 		if strings.HasPrefix(name, ".") || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") {
 			continue
 		}
 		results.totalCount++
-		templateDirectory := path.Join(r.templatesConfig.TemplatesDirectory, finalPath)
-		err := os.MkdirAll(templateDirectory, os.ModePerm)
-		if err != nil {
+		templateDirectory := filepath.Join(r.templatesConfig.TemplatesDirectory, finalPath)
+		if err := os.MkdirAll(templateDirectory, os.ModePerm); err != nil {
 			return nil, fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err)
 		}
 
-		templatePath := path.Join(templateDirectory, name)
+		templatePath := filepath.Join(templateDirectory, name)
 
 		isAddition := false
 		if _, statErr := os.Stat(templatePath); os.IsNotExist(statErr) {
 			isAddition = true
 		}
-		f, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777)
+		templateFile, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777)
 		if err != nil {
-			f.Close()
+			templateFile.Close()
 			return nil, fmt.Errorf("could not create uncompressed file: %s", err)
 		}
 
-		reader, err := file.Open()
+		zipTemplateFileReader, err := zipTemplateFile.Open()
 		if err != nil {
-			f.Close()
+			templateFile.Close()
 			return nil, fmt.Errorf("could not open archive to extract file: %s", err)
 		}
 		hasher := md5.New()
 
 		// Save file and also read into hasher for md5
-		_, err = io.Copy(f, io.TeeReader(reader, hasher))
-		if err != nil {
-			f.Close()
+		if _, err := io.Copy(templateFile, io.TeeReader(zipTemplateFileReader, hasher)); err != nil {
+			templateFile.Close()
 			return nil, fmt.Errorf("could not write template file: %s", err)
 		}
-		f.Close()
+		templateFile.Close()
 
-		oldChecksum, checksumOK := previousChecksum[templatePath]
+		oldChecksum, checksumOK := templateChecksumsMap[templatePath]
 
 		checksum := hex.EncodeToString(hasher.Sum(nil))
 		if isAddition {
-			results.additions = append(results.additions, path.Join(finalPath, name))
+			results.additions = append(results.additions, filepath.Join(finalPath, name))
 		} else if checksumOK && oldChecksum[0] != checksum {
-			results.modifications = append(results.modifications, path.Join(finalPath, name))
+			results.modifications = append(results.modifications, filepath.Join(finalPath, name))
 		}
 		results.checksums[templatePath] = checksum
 	}
 
-	// If we don't find a previous file in new download and it hasn't been
-	// changed on the disk, delete it.
-	for k, v := range previousChecksum {
-		_, ok := results.checksums[k]
-		if !ok && v[0] == v[1] {
-			os.Remove(k)
-			results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(k, r.templatesConfig.TemplatesDirectory), "/"))
+	// If we don't find the previous file in the newly downloaded list,
+	// and it hasn't been changed on the disk, delete it.
+	for templatePath, templateChecksums := range templateChecksumsMap {
+		_, ok := results.checksums[templatePath]
+		if !ok && templateChecksums[0] == templateChecksums[1] {
+			os.Remove(templatePath)
+			results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(templatePath, r.templatesConfig.TemplatesDirectory), string(os.PathSeparator)))
 		}
 	}
 	return results, nil
 }
 
-// readPreviousTemplatesChecksum reads the previous checksum file from the disk.
-//
-// It reads two checksums, the first checksum is what we expect and the second is
-// the actual checksum of the file on disk currently.
-func readPreviousTemplatesChecksum(file string) (map[string][2]string, error) {
-	f, err := os.Open(file)
+// createTemplateChecksumsMap reads the previous checksum file from the disk.
+// Creates a map of template paths and their previous and currently calculated checksums as values.
+func createTemplateChecksumsMap(checksumsFilePath string) (map[string][2]string, error) {
+	checksumFile, err := os.Open(checksumsFilePath)
 	if err != nil {
 		return nil, err
 	}
-	defer f.Close()
-	scanner := bufio.NewScanner(f)
+	defer checksumFile.Close()
+	scanner := bufio.NewScanner(checksumFile)
 
-	checksum := make(map[string][2]string)
+	templatePathChecksumsMap := make(map[string][2]string)
 	for scanner.Scan() {
 		text := scanner.Text()
 		if text == "" {
 			continue
 		}
+
 		parts := strings.Split(text, ",")
 		if len(parts) < 2 {
 			continue
 		}
-		values := [2]string{parts[1]}
+		templatePath := parts[0]
+		expectedTemplateChecksum := parts[1]
 
-		f, err := os.Open(parts[0])
+		templateFile, err := os.Open(templatePath)
 		if err != nil {
 			return nil, err
 		}
 
 		hasher := md5.New()
-		if _, err := io.Copy(hasher, f); err != nil {
+		if _, err := io.Copy(hasher, templateFile); err != nil {
 			return nil, err
 		}
-		f.Close()
+		templateFile.Close()
 
+		values := [2]string{expectedTemplateChecksum}
 		values[1] = hex.EncodeToString(hasher.Sum(nil))
-		checksum[parts[0]] = values
+		templatePathChecksumsMap[templatePath] = values
 	}
-	return checksum, nil
+	return templatePathChecksumsMap, nil
 }
 
 // writeTemplatesChecksum writes the nuclei-templates checksum data to disk.
@@ -489,60 +454,29 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st
 	table.Render()
 }
 
-// fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github
-func (r *Runner) fetchLatestVersionsFromGithub() {
-	nucleiLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei")
+// fetchLatestVersionsFromGithub fetches the latest versions of nuclei repos from GitHub
+//
+// This fetches latest nuclei/templates/ignore from https://version-check.nuclei.sh/versions
+// If you want to disable this automatic update check, use -nut flag.
+func (r *Runner) fetchLatestVersionsFromGithub(configDir string) {
+	versions, err := client.GetLatestNucleiTemplatesVersion()
 	if err != nil {
-		gologger.Warning().Msgf("Could not fetch latest nuclei release: %s", err)
-	}
-	templatesLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei-templates")
-	if err != nil {
-		gologger.Warning().Msgf("Could not fetch latest nuclei-templates release: %s", err)
+		gologger.Warning().Msgf("Could not fetch latest releases: %s", err)
+		return
 	}
 	if r.templatesConfig != nil {
-		r.templatesConfig.NucleiLatestVersion = nucleiLatest
-		r.templatesConfig.NucleiTemplatesLatestVersion = templatesLatest
+		r.templatesConfig.NucleiLatestVersion = versions.Nuclei
+		r.templatesConfig.NucleiTemplatesLatestVersion = versions.Templates
+
+		// If the fetch has resulted in new version of ignore file, update.
+		if r.templatesConfig.NucleiIgnoreHash == "" || r.templatesConfig.NucleiIgnoreHash != versions.IgnoreHash {
+			r.templatesConfig.NucleiIgnoreHash = versions.IgnoreHash
+			r.checkNucleiIgnoreFileUpdates(configDir)
+		}
 	}
 }
 
-type githubTagData struct {
-	Name string
-}
-
-// githubFetchLatestTagRepo fetches latest tag from github
-// This function was half written by github copilot AI :D.
-func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	url := fmt.Sprintf("https://api.github.com/repos/%s/tags", repo)
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
-	if err != nil {
-		return "", err
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-
-	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return "", err
-	}
-
-	var tags []githubTagData
-	err = json.Unmarshal(body, &tags)
-	if err != nil {
-		return "", err
-	}
-	if len(tags) == 0 {
-		return "", fmt.Errorf("no tags found for %s", repo)
-	}
-	return strings.TrimPrefix(tags[0].Name, "v"), nil
-}
-
-// updateNucleiVersionToLatest implements nuclei auto-updation using Github Releases.
+// updateNucleiVersionToLatest implements nuclei auto-update using GitHub Releases.
 func updateNucleiVersionToLatest(verbose bool) error {
 	if verbose {
 		log.SetLevel(log.DebugLevel)

v2/internal/runner/update_test.go

@@ -8,7 +8,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -26,7 +25,7 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
 	require.Nil(t, err, "could not create temp directory")
 	defer os.RemoveAll(baseTemplates)
 
-	err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
+	err = ioutil.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
 	require.Nil(t, err, "could not create write base file")
 
 	err = zipFromDirectory("base.zip", baseTemplates)
@@ -51,9 +50,9 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
 	require.Nil(t, err, "could not create temp directory")
 	defer os.RemoveAll(newTempDir)
 
-	err = ioutil.WriteFile(path.Join(newTempDir, "base.yaml"), []byte("id: test"), 0777)
+	err = ioutil.WriteFile(filepath.Join(newTempDir, "base.yaml"), []byte("id: test"), 0777)
 	require.Nil(t, err, "could not create base file")
-	err = ioutil.WriteFile(path.Join(newTempDir, "new.yaml"), []byte("id: test"), 0777)
+	err = ioutil.WriteFile(filepath.Join(newTempDir, "new.yaml"), []byte("id: test"), 0777)
 	require.Nil(t, err, "could not create new file")
 
 	err = zipFromDirectory("new.zip", newTempDir)
@@ -78,7 +77,7 @@ func TestDownloadReleaseAndUnzipDeletion(t *testing.T) {
 	require.Nil(t, err, "could not create temp directory")
 	defer os.RemoveAll(baseTemplates)
 
-	err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
+	err = ioutil.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
 	require.Nil(t, err, "could not create write base file")
 
 	err = zipFromDirectory("base.zip", baseTemplates)

v2/internal/testutils/integration.go

@@ -1,9 +1,11 @@
 package testutils
 
 import (
+	"errors"
 	"net"
 	"os"
 	"os/exec"
+	"regexp"
 	"strings"
 )
 
@@ -30,6 +32,23 @@ func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) (
 	return parts, nil
 }
 
+var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded[^:]*: (\d+)`)
+
+// RunNucleiBinaryAndGetLoadedTemplates returns a list of results for a template
+func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, args []string) (string, error) {
+	cmd := exec.Command(nucleiBinary, args...)
+
+	data, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", err
+	}
+	matches := templateLoaded.FindAllStringSubmatch(string(data), -1)
+	if len(matches) == 0 {
+		return "", errors.New("no matches found")
+	}
+	return matches[0][1], nil
+}
+
 // RunNucleiWorkflowAndGetResults returns a list of results for a workflow
 func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
 	cmd := exec.Command("./nuclei", "-w", template, "-target", url, "-silent")

v2/internal/testutils/testutils.go

@@ -2,14 +2,17 @@ package testutils
 
 import (
 	"github.com/logrusorgru/aurora"
+	"go.uber.org/ratelimit"
+
 	"github.com/projectdiscovery/gologger/levels"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
-	"go.uber.org/ratelimit"
 )
 
 // Init initializes the protocols and their configurations
@@ -44,9 +47,9 @@ var DefaultOptions = &types.Options{
 	Retries:            1,
 	RateLimit:          150,
 	ProjectPath:        "",
-	Severity:           []string{},
-	Target:             "",
-	Targets:            "",
+	Severities:         severity.Severities{},
+	Targets:            []string{},
+	TargetsFilePath:    "",
 	Output:             "",
 	ProxyURL:           "",
 	ProxySocksURL:      "",
@@ -95,7 +98,7 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro
 // TemplateInfo contains info for a mock executed template.
 type TemplateInfo struct {
 	ID   string
-	Info map[string]interface{}
+	Info model.Info
 	Path string
 }
 

v2/pkg/catalog/config/config.go

@@ -2,8 +2,7 @@ package config
 
 import (
 	"os"
-	"path"
-	"time"
+	"path/filepath"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/pkg/errors"
@@ -13,12 +12,10 @@ import (
 
 // Config contains the internal nuclei engine configuration
 type Config struct {
-	TemplatesDirectory string    `json:"templates-directory,omitempty"`
-	CurrentVersion     string    `json:"current-version,omitempty"`
-	LastChecked        time.Time `json:"last-checked,omitempty"`
-	IgnoreURL          string    `json:"ignore-url,omitempty"`
-	NucleiVersion      string    `json:"nuclei-version,omitempty"`
-	LastCheckedIgnore  time.Time `json:"last-checked-ignore,omitempty"`
+	TemplatesDirectory string `json:"nuclei-templates-directory,omitempty"`
+	TemplateVersion    string `json:"nuclei-templates-version,omitempty"`
+	NucleiVersion      string `json:"nuclei-version,omitempty"`
+	NucleiIgnoreHash   string `json:"nuclei-ignore-hash,omitempty"`
 
 	NucleiLatestVersion          string `json:"nuclei-latest-version"`
 	NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"`
@@ -28,16 +25,16 @@ type Config struct {
 const nucleiConfigFilename = ".templates-config.json"
 
 // Version is the current version of nuclei
-const Version = `2.4.2`
+const Version = `2.5.2`
 
 func getConfigDetails() (string, error) {
 	homeDir, err := os.UserHomeDir()
 	if err != nil {
 		return "", errors.Wrap(err, "could not get home directory")
 	}
-	configDir := path.Join(homeDir, "/.config", "/nuclei")
+	configDir := filepath.Join(homeDir, ".config", "nuclei")
 	_ = os.MkdirAll(configDir, os.ModePerm)
-	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
+	templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename)
 	return templatesConfigFile, nil
 }
 
@@ -55,24 +52,14 @@ func ReadConfiguration() (*Config, error) {
 	defer file.Close()
 
 	config := &Config{}
-	err = jsoniter.NewDecoder(file).Decode(config)
-	if err != nil {
+	if err := jsoniter.NewDecoder(file).Decode(config); err != nil {
 		return nil, err
 	}
 	return config, nil
 }
 
 // WriteConfiguration writes the updated nuclei configuration to disk
-func WriteConfiguration(config *Config, checked, checkedIgnore bool) error {
-	if config.IgnoreURL == "" {
-		config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
-	}
-	if checked {
-		config.LastChecked = time.Now()
-	}
-	if checkedIgnore {
-		config.LastCheckedIgnore = time.Now()
-	}
+func WriteConfiguration(config *Config) error {
 	config.NucleiVersion = Version
 
 	templatesConfigFile, err := getConfigDetails()
@@ -123,16 +110,16 @@ func getIgnoreFilePath() string {
 
 	home, err := os.UserHomeDir()
 	if err == nil {
-		configDir := path.Join(home, "/.config", "/nuclei")
+		configDir := filepath.Join(home, ".config", "nuclei")
 		_ = os.MkdirAll(configDir, os.ModePerm)
 
-		defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile)
+		defIgnoreFilePath = filepath.Join(configDir, nucleiIgnoreFile)
 		return defIgnoreFilePath
 	}
 	cwd, err := os.Getwd()
 	if err != nil {
 		return defIgnoreFilePath
 	}
-	cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile)
+	cwdIgnoreFilePath := filepath.Join(cwd, nucleiIgnoreFile)
 	return cwdIgnoreFilePath
 }

v2/pkg/catalog/find.go

@@ -2,7 +2,6 @@ package catalog
 
 import (
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 
@@ -83,12 +82,12 @@ func (c *Catalog) GetTemplatePath(target string) ([]string, error) {
 // before doing any operations on them regardless of them being blob, folders, files, etc.
 func (c *Catalog) convertPathToAbsolute(t string) (string, error) {
 	if strings.Contains(t, "*") {
-		file := path.Base(t)
-		absPath, err := c.ResolvePath(path.Dir(t), "")
+		file := filepath.Base(t)
+		absPath, err := c.ResolvePath(filepath.Dir(t), "")
 		if err != nil {
 			return "", err
 		}
-		return path.Join(absPath, file), nil
+		return filepath.Join(absPath, file), nil
 	}
 	return c.ResolvePath(t, "")
 }

v2/pkg/catalog/loader/filter/tag_filter.go

@@ -3,114 +3,117 @@ package filter
 import (
 	"errors"
 	"strings"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 )
 
 // TagFilter is used to filter nuclei templates for tag based execution
 type TagFilter struct {
 	allowedTags map[string]struct{}
-	severities  map[string]struct{}
+	severities  map[severity.Severity]struct{}
 	authors     map[string]struct{}
 	block       map[string]struct{}
 	matchAllows map[string]struct{}
 }
 
-// ErrExcluded is returned for execluded templates
+// ErrExcluded is returned for excluded templates
 var ErrExcluded = errors.New("the template was excluded")
 
-// Match takes a tag and whether the template was matched from user
-// input and returns true or false using a tag filter.
-//
-// If the tag was specified in deny list, it will not return true
-// unless it is explicitly specified by user in includeTags which is the
-// matchAllows section.
-//
-// It returns true if the tag is specified, or false.
-func (t *TagFilter) Match(tag, author, severity string) (bool, error) {
-	matchedAny := false
-	if len(t.allowedTags) > 0 {
-		_, ok := t.allowedTags[tag]
-		if ok {
-			matchedAny = true
+// Match filters templates based on user provided tags, authors, extraTags and severity.
+// If the template contains tags specified in the deny-list, it will not be matched
+// unless it is explicitly specified by user using the includeTags (matchAllows field).
+// Matching rule: (tag1 OR tag2...) AND (author1 OR author2...) AND (severity1 OR severity2...) AND (extraTags1 OR extraTags2...)
+// Returns true if the template matches the filter criteria, false otherwise.
+func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string) (bool, error) {
+	for _, templateTag := range templateTags {
+		_, blocked := tagFilter.block[templateTag]
+		_, allowed := tagFilter.matchAllows[templateTag]
+
+		if blocked && !allowed { // the whitelist has precedence over the blacklist
+			return false, ErrExcluded
 		}
 	}
-	_, ok := t.block[tag]
-	if ok {
-		if _, allowOk := t.matchAllows[tag]; allowOk {
-			return true, nil
-		}
-		return false, ErrExcluded
+
+	if !isExtraTagMatch(extraTags, templateTags) {
+		return false, nil
 	}
-	if len(t.authors) > 0 {
-		_, ok = t.authors[author]
-		if !ok {
-			return false, nil
-		}
-		matchedAny = true
+
+	if !isTagMatch(tagFilter, templateTags) {
+		return false, nil
 	}
-	if len(t.severities) > 0 {
-		_, ok = t.severities[severity]
-		if !ok {
-			return false, nil
-		}
-		matchedAny = true
+
+	if !isAuthorMatch(tagFilter, templateAuthors) {
+		return false, nil
 	}
-	if len(t.allowedTags) == 0 && len(t.authors) == 0 && len(t.severities) == 0 {
-		return true, nil
+
+	if !isSeverityMatch(tagFilter, templateSeverity) {
+		return false, nil
 	}
-	return matchedAny, nil
+
+	return true, nil
 }
 
-// MatchWithAllowedTags takes an addition list of allowed tags
-// and returns true if the match was successful.
-func (t *TagFilter) MatchWithAllowedTags(allowed []string, tag, author, severity string) (bool, error) {
-	matchedAny := false
+func isSeverityMatch(tagFilter *TagFilter, templateSeverity severity.Severity) bool {
+	if len(tagFilter.severities) == 0 || templateSeverity == severity.Undefined {
+		return true
+	}
 
-	allowedMap := make(map[string]struct{})
-	for _, tag := range allowed {
-		for _, val := range splitCommaTrim(tag) {
-			if _, ok := allowedMap[val]; !ok {
-				allowedMap[val] = struct{}{}
-			}
+	if _, ok := tagFilter.severities[templateSeverity]; ok {
+		return true
+	}
+
+	return false
+}
+
+func isAuthorMatch(tagFilter *TagFilter, templateAuthors []string) bool {
+	if len(tagFilter.authors) == 0 {
+		return true
+	}
+
+	templateAuthorMap := toMap(templateAuthors)
+	for requiredAuthor := range tagFilter.authors {
+		if _, ok := templateAuthorMap[requiredAuthor]; ok {
+			return true
 		}
 	}
-	if len(allowedMap) > 0 {
-		_, ok := allowedMap[tag]
-		if ok {
-			matchedAny = true
+
+	return false
+}
+
+func isExtraTagMatch(extraTags []string, templateTags []string) bool {
+	if len(extraTags) == 0 {
+		return true
+	}
+
+	templatesTagMap := toMap(templateTags)
+	for _, extraTag := range extraTags {
+		if _, ok := templatesTagMap[extraTag]; ok {
+			return true
 		}
 	}
-	_, ok := t.block[tag]
-	if ok && !matchedAny {
-		if _, allowOk := t.matchAllows[tag]; allowOk {
-			return true, nil
+
+	return false
+}
+
+func isTagMatch(tagFilter *TagFilter, templateTags []string) bool {
+	if len(tagFilter.allowedTags) == 0 {
+		return true
+	}
+
+	for _, templateTag := range templateTags {
+		if _, ok := tagFilter.allowedTags[templateTag]; ok {
+			return true
 		}
-		return false, ErrExcluded
 	}
-	if len(t.authors) > 0 {
-		_, ok = t.authors[author]
-		if !ok {
-			return false, nil
-		}
-		matchedAny = true
-	}
-	if len(t.severities) > 0 {
-		_, ok = t.severities[severity]
-		if !ok {
-			return false, nil
-		}
-		matchedAny = true
-	}
-	if len(allowedMap) == 0 && len(t.authors) == 0 && len(t.severities) == 0 {
-		return true, nil
-	}
-	return matchedAny, nil
+
+	return false
 }
 
 type Config struct {
 	Tags        []string
 	ExcludeTags []string
 	Authors     []string
-	Severities  []string
+	Severities  severity.Severities
 	IncludeTags []string
 }
 
@@ -121,7 +124,7 @@ func New(config *Config) *TagFilter {
 	filter := &TagFilter{
 		allowedTags: make(map[string]struct{}),
 		authors:     make(map[string]struct{}),
-		severities:  make(map[string]struct{}),
+		severities:  make(map[severity.Severity]struct{}),
 		block:       make(map[string]struct{}),
 		matchAllows: make(map[string]struct{}),
 	}
@@ -133,10 +136,8 @@ func New(config *Config) *TagFilter {
 		}
 	}
 	for _, tag := range config.Severities {
-		for _, val := range splitCommaTrim(tag) {
-			if _, ok := filter.severities[val]; !ok {
-				filter.severities[val] = struct{}{}
-			}
+		if _, ok := filter.severities[tag]; !ok {
+			filter.severities[tag] = struct{}{}
 		}
 	}
 	for _, tag := range config.Authors {
@@ -165,6 +166,11 @@ func New(config *Config) *TagFilter {
 	return filter
 }
 
+/*
+TODO similar logic is used over and over again. It should be extracted and reused
+Changing []string and string data types that hold string slices to StringSlice would be the preferred solution,
+which implicitly does the normalization before any other calls starting to use it.
+*/
 func splitCommaTrim(value string) []string {
 	if !strings.Contains(value, ",") {
 		return []string{strings.ToLower(value)}
@@ -176,3 +182,13 @@ func splitCommaTrim(value string) []string {
 	}
 	return final
 }
+
+func toMap(slice []string) map[string]struct{} {
+	result := make(map[string]struct{}, len(slice))
+	for _, value := range slice {
+		if _, ok := result[value]; !ok {
+			result[value] = struct{}{}
+		}
+	}
+	return result
+}

v2/pkg/catalog/loader/filter/tag_filter_test.go

@@ -4,81 +4,99 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 )
 
 func TestTagBasedFilter(t *testing.T) {
-	config := &Config{
-		Tags: []string{"cves", "2021", "jira"},
+	{
+		filter := New(&Config{
+			Tags: []string{"cves", "2021", "jira"},
+		})
+
+		t.Run("true", func(t *testing.T) {
+			matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil)
+			require.True(t, matched, "could not get correct match")
+		})
+		t.Run("false", func(t *testing.T) {
+			matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil)
+			require.False(t, matched, "could not get correct match")
+		})
+		t.Run("match-extra-tags-positive", func(t *testing.T) {
+			matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"})
+			require.True(t, matched, "could not get correct match")
+		})
+		t.Run("match-extra-tags-negative", func(t *testing.T) {
+			matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"})
+			require.False(t, matched, "could not get correct match")
+		})
 	}
-	filter := New(config)
 
-	t.Run("true", func(t *testing.T) {
-		matched, _ := filter.Match("jira", "pdteam", "low")
-		require.True(t, matched, "could not get correct match")
-	})
-	t.Run("false", func(t *testing.T) {
-		matched, _ := filter.Match("consul", "pdteam", "low")
-		require.False(t, matched, "could not get correct match")
-	})
 	t.Run("not-match-excludes", func(t *testing.T) {
-		config := &Config{
+		filter := New(&Config{
 			ExcludeTags: []string{"dos"},
-		}
-		filter := New(config)
-		matched, err := filter.Match("dos", "pdteam", "low")
+		})
+		matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil)
 		require.False(t, matched, "could not get correct match")
 		require.Equal(t, ErrExcluded, err, "could not get correct error")
 	})
 	t.Run("match-includes", func(t *testing.T) {
-		config := &Config{
+		filter := New(&Config{
 			Tags:        []string{"cves", "fuzz"},
 			ExcludeTags: []string{"dos", "fuzz"},
 			IncludeTags: []string{"fuzz"},
-		}
-		filter := New(config)
-		matched, err := filter.Match("fuzz", "pdteam", "low")
+		})
+		matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
+		require.Nil(t, err, "could not get match")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("match-includes", func(t *testing.T) {
+		filter := New(&Config{
+			Tags:        []string{"fuzz"},
+			ExcludeTags: []string{"fuzz"},
+		})
+		matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
 		require.Nil(t, err, "could not get match")
 		require.True(t, matched, "could not get correct match")
 	})
 	t.Run("match-author", func(t *testing.T) {
-		config := &Config{
+		filter := New(&Config{
 			Authors: []string{"pdteam"},
-		}
-		filter := New(config)
-		matched, _ := filter.Match("fuzz", "pdteam", "low")
+		})
+		matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
 		require.True(t, matched, "could not get correct match")
 	})
 	t.Run("match-severity", func(t *testing.T) {
-		config := &Config{
-			Severities: []string{"high"},
-		}
-		filter := New(config)
-		matched, _ := filter.Match("fuzz", "pdteam", "high")
+		filter := New(&Config{
+			Severities: severity.Severities{severity.High},
+		})
+		matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil)
 		require.True(t, matched, "could not get correct match")
 	})
 	t.Run("match-exclude-with-tags", func(t *testing.T) {
-		config := &Config{
+		filter := New(&Config{
 			Tags:        []string{"tag"},
 			ExcludeTags: []string{"another"},
-		}
-		filter := New(config)
-		matched, _ := filter.Match("another", "pdteam", "high")
+		})
+		matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil)
 		require.False(t, matched, "could not get correct match")
 	})
 	t.Run("match-conditions", func(t *testing.T) {
-		config := &Config{
+		filter := New(&Config{
 			Authors:    []string{"pdteam"},
 			Tags:       []string{"jira"},
-			Severities: []string{"high"},
-		}
-		filter := New(config)
-		matched, _ := filter.Match("jira", "pdteam", "high")
+			Severities: severity.Severities{severity.High},
+		})
+		matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil)
 		require.True(t, matched, "could not get correct match")
-		matched, _ = filter.Match("jira", "pdteam", "low")
+
+		matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil)
 		require.False(t, matched, "could not get correct match")
-		matched, _ = filter.Match("jira", "random", "low")
+
+		matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil)
 		require.False(t, matched, "could not get correct match")
-		matched, _ = filter.Match("consul", "random", "low")
+
+		matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil)
 		require.False(t, matched, "could not get correct match")
 	})
 }

v2/pkg/catalog/loader/load/load.go

@@ -1,100 +0,0 @@
-package load
-
-import (
-	"bytes"
-	"errors"
-	"io/ioutil"
-	"os"
-	"strings"
-
-	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
-	"github.com/projectdiscovery/nuclei/v2/pkg/types"
-	"gopkg.in/yaml.v2"
-)
-
-// Load loads a template by parsing metadata and running
-// all tag and path based filters on the template.
-func Load(templatePath string, workflow bool, customTags []string, tagFilter *filter.TagFilter) (bool, error) {
-	f, err := os.Open(templatePath)
-	if err != nil {
-		return false, err
-	}
-	defer f.Close()
-
-	data, err := ioutil.ReadAll(f)
-	if err != nil {
-		return false, err
-	}
-
-	template := make(map[string]interface{})
-	err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
-	if err != nil {
-		return false, err
-	}
-
-	info, ok := template["info"]
-	if !ok {
-		return false, errors.New("no template info field provided")
-	}
-	infoMap, ok := info.(map[interface{}]interface{})
-	if !ok {
-		return false, errors.New("could not get info")
-	}
-
-	if _, nameOk := infoMap["name"]; !nameOk {
-		return false, errors.New("no template name field provided")
-	}
-	author, ok := infoMap["author"]
-	if !ok {
-		return false, errors.New("no template author field provided")
-	}
-	severity, ok := infoMap["severity"]
-	if !ok {
-		severity = ""
-	}
-
-	templateTags, ok := infoMap["tags"]
-	if !ok {
-		templateTags = ""
-	}
-	tagStr := types.ToString(templateTags)
-
-	tags := strings.Split(tagStr, ",")
-	severityStr := strings.ToLower(types.ToString(severity))
-	authors := strings.Split(types.ToString(author), ",")
-
-	matched := false
-
-	_, workflowsFound := template["workflows"]
-	if !workflowsFound && workflow {
-		return false, nil
-	}
-	if workflow {
-		return true, nil
-	}
-	for _, tag := range tags {
-		for _, author := range authors {
-			var match bool
-			var err error
-
-			if len(customTags) == 0 {
-				match, err = tagFilter.Match(strings.ToLower(strings.TrimSpace(tag)), strings.ToLower(strings.TrimSpace(author)), severityStr)
-			} else {
-				match, err = tagFilter.MatchWithAllowedTags(customTags, strings.ToLower(strings.TrimSpace(tag)), strings.ToLower(strings.TrimSpace(author)), severityStr)
-			}
-			if err == filter.ErrExcluded {
-				return false, filter.ErrExcluded
-			}
-			if !matched && match {
-				matched = true
-			}
-		}
-	}
-	if !matched {
-		return false, nil
-	}
-	if workflowsFound && !workflow {
-		return false, nil
-	}
-	return true, nil
-}

v2/pkg/catalog/loader/loader.go

@@ -1,12 +1,13 @@
 package loader
 
 import (
-	"strings"
+	"errors"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
-	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/load"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
 )
@@ -21,7 +22,7 @@ type Config struct {
 	Tags        []string
 	ExcludeTags []string
 	Authors     []string
-	Severities  []string
+	Severities  severity.Severities
 	IncludeTags []string
 
 	Catalog            *catalog.Catalog
@@ -69,103 +70,104 @@ func New(config *Config) (*Store, error) {
 }
 
 // Templates returns all the templates in the store
-func (s *Store) Templates() []*templates.Template {
-	return s.templates
+func (store *Store) Templates() []*templates.Template {
+	return store.templates
 }
 
 // Workflows returns all the workflows in the store
-func (s *Store) Workflows() []*templates.Template {
-	return s.workflows
+func (store *Store) Workflows() []*templates.Template {
+	return store.workflows
 }
 
 // RegisterPreprocessor allows a custom preprocessor to be passed to the store to run against templates
-func (s *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
-	s.preprocessor = preprocessor
+func (store *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
+	store.preprocessor = preprocessor
 }
 
 // Load loads all the templates from a store, performs filtering and returns
 // the complete compiled templates for a nuclei execution configuration.
-func (s *Store) Load() {
-	s.templates = s.LoadTemplates(s.finalTemplates)
-	s.workflows = s.LoadWorkflows(s.config.Workflows)
+func (store *Store) Load() {
+	store.templates = store.LoadTemplates(store.finalTemplates)
+	store.workflows = store.LoadWorkflows(store.config.Workflows)
 }
 
 // ValidateTemplates takes a list of templates and validates them
 // erroring out on discovering any faulty templates.
-func (s *Store) ValidateTemplates(templatesList, workflowsList []string) bool {
-	includedTemplates := s.config.Catalog.GetTemplatesPath(templatesList)
-	includedWorkflows := s.config.Catalog.GetTemplatesPath(workflowsList)
-	templatesMap := s.pathFilter.Match(includedTemplates)
-	workflowsMap := s.pathFilter.Match(includedWorkflows)
+func (store *Store) ValidateTemplates(templatesList, workflowsList []string) error {
+	templatePaths := store.config.Catalog.GetTemplatesPath(templatesList)
+	workflowPaths := store.config.Catalog.GetTemplatesPath(workflowsList)
 
-	notErrored := true
-	for k := range templatesMap {
-		_, err := s.loadTemplate(k, false)
-		if err != nil {
-			if strings.Contains(err.Error(), "cannot create template executer") {
+	filteredTemplatePaths := store.pathFilter.Match(templatePaths)
+	filteredWorkflowPaths := store.pathFilter.Match(workflowPaths)
+
+	if areTemplatesValid(store, filteredTemplatePaths) && areWorkflowsValid(store, filteredWorkflowPaths) {
+		return nil
+	}
+	return errors.New("an error occurred during templates validation")
+}
+
+func areWorkflowsValid(store *Store, filteredWorkflowPaths map[string]struct{}) bool {
+	return areWorkflowOrTemplatesValid(store, filteredWorkflowPaths, true, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) {
+		return parsers.LoadWorkflow(templatePath)
+	})
+}
+
+func areTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}) bool {
+	return areWorkflowOrTemplatesValid(store, filteredTemplatePaths, false, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) {
+		return parsers.LoadTemplate(templatePath, store.tagFilter, nil)
+	})
+}
+
+func areWorkflowOrTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}, isWorkflow bool, load func(templatePath string, tagFilter *filter.TagFilter) (bool, error)) bool {
+	areTemplatesValid := true
+	for templatePath := range filteredTemplatePaths {
+		if _, err := load(templatePath, store.tagFilter); err != nil {
+			if isParsingError("Error occurred loading template %s: %s\n", templatePath, err) {
+				areTemplatesValid = false
 				continue
 			}
-			if err == filter.ErrExcluded {
-				continue
-			}
-			notErrored = false
-			gologger.Error().Msgf("Error occurred loading template %s: %s\n", k, err)
-			continue
 		}
-		_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
+
+		template, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
 		if err != nil {
-			if strings.Contains(err.Error(), "cannot create template executer") {
-				continue
+			if isParsingError("Error occurred parsing template %s: %s\n", templatePath, err) {
+				areTemplatesValid = false
 			}
-			if err == filter.ErrExcluded {
-				continue
+		} else {
+			if !isWorkflow && len(template.Workflows) > 0 {
+				return true
 			}
-			notErrored = false
-			gologger.Error().Msgf("Error occurred parsing template %s: %s\n", k, err)
 		}
 	}
-	for k := range workflowsMap {
-		_, err := s.loadTemplate(k, true)
-		if err != nil {
-			if strings.Contains(err.Error(), "cannot create template executer") {
-				continue
-			}
-			if err == filter.ErrExcluded {
-				continue
-			}
-			notErrored = false
-			gologger.Error().Msgf("Error occurred loading workflow %s: %s\n", k, err)
-		}
-		_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
-		if err != nil {
-			if strings.Contains(err.Error(), "cannot create template executer") {
-				continue
-			}
-			if err == filter.ErrExcluded {
-				continue
-			}
-			notErrored = false
-			gologger.Error().Msgf("Error occurred parsing workflow %s: %s\n", k, err)
-		}
+	return areTemplatesValid
+}
+
+func isParsingError(message string, template string, err error) bool {
+	if err == templates.ErrCreateTemplateExecutor {
+		return false
 	}
-	return notErrored
+	if err == filter.ErrExcluded {
+		return false
+	}
+	gologger.Error().Msgf(message, template, err)
+	return true
 }
 
 // LoadTemplates takes a list of templates and returns paths for them
-func (s *Store) LoadTemplates(templatesList []string) []*templates.Template {
-	includedTemplates := s.config.Catalog.GetTemplatesPath(templatesList)
-	templatesMap := s.pathFilter.Match(includedTemplates)
+func (store *Store) LoadTemplates(templatesList []string) []*templates.Template {
+	includedTemplates := store.config.Catalog.GetTemplatesPath(templatesList)
+	templatePathMap := store.pathFilter.Match(includedTemplates)
 
-	loadedTemplates := make([]*templates.Template, 0, len(templatesMap))
-	for k := range templatesMap {
-		loaded, err := s.loadTemplate(k, false)
+	loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
+	for templatePath := range templatePathMap {
+		loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil)
 		if err != nil {
-			gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
+			gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
 		}
 		if loaded {
-			parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
+			parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
 			if err != nil {
-				gologger.Warning().Msgf("Could not parse template %s: %s\n", k, err)
+				gologger.Warning().Msgf("Could not parse template %s: %s\n", templatePath, err)
 			} else if parsed != nil {
 				loadedTemplates = append(loadedTemplates, parsed)
 			}
@@ -175,20 +177,20 @@ func (s *Store) LoadTemplates(templatesList []string) []*templates.Template {
 }
 
 // LoadWorkflows takes a list of workflows and returns paths for them
-func (s *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
-	includedWorkflows := s.config.Catalog.GetTemplatesPath(s.config.Workflows)
-	workflowsMap := s.pathFilter.Match(includedWorkflows)
+func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
+	includedWorkflows := store.config.Catalog.GetTemplatesPath(workflowsList)
+	workflowPathMap := store.pathFilter.Match(includedWorkflows)
 
-	loadedWorkflows := make([]*templates.Template, 0, len(workflowsMap))
-	for k := range workflowsMap {
-		loaded, err := s.loadTemplate(k, true)
+	loadedWorkflows := make([]*templates.Template, 0, len(workflowPathMap))
+	for workflowPath := range workflowPathMap {
+		loaded, err := parsers.LoadWorkflow(workflowPath)
 		if err != nil {
-			gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err)
+			gologger.Warning().Msgf("Could not load workflow %s: %s\n", workflowPath, err)
 		}
 		if loaded {
-			parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
+			parsed, err := templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions)
 			if err != nil {
-				gologger.Warning().Msgf("Could not parse workflow %s: %s\n", k, err)
+				gologger.Warning().Msgf("Could not parse workflow %s: %s\n", workflowPath, err)
 			} else if parsed != nil {
 				loadedWorkflows = append(loadedWorkflows, parsed)
 			}
@@ -196,7 +198,3 @@ func (s *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
 	}
 	return loadedWorkflows
 }
-
-func (s *Store) loadTemplate(templatePath string, workflow bool) (bool, error) {
-	return load.Load(templatePath, workflow, nil, s.tagFilter)
-}

v2/pkg/catalog/path.go

@@ -3,9 +3,7 @@ package catalog
 import (
 	"fmt"
 	"os"
-	"path"
 	"path/filepath"
-	"strings"
 )
 
 // ResolvePath resolves the path to an absolute one in various ways.
@@ -14,12 +12,12 @@ import (
 // or checking the nuclei templates directory. If a second path is given,
 // it also tries to find paths relative to that second path.
 func (c *Catalog) ResolvePath(templateName, second string) (string, error) {
-	if strings.HasPrefix(templateName, "/") || strings.Contains(templateName, ":\\") {
+	if filepath.IsAbs(templateName) {
 		return templateName, nil
 	}
 
 	if second != "" {
-		secondBasePath := path.Join(filepath.Dir(second), templateName)
+		secondBasePath := filepath.Join(filepath.Dir(second), templateName)
 		if _, err := os.Stat(secondBasePath); !os.IsNotExist(err) {
 			return secondBasePath, nil
 		}
@@ -30,13 +28,13 @@ func (c *Catalog) ResolvePath(templateName, second string) (string, error) {
 		return "", err
 	}
 
-	templatePath := path.Join(curDirectory, templateName)
+	templatePath := filepath.Join(curDirectory, templateName)
 	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
 		return templatePath, nil
 	}
 
 	if c.templatesDirectory != "" {
-		templatePath := path.Join(c.templatesDirectory, templateName)
+		templatePath := filepath.Join(c.templatesDirectory, templateName)
 		if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
 			return templatePath, nil
 		}

v2/pkg/model/model.go

@@ -0,0 +1,105 @@
+package model
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice"
+)
+
+// Info contains metadata information about a template
+type Info struct {
+	// description: |
+	//   Name should be good short summary that identifies what the template does.
+	//
+	// examples:
+	//   - value: "\"bower.json file disclosure\""
+	//   - value: "\"Nagios Default Credentials Check\""
+	Name string `json:"name,omitempty" yaml:"name,omitempty" jsonschema:"title=name of the template,description=Name is a short summary of what the template does,example=Nagios Default Credentials Check"`
+	// description: |
+	//   Author of the template.
+	//
+	//   Multiple values can also be specified separated by commas.
+	// examples:
+	//   - value: "\"<username>\""
+	Authors stringslice.StringSlice `json:"author,omitempty" yaml:"author,omitempty" jsonschema:"title=author of the template,description=Author is the author of the template,example=username"`
+	// description: |
+	//   Any tags for the template.
+	//
+	//   Multiple values can also be specified separated by commas.
+	//
+	// examples:
+	//   - name: Example tags
+	//     value: "\"cve,cve2019,grafana,auth-bypass,dos\""
+	Tags stringslice.StringSlice `json:"tags,omitempty" yaml:"tags,omitempty" jsonschema:"title=tags of the template,description=Any tags for the template"`
+	// description: |
+	//   Description of the template.
+	//
+	//   You can go in-depth here on what the template actually does.
+	//
+	// examples:
+	//   - value: "\"Bower is a package manager which stores package information in the bower.json file\""
+	//   - value: "\"Subversion ALM for the enterprise before 8.8.2 allows reflected XSS at multiple locations\""
+	Description string `json:"description,omitempty" yaml:"description,omitempty" jsonschema:"title=description of the template,description=In-depth explanation on what the template does,example=Bower is a package manager which stores package information in the bower.json file"`
+	// description: |
+	//   References for the template.
+	//
+	//   This should contain links relevant to the template.
+	//
+	// examples:
+	//   - value: >
+	//       []string{"https://github.com/strapi/strapi", "https://github.com/getgrav/grav"}
+	Reference stringslice.StringSlice `json:"reference,omitempty" yaml:"reference,omitempty" jsonschema:"title=references for the template,description=Links relevant to the template"`
+	// description: |
+	//   Severity of the template.
+	//
+	// values:
+	//   - info
+	//   - low
+	//   - medium
+	//   - high
+	//   - critical
+	SeverityHolder severity.Holder `json:"severity,omitempty" yaml:"severity,omitempty"`
+	// description: |
+	//   Metadata of the template.
+	//
+	// examples:
+	//   - value: >
+	//       map[string]string{"customField1":"customValue1"}
+	Metadata map[string]string `json:"metadata,omitempty" yaml:"metadata,omitempty" jsonschema:"title=additional metadata for the template,description=Additional metadata fields for the template"`
+
+	// description: |
+	//   Classification contains classification information about the template.
+	Classification *Classification `json:"classification,omitempty" yaml:"classification,omitempty" jsonschema:"title=classification info for the template,description=Classification information for the template"`
+
+	// description: |
+	//   Remediation steps for the template.
+	//
+	//   You can go in-depth here on how to mitigate the problem found by this template.
+	//
+	// examples:
+	//   - value: "\"Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties\""
+	Remediation string `json:"remediation,omitempty" yaml:"remediation,omitempty" jsonschema:"title=remediation steps for the template,description=In-depth explanation on how to fix the issues found by the template,example=Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties"`
+}
+
+// Classification contains the vulnerability classification data for a template.
+type Classification struct {
+	// description: |
+	//   CVE ID for the template
+	// examples:
+	//   - value: "\"CVE-2020-14420\""
+	CVEID stringslice.StringSlice `json:"cve-id,omitempty" yaml:"cve-id,omitempty" jsonschema:"title=cve ids for the template,description=CVE IDs for the template,example=CVE-2020-14420"`
+	// description: |
+	//   CWE ID for the template.
+	// examples:
+	//   - value: "\"CWE-22\""
+	CWEID stringslice.StringSlice `json:"cwe-id,omitempty" yaml:"cwe-id,omitempty" jsonschema:"title=cwe ids for the template,description=CWE IDs for the template,example=CWE-22"`
+	// description: |
+	//   CVSS Metrics for the template.
+	// examples:
+	//   - value: "\"3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\""
+	CVSSMetrics string `json:"cvss-metrics,omitempty" yaml:"cvss-metrics,omitempty" jsonschema:"title=cvss metrics for the template,description=CVSS Metrics for the template,example=3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"`
+	// description: |
+	//   CVSS Score for the template.
+	// examples:
+	//   - value: "\"9.8\""
+	CVSSScore float64 `json:"cvss-score,omitempty" yaml:"cvss-score,omitempty" jsonschema:"title=cvss score for the template,description=CVSS Score for the template,example=9.8"`
+}

v2/pkg/model/model_test.go

@@ -0,0 +1,117 @@
+package model
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice"
+
+	"gopkg.in/yaml.v2"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestInfoJsonMarshal(t *testing.T) {
+	info := Info{
+		Name:           "Test Template Name",
+		Authors:        stringslice.StringSlice{Value: []string{"forgedhallpass", "ice3man"}},
+		Description:    "Test description",
+		SeverityHolder: severity.Holder{Severity: severity.High},
+		Tags:           stringslice.StringSlice{Value: []string{"cve", "misc"}},
+		Reference:      stringslice.StringSlice{Value: "reference1"},
+	}
+
+	result, err := json.Marshal(&info)
+	assert.Nil(t, err)
+
+	expected := `{"name":"Test Template Name","author":["forgedhallpass","ice3man"],"tags":["cve","misc"],"description":"Test description","reference":"reference1","severity":"high"}`
+	assert.Equal(t, expected, string(result))
+}
+
+func TestInfoYamlMarshal(t *testing.T) {
+	info := Info{
+		Name:           "Test Template Name",
+		Authors:        stringslice.StringSlice{Value: []string{"forgedhallpass", "ice3man"}},
+		Description:    "Test description",
+		SeverityHolder: severity.Holder{Severity: severity.High},
+		Tags:           stringslice.StringSlice{Value: []string{"cve", "misc"}},
+		Reference:      stringslice.StringSlice{Value: "reference1"},
+	}
+
+	result, err := yaml.Marshal(&info)
+	assert.Nil(t, err)
+
+	expected := `name: Test Template Name
+author:
+- forgedhallpass
+- ice3man
+tags:
+- cve
+- misc
+description: Test description
+reference: reference1
+severity: high
+`
+	assert.Equal(t, expected, string(result))
+}
+
+func TestUnmarshal(t *testing.T) {
+	templateName := "Test Template"
+	authors := []string{"forgedhallpass", "ice3man"}
+	tags := []string{"cve", "misc"}
+	references := []string{"http://test.com", "http://domain.com"}
+
+	dynamicKey1 := "customDynamicKey1"
+	dynamicKey2 := "customDynamicKey2"
+
+	dynamicKeysMap := map[string]string{
+		dynamicKey1: "customDynamicValue1",
+		dynamicKey2: "customDynamicValue2",
+	}
+
+	assertUnmarshalledTemplateInfo := func(t *testing.T, yamlPayload string) Info {
+		info := Info{}
+		err := yaml.Unmarshal([]byte(yamlPayload), &info)
+		assert.Nil(t, err)
+		assert.Equal(t, info.Name, templateName)
+		assert.Equal(t, info.Authors.ToSlice(), authors)
+		assert.Equal(t, info.Tags.ToSlice(), tags)
+		assert.Equal(t, info.SeverityHolder.Severity, severity.Critical)
+		assert.Equal(t, info.Reference.ToSlice(), references)
+		assert.Equal(t, info.Metadata, dynamicKeysMap)
+		return info
+	}
+
+	yamlPayload1 := `
+  name: ` + templateName + `
+  author: ` + strings.Join(authors, ", ") + `
+  tags: ` + strings.Join(tags, ", ") + `
+  severity: critical
+  reference: ` + strings.Join(references, ", ") + `
+  metadata:
+     ` + dynamicKey1 + `: ` + dynamicKeysMap[dynamicKey1] + `
+     ` + dynamicKey2 + `: ` + dynamicKeysMap[dynamicKey2] + `
+`
+	yamlPayload2 := `
+  name: ` + templateName + `
+  author:
+    - ` + authors[0] + `
+    - ` + authors[1] + `
+  tags:
+    - ` + tags[0] + `
+    - ` + tags[1] + `
+  severity: critical
+  reference:
+    - ` + references[0] + ` # comments are not unmarshalled
+    - ` + references[1] + `
+  metadata:
+     ` + dynamicKey1 + `: ` + dynamicKeysMap[dynamicKey1] + `
+     ` + dynamicKey2 + `: ` + dynamicKeysMap[dynamicKey2] + `
+`
+
+	info1 := assertUnmarshalledTemplateInfo(t, yamlPayload1)
+	info2 := assertUnmarshalledTemplateInfo(t, yamlPayload2)
+	assert.Equal(t, info1, info2)
+}

v2/pkg/model/types/severity/severities.go

@@ -0,0 +1,62 @@
+package severity
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/projectdiscovery/goflags"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice"
+)
+
+// Severities used by the goflags library for parsing an array of Severity types, passed as CLI arguments from the user
+type Severities []Severity
+
+func (severities *Severities) Set(values string) error {
+	inputSeverities, err := goflags.ToNormalizedStringSlice(values)
+	if err != nil {
+		return err
+	}
+
+	for _, inputSeverity := range inputSeverities {
+		if err := setSeverity(severities, inputSeverity); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (severities *Severities) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var stringSliceValue stringslice.StringSlice
+	if err := unmarshal(&stringSliceValue); err != nil {
+		return err
+	}
+
+	stringSLice := stringSliceValue.ToSlice()
+	var result = make(Severities, 0, len(stringSLice))
+	for _, severityString := range stringSLice {
+		if err := setSeverity(&result, severityString); err != nil {
+			return err
+		}
+	}
+	*severities = result
+	return nil
+}
+
+func (severities Severities) String() string {
+	var stringSeverities []string
+	for _, severity := range severities {
+		stringSeverities = append(stringSeverities, severity.String())
+	}
+	return strings.Join(stringSeverities, ", ")
+}
+
+func setSeverity(severities *Severities, value string) error {
+	computedSeverity, err := toSeverity(value)
+	if err != nil {
+		return fmt.Errorf("'%s' is not a valid severity", value)
+	}
+
+	// TODO change the Severities type to map[Severity]interface{}, where the values are struct{}{}, to "simulates" a "set" data structure
+	*severities = append(*severities, computedSeverity)
+	return nil
+}

v2/pkg/model/types/severity/severity.go

@@ -0,0 +1,53 @@
+package severity
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+type Severity int
+
+const (
+	Undefined Severity = iota
+	Info
+	Low
+	Medium
+	High
+	Critical
+	limit
+)
+
+var severityMappings = map[Severity]string{
+	Info:     "info",
+	Low:      "low",
+	Medium:   "medium",
+	High:     "high",
+	Critical: "critical",
+}
+
+func GetSupportedSeverities() Severities {
+	var result []Severity
+	for index := Severity(1); index < limit; index++ {
+		result = append(result, index)
+	}
+	return result
+}
+
+func toSeverity(valueToMap string) (Severity, error) {
+	normalizedValue := normalizeValue(valueToMap)
+	for key, currentValue := range severityMappings {
+		if normalizedValue == currentValue {
+			return key, nil
+		}
+	}
+	return -1, errors.New("Invalid severity: " + valueToMap)
+}
+
+func normalizeValue(value string) string {
+	return strings.TrimSpace(strings.ToLower(value))
+}
+
+func (severity Severity) String() string {
+	return severityMappings[severity]
+}

v2/pkg/model/types/severity/severity_holder.go

@@ -0,0 +1,48 @@
+package severity
+
+import (
+	"encoding/json"
+
+	"github.com/alecthomas/jsonschema"
+)
+
+//nolint:exported,revive //prefer to be explicit about the name, and make it refactor-safe
+// Holder holds a Severity type. Required for un/marshalling purposes
+type Holder struct {
+	Severity Severity
+}
+
+func (severityHolder Holder) JSONSchemaType() *jsonschema.Type {
+	gotType := &jsonschema.Type{
+		Type:        "string",
+		Title:       "severity of the template",
+		Description: "Seriousness of the implications of the template",
+	}
+	for _, severity := range GetSupportedSeverities() {
+		gotType.Enum = append(gotType.Enum, severity.String())
+	}
+	return gotType
+}
+
+func (severityHolder *Holder) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var marshalledSeverity string
+	if err := unmarshal(&marshalledSeverity); err != nil {
+		return err
+	}
+
+	computedSeverity, err := toSeverity(marshalledSeverity)
+	if err != nil {
+		return err
+	}
+
+	severityHolder.Severity = computedSeverity
+	return nil
+}
+
+func (severityHolder *Holder) MarshalJSON() ([]byte, error) {
+	return json.Marshal(severityHolder.Severity.String())
+}
+
+func (severityHolder Holder) MarshalYAML() (interface{}, error) {
+	return severityHolder.Severity.String(), nil
+}

v2/pkg/model/types/severity/severity_test.go

@@ -0,0 +1,65 @@
+package severity
+
+import (
+	"testing"
+
+	"gopkg.in/yaml.v2"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestYamlUnmarshal(t *testing.T) {
+	testUnmarshal(t, yaml.Unmarshal, func(value string) string { return value })
+}
+
+func TestYamlMarshal(t *testing.T) {
+	severity := Holder{Severity: High}
+
+	marshalled, err := severity.MarshalYAML()
+	assert.Nil(t, err, "could not marshal yaml")
+	assert.Equal(t, "high", marshalled, "could not marshal severity correctly")
+}
+
+func TestYamlUnmarshalFail(t *testing.T) {
+	testUnmarshalFail(t, yaml.Unmarshal, createYAML)
+}
+
+func TestGetSupportedSeverities(t *testing.T) {
+	severities := GetSupportedSeverities()
+	assert.Equal(t, severities, Severities{Info, Low, Medium, High, Critical})
+}
+
+func testUnmarshal(t *testing.T, unmarshaller func(data []byte, v interface{}) error, payloadCreator func(value string) string) {
+	payloads := [...]string{
+		payloadCreator("Info"),
+		payloadCreator("info"),
+		payloadCreator("inFo "),
+		payloadCreator("infO "),
+		payloadCreator(" INFO "),
+	}
+
+	for _, payload := range payloads { // nolint:scopelint // false-positive
+		t.Run(payload, func(t *testing.T) {
+			result := unmarshal(payload, unmarshaller)
+			assert.Equal(t, result.Severity, Info)
+			assert.Equal(t, result.Severity.String(), "info")
+		})
+	}
+}
+
+func testUnmarshalFail(t *testing.T, unmarshaller func(data []byte, v interface{}) error, payloadCreator func(value string) string) {
+	assert.Panics(t, func() { unmarshal(payloadCreator("invalid"), unmarshaller) })
+}
+
+func unmarshal(value string, unmarshaller func(data []byte, v interface{}) error) Holder {
+	severityStruct := Holder{}
+	var err = unmarshaller([]byte(value), &severityStruct)
+	if err != nil {
+		panic(err)
+	}
+	return severityStruct
+}
+
+func createYAML(value string) string {
+	return "severity: " + value + "\n"
+}

v2/pkg/model/types/stringslice/stringslice.go

@@ -0,0 +1,92 @@
+package stringslice
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/alecthomas/jsonschema"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
+)
+
+// StringSlice represents a single (in-lined) or multiple string value(s).
+// The unmarshaller does not automatically convert in-lined strings to []string, hence the interface{} type is required.
+type StringSlice struct {
+	Value interface{}
+}
+
+func (stringSlice StringSlice) JSONSchemaType() *jsonschema.Type {
+	gotType := &jsonschema.Type{
+		OneOf: []*jsonschema.Type{{Type: "string"}, {Type: "array"}},
+	}
+	return gotType
+}
+
+func (stringSlice *StringSlice) IsEmpty() bool {
+	return len(stringSlice.ToSlice()) == 0
+}
+
+func (stringSlice StringSlice) ToSlice() []string {
+	switch value := stringSlice.Value.(type) {
+	case string:
+		return []string{value}
+	case []string:
+		return value
+	case nil:
+		return []string{}
+	default:
+		panic(fmt.Sprintf("Unexpected StringSlice type: '%T'", value))
+	}
+}
+
+func (stringSlice StringSlice) String() string {
+	return strings.Join(stringSlice.ToSlice(), ", ")
+}
+
+func (stringSlice *StringSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	marshalledSlice, err := marshalStringToSlice(unmarshal)
+	if err != nil {
+		return err
+	}
+
+	result := make([]string, 0, len(marshalledSlice))
+	//nolint:gosimple,nolintlint //cannot be replaced with result = append(result, slices...) because the values are being normalized
+	for _, value := range marshalledSlice {
+		result = append(result, strings.ToLower(strings.TrimSpace(value))) // TODO do we need to introduce RawStringSlice and/or NormalizedStringSlices?
+	}
+	stringSlice.Value = result
+	return nil
+}
+
+func (stringSlice StringSlice) MarshalYAML() (interface{}, error) {
+	return stringSlice.Value, nil
+}
+
+func (stringSlice StringSlice) MarshalJSON() ([]byte, error) {
+	return json.Marshal(stringSlice.Value)
+}
+
+func marshalStringToSlice(unmarshal func(interface{}) error) ([]string, error) {
+	var marshalledValueAsString string
+	var marshalledValuesAsSlice []string
+
+	sliceMarshalError := unmarshal(&marshalledValuesAsSlice)
+	if sliceMarshalError != nil {
+		stringMarshalError := unmarshal(&marshalledValueAsString)
+		if stringMarshalError != nil {
+			return nil, stringMarshalError
+		}
+	}
+
+	var result []string
+	if len(marshalledValuesAsSlice) > 0 {
+		result = marshalledValuesAsSlice
+	} else if utils.IsNotBlank(marshalledValueAsString) {
+		result = strings.Split(marshalledValueAsString, ",")
+	} else {
+		result = []string{}
+	}
+
+	return result, nil
+}

v2/pkg/model/worflow_loader.go

@@ -0,0 +1,12 @@
+package model
+
+// TODO shouldn't this rather be TemplateLoader?
+
+// WorkflowLoader is a loader interface required for workflow initialization.
+type WorkflowLoader interface {
+	// GetTemplatePathsByTags returns a list of template paths based on the provided tags from the templates directory
+	GetTemplatePathsByTags(tags []string) []string
+
+	// GetTemplatePaths takes a list of templates and returns paths for them
+	GetTemplatePaths(templatesList []string, noValidate bool) []string
+}

v2/pkg/operators/common/dsl/dsl.go

@@ -17,6 +17,7 @@ import (
 	"time"
 
 	"github.com/Knetic/govaluate"
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/deserialization"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/spaolacci/murmur3"
@@ -52,7 +53,7 @@ var functions = map[string]govaluate.ExpressionFunction{
 		return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil
 	},
 	"trim": func(args ...interface{}) (interface{}, error) {
-		return strings.Trim(types.ToString(args[0]), types.ToString(args[2])), nil
+		return strings.Trim(types.ToString(args[0]), types.ToString(args[1])), nil
 	},
 	"trimleft": func(args ...interface{}) (interface{}, error) {
 		return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil
@@ -87,10 +88,10 @@ var functions = map[string]govaluate.ExpressionFunction{
 		return base64.StdEncoding.DecodeString(types.ToString(args[0]))
 	},
 	"url_encode": func(args ...interface{}) (interface{}, error) {
-		return url.PathEscape(types.ToString(args[0])), nil
+		return url.QueryEscape(types.ToString(args[0])), nil
 	},
 	"url_decode": func(args ...interface{}) (interface{}, error) {
-		return url.PathUnescape(types.ToString(args[0]))
+		return url.QueryUnescape(types.ToString(args[0]))
 	},
 	"hex_encode": func(args ...interface{}) (interface{}, error) {
 		return hex.EncodeToString([]byte(types.ToString(args[0]))), nil
@@ -113,18 +114,14 @@ var functions = map[string]govaluate.ExpressionFunction{
 	},
 	"sha256": func(args ...interface{}) (interface{}, error) {
 		h := sha256.New()
-		_, err := h.Write([]byte(types.ToString(args[0])))
-
-		if err != nil {
+		if _, err := h.Write([]byte(types.ToString(args[0]))); err != nil {
 			return nil, err
 		}
 		return hex.EncodeToString(h.Sum(nil)), nil
 	},
 	"sha1": func(args ...interface{}) (interface{}, error) {
 		h := sha1.New()
-		_, err := h.Write([]byte(types.ToString(args[0])))
-
-		if err != nil {
+		if _, err := h.Write([]byte(types.ToString(args[0]))); err != nil {
 			return nil, err
 		}
 		return hex.EncodeToString(h.Sum(nil)), nil
@@ -162,7 +159,7 @@ var functions = map[string]govaluate.ExpressionFunction{
 		base := letters + numbers
 
 		if len(args) >= 1 {
-			l = args[0].(int)
+			l = int(args[0].(float64))
 		}
 		if len(args) >= withCutSetArgsSize {
 			bad = types.ToString(args[1])
@@ -179,7 +176,7 @@ var functions = map[string]govaluate.ExpressionFunction{
 		chars := letters + numbers
 
 		if len(args) >= 1 {
-			l = args[0].(int)
+			l = int(args[0].(float64))
 		}
 		if len(args) >= withCutSetArgsSize {
 			bad = types.ToString(args[1])
@@ -193,7 +190,7 @@ var functions = map[string]govaluate.ExpressionFunction{
 		chars := letters
 
 		if len(args) >= 1 {
-			l = args[0].(int)
+			l = int(args[0].(float64))
 		}
 		if len(args) >= withCutSetArgsSize {
 			bad = types.ToString(args[1])
@@ -207,7 +204,7 @@ var functions = map[string]govaluate.ExpressionFunction{
 		chars := numbers
 
 		if len(args) >= 1 {
-			l = args[0].(int)
+			l = int(args[0].(float64))
 		}
 		if len(args) >= withCutSetArgsSize {
 			bad = types.ToString(args[1])
@@ -220,10 +217,10 @@ var functions = map[string]govaluate.ExpressionFunction{
 		max := math.MaxInt32
 
 		if len(args) >= 1 {
-			min = args[0].(int)
+			min = int(args[0].(float64))
 		}
 		if len(args) >= withMaxRandArgsSize {
-			max = args[1].(int)
+			max = int(args[1].(float64))
 		}
 		return rand.Intn(max-min) + min, nil
 	},
@@ -245,6 +242,11 @@ var functions = map[string]govaluate.ExpressionFunction{
 		data := deserialization.GenerateJavaGadget(gadget, cmd, encoding)
 		return data, nil
 	},
+	// for debug purposes
+	"print_debug": func(args ...interface{}) (interface{}, error) {
+		gologger.Info().Msgf("print_debug value: %s", fmt.Sprint(args))
+		return true, nil
+	},
 }
 
 // HelperFunctions returns the dsl helper functions
@@ -252,7 +254,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 	return functions
 }
 
-// AddHelperFunction allows creation of additiona helper functions to be supported with templates
+// AddHelperFunction allows creation of additional helper functions to be supported with templates
 func AddHelperFunction(key string, value func(args ...interface{}) (interface{}, error)) error {
 	if _, ok := functions[key]; !ok {
 		functions[key] = value

v2/pkg/operators/common/dsl/dsl_test.go

@@ -0,0 +1,19 @@
+package dsl
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDSLURLEncodeDecode(t *testing.T) {
+	functions := HelperFunctions()
+
+	encoded, err := functions["url_encode"]("&test\"")
+	require.Nil(t, err, "could not url encode")
+	require.Equal(t, "%26test%22", encoded, "could not get url encoded data")
+
+	decoded, err := functions["url_decode"]("%26test%22")
+	require.Nil(t, err, "could not url encode")
+	require.Equal(t, "&test\"", decoded, "could not get url decoded data")
+}

v2/pkg/operators/extractors/compile.go

@@ -4,12 +4,14 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
+
+	"github.com/itchyny/gojq"
 )
 
-// CompileExtractors performs the initial setup operation on a extractor
+// CompileExtractors performs the initial setup operation on an extractor
 func (e *Extractor) CompileExtractors() error {
 	var ok bool
-	// Setup the extractor type
+	// Set up the extractor type
 	e.extractorType, ok = ExtractorTypes[e.Type]
 	if !ok {
 		return fmt.Errorf("unknown extractor type specified: %s", e.Type)
@@ -28,7 +30,19 @@ func (e *Extractor) CompileExtractors() error {
 		e.KVal[i] = strings.ToLower(kval)
 	}
 
-	// Setup the part of the request to match, if any.
+	for _, query := range e.JSON {
+		query, err := gojq.Parse(query)
+		if err != nil {
+			return fmt.Errorf("could not parse json: %s", query)
+		}
+		compiled, err := gojq.Compile(query)
+		if err != nil {
+			return fmt.Errorf("could not compile json: %s", query)
+		}
+		e.jsonCompiled = append(e.jsonCompiled, compiled)
+	}
+
+	// Set up the part of the request to match, if any.
 	if e.Part == "" {
 		e.Part = "body"
 	}

v2/pkg/operators/extractors/extract.go

@@ -1,6 +1,12 @@
 package extractors
 
 import (
+	"strings"
+
+	"encoding/json"
+
+	"github.com/antchfx/htmlquery"
+
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
@@ -42,3 +48,68 @@ func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{}
 	}
 	return results
 }
+
+// ExtractHTML extracts items from text using XPath selectors
+func (e *Extractor) ExtractHTML(corpus string) map[string]struct{} {
+	results := make(map[string]struct{})
+
+	doc, err := htmlquery.Parse(strings.NewReader(corpus))
+	if err != nil {
+		return results
+	}
+	for _, k := range e.XPath {
+		nodes, err := htmlquery.QueryAll(doc, k)
+		if err != nil {
+			continue
+		}
+		for _, node := range nodes {
+			var value string
+
+			if e.Attribute != "" {
+				value = htmlquery.SelectAttr(node, e.Attribute)
+			} else {
+				value = htmlquery.InnerText(node)
+			}
+			if _, ok := results[value]; !ok {
+				results[value] = struct{}{}
+			}
+		}
+	}
+	return results
+}
+
+// ExtractJSON extracts text from a corpus using JQ queries and returns it
+func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} {
+	results := make(map[string]struct{})
+
+	var jsonObj interface{}
+
+	if err := json.Unmarshal([]byte(corpus), &jsonObj); err != nil {
+		return results
+	}
+
+	for _, k := range e.jsonCompiled {
+		iter := k.Run(jsonObj)
+		for {
+			v, ok := iter.Next()
+			if !ok {
+				break
+			}
+			if _, ok := v.(error); ok {
+				break
+			}
+			var result string
+			if res, err := types.JSONScalarToString(v); err == nil {
+				result = res
+			} else if res, err := json.Marshal(v); err == nil {
+				result = string(res)
+			} else {
+				result = types.ToString(v)
+			}
+			if _, ok := results[result]; !ok {
+				results[result] = struct{}{}
+			}
+		}
+	}
+	return results
+}

v2/pkg/operators/extractors/extractors.go

@@ -1,32 +1,110 @@
 package extractors
 
-import "regexp"
+import (
+	"regexp"
+
+	"github.com/itchyny/gojq"
+)
 
 // Extractor is used to extract part of response using a regex.
 type Extractor struct {
-	// Name is the extractor's name
-	Name string `yaml:"name,omitempty"`
-	// Type is the type of the extractor
-	Type string `yaml:"type"`
+	// description: |
+	//   Name of the extractor. Name should be lowercase and must not contain
+	//   spaces or underscores (_).
+	// examples:
+	//   - value: "\"cookie-extractor\""
+	Name string `yaml:"name,omitempty" jsonschema:"title=name of the extractor,description=Name of the extractor"`
+	// description: |
+	//   Type is the type of the extractor.
+	// values:
+	//   - "regex"
+	//   - "kval"
+	//   - "json"
+	//   - "xpath"
+	Type string `yaml:"type" jsonschema:"title=type of the extractor,description=Type of the extractor,enum=regex,enum=kval,enum=json,enum=xpath"`
 	// extractorType is the internal type of the extractor
 	extractorType ExtractorType
 
-	// Regex are the regex pattern required to be present in the response
-	Regex []string `yaml:"regex"`
-	// RegexGroup specifies a group to extract from the regex
-	RegexGroup int `yaml:"group"`
+	// description: |
+	//   Regex contains the regular expression patterns to extract from a part.
+	//
+	//   Go regex engine does not support lookaheads or lookbehinds, so as a result
+	//   they are also not supported in nuclei.
+	// examples:
+	//   - name: Braintree Access Token Regex
+	//     value: >
+	//       []string{"access_token\\$production\\$[0-9a-z]{16}\\$[0-9a-f]{32}"}
+	//   - name: Wordpress Author Extraction regex
+	//     value: >
+	//       []string{"Author:(?:[A-Za-z0-9 -\\_=\"]+)?<span(?:[A-Za-z0-9 -\\_=\"]+)?>([A-Za-z0-9]+)<\\/span>"}
+	Regex []string `yaml:"regex,omitempty" jsonschema:"title=regex to extract from part,description=Regex to extract from part"`
+	// description: |
+	//   Group specifies a numbered group to extract from the regex.
+	// examples:
+	//   - name: Example Regex Group
+	//     value: "1"
+	RegexGroup int `yaml:"group,omitempty" jsonschema:"title=group to extract from regex,description=Group to extract from regex"`
 	// regexCompiled is the compiled variant
 	regexCompiled []*regexp.Regexp
 
-	// KVal are the kval to be present in the response headers/cookies
-	KVal []string `yaml:"kval,omitempty"`
-
-	// Part is the part of the request to match
+	// description: |
+	//   kval contains the key-value pairs present in the HTTP response header.
+	//   kval extractor can be used to extract HTTP response header and cookie key-value pairs.
+	//   kval extractor inputs are case-insensitive, and does not support dash (-) in input which can replaced with underscores (_)
+	// 	 For example, Content-Type should be replaced with content_type
 	//
-	// By default, matching is performed in request body.
-	Part string `yaml:"part,omitempty"`
-	// Internal defines if this is used internally
-	Internal bool `yaml:"internal,omitempty"`
+	//   A list of supported parts is available in docs for request types.
+	// examples:
+	//   - name: Extract Server Header From HTTP Response
+	//     value: >
+	//       []string{"server"}
+	//   - name: Extracting value of PHPSESSID Cookie
+	//     value: >
+	//       []string{"phpsessid"}
+	//   - name: Extracting value of Content-Type Cookie
+	//     value: >
+	//       []string{"content_type"}
+	KVal []string `yaml:"kval,omitempty" jsonschema:"title=kval pairs to extract from response,description=Kval pairs to extract from response"`
+
+	// description: |
+	//   JSON allows using jq-style syntax to extract items from json response
+	//
+	// examples:
+	//   - value: >
+	//       []string{".[] | .id"}
+	//   - value: >
+	//       []string{".batters | .batter | .[] | .id"}
+	JSON []string `yaml:"json,omitempty" jsonschema:"title=json jq expressions to extract data,description=JSON JQ expressions to evaluate from response part"`
+	// description: |
+	//   XPath allows using xpath expressions to extract items from html response
+	//
+	// examples:
+	//   - value: >
+	//       []string{"/html/body/div/p[2]/a"}
+	XPath []string `yaml:"xpath,omitempty" jsonschema:"title=html xpath expressions to extract data,description=XPath allows using xpath expressions to extract items from html response"`
+	// description: |
+	//   Attribute is an optional attribute to extract from response XPath.
+	//
+	// examples:
+	//   - value: "\"href\""
+	Attribute string `yaml:"attribute,omitempty" jsonschema:"title=optional attribute to extract from xpath,description=Optional attribute to extract from response XPath"`
+
+	// jsonCompiled is the compiled variant
+	jsonCompiled []*gojq.Code
+
+	// description: |
+	//   Part is the part of the request response to extract data from.
+	//
+	//   Each protocol exposes a lot of different parts which are well
+	//   documented in docs for each request type.
+	// examples:
+	//   - value: "\"body\""
+	//   - value: "\"raw\""
+	Part string `yaml:"part,omitempty" jsonschema:"title=part of response to extract data from,description=Part of the request response to extract data from"`
+	// description: |
+	//   Internal, when set to true will allow using the value extracted
+	//   in the next request for some protocols (like HTTP).
+	Internal bool `yaml:"internal,omitempty" jsonschema:"title=mark extracted value for internal variable use,description=Internal when set to true will allow using the value extracted in the next request for some protocols"`
 }
 
 // ExtractorType is the type of the extractor specified
@@ -37,12 +115,18 @@ const (
 	RegexExtractor ExtractorType = iota + 1
 	// KValExtractor extracts responses with key:value
 	KValExtractor
+	// XPathExtractor extracts responses with Xpath selectors
+	XPathExtractor
+	// JSONExtractor extracts responses with json
+	JSONExtractor
 )
 
-// ExtractorTypes is an table for conversion of extractor type from string.
+// ExtractorTypes is a table for conversion of extractor type from string.
 var ExtractorTypes = map[string]ExtractorType{
 	"regex": RegexExtractor,
 	"kval":  KValExtractor,
+	"xpath": XPathExtractor,
+	"json":  JSONExtractor,
 }
 
 // GetType returns the type of the matcher

v2/pkg/operators/matchers/compile.go

@@ -6,6 +6,7 @@ import (
 	"regexp"
 
 	"github.com/Knetic/govaluate"
+
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
 )
 
@@ -22,7 +23,7 @@ func (m *Matcher) CompileMatchers() error {
 		}
 	}
 
-	// Setup the matcher type
+	// Set up the matcher type
 	m.matcherType, ok = MatcherTypes[m.Type]
 	if !ok {
 		return fmt.Errorf("unknown matcher type specified: %s", m.Type)
@@ -50,7 +51,7 @@ func (m *Matcher) CompileMatchers() error {
 		m.dslCompiled = append(m.dslCompiled, compiled)
 	}
 
-	// Setup the condition type, if any.
+	// Set up the condition type, if any.
 	if m.Condition != "" {
 		m.condition, ok = ConditionTypes[m.Condition]
 		if !ok {

v2/pkg/operators/matchers/match.go

@@ -48,7 +48,7 @@ func (m *Matcher) MatchWords(corpus string) bool {
 			if m.condition == ANDCondition {
 				return false
 			}
-			// Continue with the flow since its an OR Condition.
+			// Continue with the flow since it's an OR Condition.
 			continue
 		}
 
@@ -76,7 +76,7 @@ func (m *Matcher) MatchRegex(corpus string) bool {
 			if m.condition == ANDCondition {
 				return false
 			}
-			// Continue with the flow since its an OR Condition.
+			// Continue with the flow since it's an OR Condition.
 			continue
 		}
 
@@ -105,7 +105,7 @@ func (m *Matcher) MatchBinary(corpus string) bool {
 			if m.condition == ANDCondition {
 				return false
 			}
-			// Continue with the flow since its an OR Condition.
+			// Continue with the flow since it's an OR Condition.
 			continue
 		}
 
@@ -141,7 +141,7 @@ func (m *Matcher) MatchDSL(data map[string]interface{}) bool {
 			if m.condition == ANDCondition {
 				return false
 			}
-			// Continue with the flow since its an OR Condition.
+			// Continue with the flow since it's an OR Condition.
 			continue
 		}
 

v2/pkg/operators/matchers/matchers.go

@@ -8,36 +8,103 @@ import (
 
 // Matcher is used to match a part in the output from a protocol.
 type Matcher struct {
-	// Type is the type of the matcher
-	Type string `yaml:"type"`
-	// Condition is the optional condition between two matcher variables
+	// description: |
+	//   Type is the type of the matcher.
+	// values:
+	//   - "status"
+	//   - "size"
+	//   - "word"
+	//   - "regex"
+	//   - "binary"
+	//   - "dsl"
+	Type string `yaml:"type" jsonschema:"title=type of matcher,description=Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl"`
+	// description: |
+	//   Condition is the optional condition between two matcher variables. By default,
+	//   the condition is assumed to be OR.
+	// values:
+	//   - "and"
+	//   - "or"
+	Condition string `yaml:"condition,omitempty" jsonschema:"title=condition between matcher variables,description=Condition between the matcher variables,enum=and,enum=or"`
+
+	// description: |
+	//   Part is the part of the request response to match data from.
 	//
-	// By default, the condition is assumed to be OR.
-	Condition string `yaml:"condition,omitempty"`
+	//   Each protocol exposes a lot of different parts which are well
+	//   documented in docs for each request type.
+	// examples:
+	//   - value: "\"body\""
+	//   - value: "\"raw\""
+	Part string `yaml:"part,omitempty" jsonschema:"title=part of response to match,description=Part of response to match data from"`
 
-	// Part is the part of the data to match
-	Part string `yaml:"part,omitempty"`
+	// description: |
+	//   Negative specifies if the match should be reversed
+	//   It will only match if the condition is not true.
+	Negative bool `yaml:"negative,omitempty" jsonschema:"title=negative specifies if match reversed,description=Negative specifies if the match should be reversed. It will only match if the condition is not true"`
 
-	// Negative specifies if the match should be reversed
-	// It will only match if the condition is not true.
-	Negative bool `yaml:"negative,omitempty"`
-
-	// Name is matcher Name
-	Name string `yaml:"name,omitempty"`
-	// Status are the acceptable status codes for the response
-	Status []int `yaml:"status,omitempty"`
-	// Size is the acceptable size for the response
-	Size []int `yaml:"size,omitempty"`
-	// Words are the words required to be present in the response
-	Words []string `yaml:"words,omitempty"`
-	// Regex are the regex pattern required to be present in the response
-	Regex []string `yaml:"regex,omitempty"`
-	// Binary are the binary characters required to be present in the response
-	Binary []string `yaml:"binary,omitempty"`
-	// DSL are the dsl queries
-	DSL []string `yaml:"dsl,omitempty"`
-	// Encoding specifies the encoding for the word content if any.
-	Encoding string `yaml:"encoding,omitempty"`
+	// description: |
+	//   Name of the matcher. Name should be lowercase and must not contain
+	//   spaces or underscores (_).
+	// examples:
+	//   - value: "\"cookie-matcher\""
+	Name string `yaml:"name,omitempty" jsonschema:"title=name of the matcher,description=Name of the matcher"`
+	// description: |
+	//   Status are the acceptable status codes for the response.
+	// examples:
+	//   - value: >
+	//       []int{200, 302}
+	Status []int `yaml:"status,omitempty" jsonschema:"title=status to match,description=Status to match for the response"`
+	// description: |
+	//   Size is the acceptable size for the response
+	// examples:
+	//   - value: >
+	//       []int{3029, 2042}
+	Size []int `yaml:"size,omitempty" jsonschema:"title=acceptable size for response,description=Size is the acceptable size for the response"`
+	// description: |
+	//   Words contains word patterns required to be present in the response part.
+	// examples:
+	//   - name: Match for outlook mail protection domain
+	//     value: >
+	//       []string{"mail.protection.outlook.com"}
+	//   - name: Match for application/json in response headers
+	//     value: >
+	//       []string{"application/json"}
+	Words []string `yaml:"words,omitempty" jsonschema:"title=words to match in response,description= Words contains word patterns required to be present in the response part"`
+	// description: |
+	//   Regex contains Regular Expression patterns required to be present in the response part.
+	// examples:
+	//   - name: Match for Linkerd Service via Regex
+	//     value: >
+	//       []string{`(?mi)^Via\\s*?:.*?linkerd.*$`}
+	//   - name: Match for Open Redirect via Location header
+	//     value: >
+	//       []string{`(?m)^(?:Location\\s*?:\\s*?)(?:https?://|//)?(?:[a-zA-Z0-9\\-_\\.@]*)example\\.com.*$`}
+	Regex []string `yaml:"regex,omitempty" jsonschema:"title=regex to match in response,description=Regex contains regex patterns required to be present in the response part"`
+	// description: |
+	//   Binary are the binary patterns required to be present in the response part.
+	// examples:
+	//   - name: Match for Springboot Heapdump Actuator "JAVA PROFILE", "HPROF", "Gunzip magic byte"
+	//     value: >
+	//       []string{"4a4156412050524f46494c45", "4850524f46", "1f8b080000000000"}
+	//   - name: Match for 7zip files
+	//     value: >
+	//       []string{"377ABCAF271C"}
+	Binary []string `yaml:"binary,omitempty" jsonschema:"title=binary patterns to match in response,description=Binary are the binary patterns required to be present in the response part"`
+	// description: |
+	//   DSL are the dsl expressions that will be evaluated as part of nuclei matching rules.
+	//   A list of these helper functions are available [here](https://nuclei.projectdiscovery.io/templating-guide/helper-functions/).
+	// examples:
+	//   - name: DSL Matcher for package.json file
+	//     value: >
+	//       []string{"contains(body, 'packages') && contains(tolower(all_headers), 'application/octet-stream') && status_code == 200"}
+	//   - name: DSL Matcher for missing strict transport security header
+	//     value: >
+	//       []string{"!contains(tolower(all_headers), ''strict-transport-security'')"}
+	DSL []string `yaml:"dsl,omitempty" jsonschema:"title=dsl expressions to match in response,description=DSL are the dsl expressions that will be evaluated as part of nuclei matching rules"`
+	// description: |
+	//   Encoding specifies the encoding for the words field if any.
+	// values:
+	//   - "hex"
+	Encoding string `yaml:"encoding,omitempty" jsonschema:"title=encoding for word field,description=Optional encoding for the word fields,enum=hex"`
 
 	// cached data for the compiled matcher
 	condition     ConditionType
@@ -64,7 +131,7 @@ const (
 	DSLMatcher
 )
 
-// MatcherTypes is an table for conversion of matcher type from string.
+// MatcherTypes is a table for conversion of matcher type from string.
 var MatcherTypes = map[string]MatcherType{
 	"status": StatusMatcher,
 	"size":   SizeMatcher,
@@ -84,7 +151,7 @@ const (
 	ORCondition
 )
 
-// ConditionTypes is an table for conversion of condition type from string.
+// ConditionTypes is a table for conversion of condition type from string.
 var ConditionTypes = map[string]ConditionType{
 	"and": ANDCondition,
 	"or":  ORCondition,

v2/pkg/operators/operators.go

@@ -2,21 +2,31 @@ package operators
 
 import (
 	"github.com/pkg/errors"
+
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 )
 
 // Operators contains the operators that can be applied on protocols
 type Operators struct {
-	// Matchers contains the detection mechanism for the request to identify
-	// whether the request was successful
-	Matchers []*matchers.Matcher `yaml:"matchers,omitempty"`
-	// Extractors contains the extraction mechanism for the request to identify
-	// and extract parts of the response.
-	Extractors []*extractors.Extractor `yaml:"extractors,omitempty"`
-	// MatchersCondition is the condition of the matchers
-	// whether to use AND or OR. Default is OR.
-	MatchersCondition string `yaml:"matchers-condition,omitempty"`
+	// description: |
+	//   Matchers contains the detection mechanism for the request to identify
+	//   whether the request was successful by doing pattern matching
+	//   on request/responses.
+	//
+	//   Multiple matchers can be combined with `matcher-condition` flag
+	//   which accepts either `and` or `or` as argument.
+	Matchers []*matchers.Matcher `yaml:"matchers,omitempty" jsonschema:"title=matchers to run on response,description=Detection mechanism to identify whether the request was successful by doing pattern matching"`
+	// description: |
+	//   Extractors contains the extraction mechanism for the request to identify
+	//   and extract parts of the response.
+	Extractors []*extractors.Extractor `yaml:"extractors,omitempty" jsonschema:"title=extractors to run on response,description=Extractors contains the extraction mechanism for the request to identify and extract parts of the response"`
+	// description: |
+	//   MatchersCondition is the condition between the matchers. Default is OR.
+	// values:
+	//   - "and"
+	//   - "or"
+	MatchersCondition string `yaml:"matchers-condition,omitempty" jsonschema:"title=condition between the matchers,description=Conditions between the matchers,enum=and,enum=or"`
 	// cached variables that may be used along with request.
 	matchersCondition matchers.ConditionType
 }
@@ -92,7 +102,7 @@ func (r *Result) Merge(result *Result) {
 // MatchFunc performs matching operation for a matcher on model and returns true or false.
 type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) bool
 
-// ExtractFunc performs extracting operation for a extractor on model and returns true or false.
+// ExtractFunc performs extracting operation for an extractor on model and returns true or false.
 type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
 
 // Execute executes the operators on data and returns a result structure
@@ -137,7 +147,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac
 				return nil, false
 			}
 		} else {
-			// If the matcher has matched, and its an OR
+			// If the matcher has matched, and it's an OR
 			// write the first output then move to next matcher.
 			if matcherCondition == matchers.ORCondition && matcher.Name != "" {
 				result.Matches[matcher.Name] = struct{}{}
@@ -151,7 +161,8 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac
 	if len(result.DynamicValues) > 0 {
 		return result, true
 	}
-	// Don't print if we have matchers and they have not matched, irregardless of extractor
+
+	// Don't print if we have matchers and they have not matched, regardless of extractor
 	if len(r.Matchers) > 0 && !matches {
 		return nil, false
 	}

v2/pkg/output/file_output_writer.go

@@ -20,11 +20,10 @@ func newFileOutputWriter(file string) (*fileWriter, error) {
 
 // WriteString writes an output to the underlying file
 func (w *fileWriter) Write(data []byte) error {
-	_, err := w.file.Write(data)
-	if err != nil {
+	if _, err := w.file.Write(data); err != nil {
 		return err
 	}
-	_, err = w.file.Write([]byte("\n"))
+	_, err := w.file.Write([]byte("\n"))
 	return err
 }
 

v2/pkg/output/format_json.go

@@ -6,5 +6,9 @@ import (
 
 // formatJSON formats the output for json based formatting
 func (w *StandardWriter) formatJSON(output *ResultEvent) ([]byte, error) {
+	if !w.jsonReqResp { // don't show request-response in json if not asked
+		output.Request = ""
+		output.Response = ""
+	}
 	return jsoniter.Marshal(output)
 }

v2/pkg/output/format_screen.go

@@ -11,10 +11,11 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
 	builder := &bytes.Buffer{}
 
 	if !w.noMetadata {
-		builder.WriteRune('[')
-		builder.WriteString(w.aurora.Cyan(output.Timestamp.Format("2006-01-02 15:04:05")).String())
-		builder.WriteString("] ")
-
+		if !w.noTimestamp {
+			builder.WriteRune('[')
+			builder.WriteString(w.aurora.Cyan(output.Timestamp.Format("2006-01-02 15:04:05")).String())
+			builder.WriteString("] ")
+		}
 		builder.WriteRune('[')
 		builder.WriteString(w.aurora.BrightGreen(output.TemplateID).String())
 
@@ -31,7 +32,7 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
 		builder.WriteString("] ")
 
 		builder.WriteString("[")
-		builder.WriteString(w.severityColors.Data[types.ToString(output.Info["severity"])])
+		builder.WriteString(w.severityColors(output.Info.SeverityHolder.Severity))
 		builder.WriteString("] ")
 	}
 	builder.WriteString(output.Matched)

v2/pkg/output/output.go

@@ -6,11 +6,15 @@ import (
 	"sync"
 	"time"
 
+	"github.com/pkg/errors"
+
 	jsoniter "github.com/json-iterator/go"
 	"github.com/logrusorgru/aurora"
-	"github.com/pkg/errors"
+
 	"github.com/projectdiscovery/interactsh/pkg/server"
 	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 )
 
@@ -29,13 +33,15 @@ type Writer interface {
 // StandardWriter is a writer writing output to file and screen for results.
 type StandardWriter struct {
 	json           bool
+	jsonReqResp    bool
+	noTimestamp    bool
 	noMetadata     bool
 	aurora         aurora.Aurora
 	outputFile     *fileWriter
 	outputMutex    *sync.Mutex
 	traceFile      *fileWriter
 	traceMutex     *sync.Mutex
-	severityColors *colorizer.Colorizer
+	severityColors func(severity.Severity) string
 }
 
 var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
@@ -57,7 +63,7 @@ type ResultEvent struct {
 	// TemplatePath is the path of template
 	TemplatePath string `json:"-"`
 	// Info contains information block of the template for the result.
-	Info map[string]interface{} `json:"info,inline"`
+	Info model.Info `json:"info,inline"`
 	// MatcherName is the name of the matcher matched if any.
 	MatcherName string `json:"matcher_name,omitempty"`
 	// ExtractorName is the name of the extractor matched if any.
@@ -72,9 +78,9 @@ type ResultEvent struct {
 	Matched string `json:"matched,omitempty"`
 	// ExtractedResults contains the extraction result from the inputs.
 	ExtractedResults []string `json:"extracted_results,omitempty"`
-	// Request is the optional dumped request for the match.
+	// Request is the optional, dumped request for the match.
 	Request string `json:"request,omitempty"`
-	// Response is the optional dumped response for the match.
+	// Response is the optional, dumped response for the match.
 	Response string `json:"response,omitempty"`
 	// Metadata contains any optional metadata for the event
 	Metadata map[string]interface{} `json:"meta,omitempty"`
@@ -89,7 +95,7 @@ type ResultEvent struct {
 }
 
 // NewStandardWriter creates a new output writer based on user configurations
-func NewStandardWriter(colors, noMetadata, json bool, file, traceFile string) (*StandardWriter, error) {
+func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, file, traceFile string) (*StandardWriter, error) {
 	auroraColorizer := aurora.NewAurora(colors)
 
 	var outputFile *fileWriter
@@ -110,7 +116,9 @@ func NewStandardWriter(colors, noMetadata, json bool, file, traceFile string) (*
 	}
 	writer := &StandardWriter{
 		json:           json,
+		jsonReqResp:    jsonReqResp,
 		noMetadata:     noMetadata,
+		noTimestamp:    noTimestamp,
 		aurora:         auroraColorizer,
 		outputFile:     outputFile,
 		outputMutex:    &sync.Mutex{},

v2/pkg/parsers/parser.go

@@ -0,0 +1,141 @@
+package parsers
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+
+	"gopkg.in/yaml.v2"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates/cache"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
+)
+
+const mandatoryFieldMissingTemplate = "mandatory '%s' field is missing"
+
+// LoadTemplate returns true if the template is valid and matches the filtering criteria.
+func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string) (bool, error) {
+	template, templateParseError := ParseTemplate(templatePath)
+	if templateParseError != nil {
+		return false, templateParseError
+	}
+
+	if len(template.Workflows) > 0 {
+		return false, nil
+	}
+
+	templateInfo := template.Info
+	if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
+		return false, validationError
+	}
+
+	return isTemplateInfoMetadataMatch(tagFilter, &templateInfo, extraTags)
+}
+
+// LoadWorkflow returns true if the workflow is valid and matches the filtering criteria.
+func LoadWorkflow(templatePath string) (bool, error) {
+	template, templateParseError := ParseTemplate(templatePath)
+	if templateParseError != nil {
+		return false, templateParseError
+	}
+
+	templateInfo := template.Info
+
+	if len(template.Workflows) > 0 {
+		if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
+			return false, validationError
+		}
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string) (bool, error) {
+	templateTags := templateInfo.Tags.ToSlice()
+	templateAuthors := templateInfo.Authors.ToSlice()
+	templateSeverity := templateInfo.SeverityHolder.Severity
+
+	match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags)
+
+	if err == filter.ErrExcluded {
+		return false, filter.ErrExcluded
+	}
+
+	return match, err
+}
+
+func validateMandatoryInfoFields(info *model.Info) error {
+	if info == nil {
+		return fmt.Errorf(mandatoryFieldMissingTemplate, "info")
+	}
+
+	if utils.IsBlank(info.Name) {
+		return fmt.Errorf(mandatoryFieldMissingTemplate, "name")
+	}
+
+	if info.Authors.IsEmpty() {
+		return fmt.Errorf(mandatoryFieldMissingTemplate, "author")
+	}
+	return nil
+}
+
+var (
+	parsedTemplatesCache *cache.Templates
+	ShouldValidate       bool
+	fieldErrorRegexp     = regexp.MustCompile(`not found in`)
+)
+
+const (
+	SyntaxWarningStats = "syntax-warnings"
+	SyntaxErrorStats   = "syntax-errors"
+)
+
+func init() {
+
+	parsedTemplatesCache = cache.New()
+
+	stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)")
+	stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)")
+}
+
+// ParseTemplate parses a template and returns a *templates.Template structure
+func ParseTemplate(templatePath string) (*templates.Template, error) {
+	if value, err := parsedTemplatesCache.Has(templatePath); value != nil {
+		return value.(*templates.Template), err
+	}
+
+	f, err := os.Open(templatePath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+
+	template := &templates.Template{}
+	if err := yaml.UnmarshalStrict(data, template); err != nil {
+		errString := err.Error()
+		if !fieldErrorRegexp.MatchString(errString) {
+			stats.Increment(SyntaxErrorStats)
+			return nil, err
+		}
+		stats.Increment(SyntaxWarningStats)
+		if ShouldValidate {
+			gologger.Error().Msgf("Syntax warnings for template %s: %s", templatePath, err)
+		} else {
+			gologger.Warning().Msgf("Syntax warnings for template %s: %s", templatePath, err)
+		}
+	}
+	parsedTemplatesCache.Store(templatePath, template, nil)
+	return template, nil
+}

v2/pkg/parsers/workflow_loader.go

@@ -0,0 +1,62 @@
+package parsers
+
+import (
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+)
+
+type workflowLoader struct {
+	pathFilter *filter.PathFilter
+	tagFilter  *filter.TagFilter
+	options    *protocols.ExecuterOptions
+}
+
+// NewLoader returns a new workflow loader structure
+func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error) {
+	tagFilter := filter.New(&filter.Config{
+		Tags:        options.Options.Tags,
+		ExcludeTags: options.Options.ExcludeTags,
+		Authors:     options.Options.Author,
+		Severities:  options.Options.Severities,
+		IncludeTags: options.Options.IncludeTags,
+	})
+	pathFilter := filter.NewPathFilter(&filter.PathFilterConfig{
+		IncludedTemplates: options.Options.IncludeTemplates,
+		ExcludedTemplates: options.Options.ExcludedTemplates,
+	}, options.Catalog)
+	return &workflowLoader{pathFilter: pathFilter, tagFilter: tagFilter, options: options}, nil
+}
+
+func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string {
+	includedTemplates := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory})
+	templatePathMap := w.pathFilter.Match(includedTemplates)
+
+	loadedTemplates := make([]string, 0, len(templatePathMap))
+	for templatePath := range templatePathMap {
+		loaded, err := LoadTemplate(templatePath, w.tagFilter, templateTags)
+		if err != nil {
+			gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
+		} else if loaded {
+			loadedTemplates = append(loadedTemplates, templatePath)
+		}
+	}
+	return loadedTemplates
+}
+
+func (w *workflowLoader) GetTemplatePaths(templatesList []string, noValidate bool) []string {
+	includedTemplates := w.options.Catalog.GetTemplatesPath(templatesList)
+	templatesPathMap := w.pathFilter.Match(includedTemplates)
+
+	loadedTemplates := make([]string, 0, len(templatesPathMap))
+	for templatePath := range templatesPathMap {
+		matched, err := LoadTemplate(templatePath, w.tagFilter, nil)
+		if err != nil {
+			gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
+		} else if matched || noValidate {
+			loadedTemplates = append(loadedTemplates, templatePath)
+		}
+	}
+	return loadedTemplates
+}

v2/pkg/projectfile/httputil.go

@@ -18,8 +18,7 @@ func hash(v interface{}) (string, error) {
 
 	sh := sha256.New()
 
-	_, err = io.WriteString(sh, string(data))
-	if err != nil {
+	if _, err = io.WriteString(sh, string(data)); err != nil {
 		return "", err
 	}
 	return hex.EncodeToString(sh.Sum(nil)), nil
@@ -28,8 +27,7 @@ func hash(v interface{}) (string, error) {
 func marshal(data interface{}) ([]byte, error) {
 	var b bytes.Buffer
 	enc := gob.NewEncoder(&b)
-	err := enc.Encode(data)
-	if err != nil {
+	if err := enc.Encode(data); err != nil {
 		return nil, err
 	}
 
@@ -38,8 +36,7 @@ func marshal(data interface{}) ([]byte, error) {
 
 func unmarshal(data []byte, obj interface{}) error {
 	dec := gob.NewDecoder(bytes.NewBuffer(data))
-	err := dec.Decode(obj)
-	if err != nil {
+	if err := dec.Decode(obj); err != nil {
 		return err
 	}
 

v2/pkg/projectfile/project.go

@@ -44,8 +44,7 @@ func (pf *ProjectFile) Get(req []byte) (*http.Response, error) {
 
 	var httprecord HTTPRecord
 	httprecord.Response = newInternalResponse()
-	err = unmarshal(data, &httprecord)
-	if err != nil {
+	if err := unmarshal(data, &httprecord); err != nil {
 		return nil, err
 	}
 

v2/pkg/protocols/common/clusterer/executer.go

@@ -2,6 +2,7 @@ package clusterer
 
 import (
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
@@ -23,7 +24,7 @@ type Executer struct {
 type clusteredOperator struct {
 	templateID   string
 	templatePath string
-	templateInfo map[string]interface{}
+	templateInfo model.Info
 	operator     *operators.Operators
 }
 
@@ -86,6 +87,9 @@ func (e *Executer) Execute(input string) (bool, error) {
 			}
 		}
 	})
+	if err != nil && e.options.HostErrorsCache != nil && e.options.HostErrorsCache.CheckError(err) {
+		e.options.HostErrorsCache.MarkFailed(input)
+	}
 	return results, err
 }
 
@@ -105,5 +109,8 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve
 			}
 		}
 	})
+	if err != nil && e.options.HostErrorsCache != nil && e.options.HostErrorsCache.CheckError(err) {
+		e.options.HostErrorsCache.MarkFailed(input)
+	}
 	return err
 }

v2/pkg/protocols/common/executer/executer.go

@@ -24,8 +24,7 @@ func NewExecuter(requests []protocols.Request, options *protocols.ExecuterOption
 // Compile compiles the execution generators preparing any requests possible.
 func (e *Executer) Compile() error {
 	for _, request := range e.requests {
-		err := request.Compile(e.options)
-		if err != nil {
+		if err := request.Compile(e.options); err != nil {
 			return err
 		}
 	}
@@ -77,6 +76,11 @@ func (e *Executer) Execute(input string) (bool, error) {
 			}
 		})
 		if err != nil {
+			if e.options.HostErrorsCache != nil {
+				if e.options.HostErrorsCache.CheckError(err) {
+					e.options.HostErrorsCache.MarkFailed(input)
+				}
+			}
 			gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err)
 		}
 	}
@@ -109,6 +113,11 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve
 			callback(event)
 		})
 		if err != nil {
+			if e.options.HostErrorsCache != nil {
+				if e.options.HostErrorsCache.CheckError(err) {
+					e.options.HostErrorsCache.MarkFailed(input)
+				}
+			}
 			gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err)
 		}
 	}

v2/pkg/protocols/common/generators/env.go

@@ -0,0 +1,28 @@
+package generators
+
+import (
+	"os"
+
+	"github.com/projectdiscovery/stringsutil"
+)
+
+var envVars map[string]interface{}
+
+func parseEnvVars() map[string]interface{} {
+	sliceEnvVars := os.Environ()
+	parsedEnvVars := make(map[string]interface{}, len(sliceEnvVars))
+	for _, envVar := range sliceEnvVars {
+		key, val := stringsutil.Before(envVar, "="), stringsutil.After(envVar, "=")
+		parsedEnvVars[key] = val
+	}
+	return parsedEnvVars
+}
+
+// EnvVars returns a map with all environment variables into a map
+func EnvVars() map[string]interface{} {
+	if envVars == nil {
+		envVars = parseEnvVars()
+	}
+
+	return envVars
+}

v2/pkg/protocols/common/generators/generators.go

@@ -2,10 +2,6 @@
 
 package generators
 
-import (
-	"errors"
-)
-
 // Generator is the generator struct for generating payloads
 type Generator struct {
 	Type     Type
@@ -16,7 +12,7 @@ type Generator struct {
 type Type int
 
 const (
-	// Sniper replaces each variables with values at a time.
+	// Sniper replaces each variable with values at a time.
 	Sniper Type = iota + 1
 	// PitchFork replaces variables with positional value from multiple wordlists
 	PitchFork
@@ -24,7 +20,7 @@ const (
 	ClusterBomb
 )
 
-// StringToType is an table for conversion of attack type from string.
+// StringToType is a table for conversion of attack type from string.
 var StringToType = map[string]Type{
 	"sniper":      Sniper,
 	"pitchfork":   PitchFork,
@@ -45,16 +41,6 @@ func New(payloads map[string]interface{}, payloadType Type, templatePath string)
 	generator.Type = payloadType
 	generator.payloads = compiled
 
-	// Validate the payload types
-	if payloadType == PitchFork {
-		var totalLength int
-		for v := range compiled {
-			if totalLength != 0 && totalLength != len(v) {
-				return nil, errors.New("pitchfork payloads must be of equal number")
-			}
-			totalLength = len(v)
-		}
-	}
 	return generator, nil
 }
 
@@ -107,6 +93,11 @@ func (i *Iterator) Total() int {
 		}
 	case PitchFork:
 		count = len(i.payloads[0].values)
+		for _, p := range i.payloads {
+			if count > len(p.values) {
+				count = len(p.values)
+			}
+		}
 	case ClusterBomb:
 		count = 1
 		for _, p := range i.payloads {

v2/pkg/protocols/common/generators/generators_test.go

@@ -27,7 +27,7 @@ func TestSniperGenerator(t *testing.T) {
 
 func TestPitchforkGenerator(t *testing.T) {
 	usernames := []string{"admin", "token"}
-	passwords := []string{"admin", "password"}
+	passwords := []string{"password1", "password2", "password3"}
 
 	generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchFork, "")
 	require.Nil(t, err, "could not create generator")
@@ -43,7 +43,7 @@ func TestPitchforkGenerator(t *testing.T) {
 		require.Contains(t, usernames, value["username"], "Could not get correct pitchfork username")
 		require.Contains(t, passwords, value["password"], "Could not get correct pitchfork password")
 	}
-	require.Equal(t, len(passwords), count, "could not get correct pitchfork counts")
+	require.Equal(t, len(usernames), count, "could not get correct pitchfork counts")
 }
 
 func TestClusterbombGenerator(t *testing.T) {

v2/pkg/protocols/common/generators/maps.go

@@ -1,6 +1,8 @@
 package generators
 
-import "strings"
+import (
+	"strings"
+)
 
 // MergeMaps merges two maps into a new map
 func MergeMaps(m1, m2 map[string]interface{}) map[string]interface{} {
@@ -14,7 +16,7 @@ func MergeMaps(m1, m2 map[string]interface{}) map[string]interface{} {
 	return m
 }
 
-// ExpandMapValues converts values from flat string to strings slice
+// ExpandMapValues converts values from flat string to string slice
 func ExpandMapValues(m map[string]string) map[string][]string {
 	m1 := make(map[string][]string, len(m))
 	for k, v := range m {

v2/pkg/protocols/common/generators/slice.go

@@ -0,0 +1,16 @@
+package generators
+
+import "github.com/projectdiscovery/stringsutil"
+
+// SliceToMap converts a slice of strings to map of string splitting each item at sep as "key sep value"
+func SliceToMap(s []string, sep string) map[string]interface{} {
+	m := make(map[string]interface{})
+	for _, sliceItem := range s {
+		key := stringsutil.Before(sliceItem, sep)
+		value := stringsutil.After(sliceItem, sep)
+		if key != "" {
+			m[key] = value
+		}
+	}
+	return m
+}

v2/pkg/protocols/common/generators/validate.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
@@ -26,10 +26,10 @@ func (g *Generator) validate(payloads map[string]interface{}, templatePath strin
 			}
 
 			changed := false
-			pathTokens := strings.Split(templatePath, "/")
+			pathTokens := strings.Split(templatePath, string(os.PathSeparator))
 
 			for i := range pathTokens {
-				tpath := path.Join(strings.Join(pathTokens[:i], "/"), pt)
+				tpath := filepath.Join(filepath.Join(pathTokens[:i]...), pt)
 				if fileExists(tpath) {
 					payloads[name] = tpath
 					changed = true

v2/pkg/protocols/common/helpers/deserialization/java.go

@@ -58,6 +58,8 @@ func gadgetEncodingHelper(returnData []byte, encoding string) string {
 			return ""
 		}
 		return urlsafeBase64Encode(buffer.Bytes())
+	case "base64-raw":
+		return base64.StdEncoding.EncodeToString(returnData)
 	default:
 		return urlsafeBase64Encode(returnData)
 	}

v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.java

@@ -1,6 +1,6 @@
 import java.io.*;
 
-class Deserialize{  
+class Deserialize {  
     public static void main(String args[]) {
         FileInputStream fileIn = null;
         ObjectInputStream in = null;

v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go

@@ -0,0 +1,126 @@
+package hosterrorscache
+
+import (
+	"net"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/bluele/gcache"
+	"github.com/projectdiscovery/gologger"
+)
+
+// Cache is a cache for host based errors. It allows skipping
+// certain hosts based on an error threshold.
+//
+// It uses an LRU cache internally for skipping unresponsive hosts
+// that remain so for a duration.
+type Cache struct {
+	MaxHostError int
+	verbose       bool
+	failedTargets gcache.Cache
+}
+
+const DefaultMaxHostsCount = 10000
+
+// New returns a new host max errors cache
+func New(MaxHostError, maxHostsCount int) *Cache {
+	gc := gcache.New(maxHostsCount).
+		ARC().
+		Build()
+	return &Cache{failedTargets: gc, MaxHostError: MaxHostError}
+}
+
+// SetVerbose sets the cache to log at verbose level
+func (c *Cache) SetVerbose(verbose bool) *Cache {
+	c.verbose = verbose
+	return c
+}
+
+// Close closes the host errors cache
+func (c *Cache) Close() {
+	c.failedTargets.Purge()
+}
+
+func (c *Cache) normalizeCacheValue(value string) string {
+	finalValue := value
+	if strings.HasPrefix(value, "http") {
+		if parsed, err := url.Parse(value); err == nil {
+
+			hostname := parsed.Host
+			finalPort := parsed.Port()
+			if finalPort == "" {
+				if parsed.Scheme == "https" {
+					finalPort = "443"
+				} else {
+					finalPort = "80"
+				}
+				hostname = net.JoinHostPort(parsed.Host, finalPort)
+			}
+			finalValue = hostname
+		}
+	}
+	return finalValue
+}
+
+// ErrUnresponsiveHost is returned when a host is unresponsive
+//var ErrUnresponsiveHost = errors.New("skipping as host is unresponsive")
+
+// Check returns true if a host should be skipped as it has been
+// unresponsive for a certain number of times.
+//
+// The value can be many formats -
+//   - URL: https?:// type
+//   - Host:port type
+//   - host type
+func (c *Cache) Check(value string) bool {
+	finalValue := c.normalizeCacheValue(value)
+	if !c.failedTargets.Has(finalValue) {
+		return false
+	}
+
+	numberOfErrors, err := c.failedTargets.GetIFPresent(finalValue)
+	if err != nil {
+		return false
+	}
+	numberOfErrorsValue := numberOfErrors.(int)
+
+	if numberOfErrors == -1 {
+		return true
+	}
+	if numberOfErrorsValue >= c.MaxHostError {
+		_ = c.failedTargets.Set(finalValue, -1)
+		if c.verbose {
+			gologger.Verbose().Msgf("Skipping %s as previously unresponsive %d times", finalValue, numberOfErrorsValue)
+		}
+		return true
+	}
+	return false
+}
+
+// MarkFailed marks a host as failed previously
+func (c *Cache) MarkFailed(value string) {
+	finalValue := c.normalizeCacheValue(value)
+	if !c.failedTargets.Has(finalValue) {
+		_ = c.failedTargets.Set(finalValue, 1)
+		return
+	}
+
+	numberOfErrors, err := c.failedTargets.GetIFPresent(finalValue)
+	if err != nil || numberOfErrors == nil {
+		_ = c.failedTargets.Set(finalValue, 1)
+		return
+	}
+	numberOfErrorsValue := numberOfErrors.(int)
+
+	_ = c.failedTargets.Set(finalValue, numberOfErrorsValue+1)
+}
+
+var checkErrorRegexp = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host)`)
+
+// CheckError checks if an error represents a type that should be
+// added to the host skipping table.
+func (c *Cache) CheckError(err error) bool {
+	errString := err.Error()
+	return checkErrorRegexp.MatchString(errString)
+}

v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go

@@ -0,0 +1,30 @@
+package hosterrorscache
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestCacheCheckMarkFailed(t *testing.T) {
+	cache := New(3, DefaultMaxHostsCount)
+
+	cache.MarkFailed("http://example.com:80")
+	if value, err := cache.failedTargets.Get("http://example.com:80"); err == nil && value != nil {
+		require.Equal(t, 1, value, "could not get correct markfailed")
+	}
+	cache.MarkFailed("example.com:80")
+	if value, err := cache.failedTargets.Get("example.com:80"); err == nil && value != nil {
+		require.Equal(t, 2, value, "could not get correct markfailed")
+	}
+	cache.MarkFailed("example.com")
+	if value, err := cache.failedTargets.Get("example.com"); err == nil && value != nil {
+		require.Equal(t, 1, value, "could not get correct markfailed")
+	}
+	for i := 0; i < 3; i++ {
+		cache.MarkFailed("test")
+	}
+
+	value := cache.Check("test")
+	require.Equal(t, true, value, "could not get checked value")
+}

v2/pkg/protocols/common/interactsh/interactsh.go

@@ -1,13 +1,18 @@
 package interactsh
 
 import (
+	"bytes"
+	"fmt"
 	"net/url"
+	"os"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/karlseguin/ccache"
 	"github.com/pkg/errors"
+
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/interactsh/pkg/client"
 	"github.com/projectdiscovery/interactsh/pkg/server"
@@ -32,8 +37,9 @@ type Client struct {
 	pollDuration     time.Duration
 	cooldownDuration time.Duration
 
-	generated uint32 // decide to wait if we have a generated url
-	matched   bool
+	firstTimeGroup sync.Once
+	generated      uint32 // decide to wait if we have a generated url
+	matched        bool
 }
 
 var (
@@ -45,6 +51,8 @@ var (
 type Options struct {
 	// ServerURL is the URL of the interactsh server.
 	ServerURL string
+	// Authorization is the Authorization header value
+	Authorization string
 	// CacheSize is the numbers of requests to keep track of at a time.
 	// Older items are discarded in LRU manner in favor of new requests.
 	CacheSize int64
@@ -62,6 +70,8 @@ type Options struct {
 	IssuesClient *reporting.Client
 	// Progress is the nuclei progress bar implementation.
 	Progress progress.Progress
+	// Debug specifies whether debugging output should be shown for interactsh-client
+	Debug bool
 }
 
 const defaultMaxInteractionsCount = 5000
@@ -73,13 +83,6 @@ func New(options *Options) (*Client, error) {
 		return nil, errors.Wrap(err, "could not parse server url")
 	}
 
-	interactsh, err := client.New(&client.Options{
-		ServerURL:         options.ServerURL,
-		PersistentSession: false,
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "could not create client")
-	}
 	configure := ccache.Configure()
 	configure = configure.MaxSize(options.CacheSize)
 	cache := ccache.New(configure)
@@ -89,7 +92,6 @@ func New(options *Options) (*Client, error) {
 	interactionsCache := ccache.New(interactionsCfg)
 
 	interactClient := &Client{
-		interactsh:       interactsh,
 		eviction:         options.Eviction,
 		interactions:     interactionsCache,
 		dotHostname:      "." + parsed.Host,
@@ -98,18 +100,34 @@ func New(options *Options) (*Client, error) {
 		pollDuration:     options.PollDuration,
 		cooldownDuration: options.ColldownPeriod,
 	}
+	return interactClient, nil
+}
 
-	interactClient.interactsh.StartPolling(interactClient.pollDuration, func(interaction *server.Interaction) {
-		item := interactClient.requests.Get(interaction.UniqueID)
+func (c *Client) firstTimeInitializeClient() error {
+	interactsh, err := client.New(&client.Options{
+		ServerURL:         c.options.ServerURL,
+		Token:             c.options.Authorization,
+		PersistentSession: false,
+	})
+	if err != nil {
+		return errors.Wrap(err, "could not create client")
+	}
+	c.interactsh = interactsh
+
+	interactsh.StartPolling(c.pollDuration, func(interaction *server.Interaction) {
+		if c.options.Debug {
+			debugPrintInteraction(interaction)
+		}
+		item := c.requests.Get(interaction.UniqueID)
 		if item == nil {
 			// If we don't have any request for this ID, add it to temporary
-			// lru cache so we can correlate when we get an add request.
-			gotItem := interactClient.interactions.Get(interaction.UniqueID)
+			// lru cache, so we can correlate when we get an add request.
+			gotItem := c.interactions.Get(interaction.UniqueID)
 			if gotItem == nil {
-				interactClient.interactions.Set(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration)
+				c.interactions.Set(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration)
 			} else if items, ok := gotItem.Value().([]*server.Interaction); ok {
 				items = append(items, interaction)
-				interactClient.interactions.Set(interaction.UniqueID, items, defaultInteractionDuration)
+				c.interactions.Set(interaction.UniqueID, items, defaultInteractionDuration)
 			}
 			return
 		}
@@ -117,9 +135,9 @@ func New(options *Options) (*Client, error) {
 		if !ok {
 			return
 		}
-		_ = interactClient.processInteractionForRequest(interaction, request)
+		_ = c.processInteractionForRequest(interaction, request)
 	})
-	return interactClient, nil
+	return nil
 }
 
 // processInteractionForRequest processes an interaction for a request
@@ -159,6 +177,14 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
 
 // URL returns a new URL that can be interacted with
 func (c *Client) URL() string {
+	c.firstTimeGroup.Do(func() {
+		if err := c.firstTimeInitializeClient(); err != nil {
+			gologger.Error().Msgf("Could not initialize interactsh client: %s", err)
+		}
+	})
+	if c.interactsh == nil {
+		return ""
+	}
 	atomic.CompareAndSwapUint32(&c.generated, 0, 1)
 	return c.interactsh.URL()
 }
@@ -168,8 +194,10 @@ func (c *Client) Close() bool {
 	if c.cooldownDuration > 0 && atomic.LoadUint32(&c.generated) == 1 {
 		time.Sleep(c.cooldownDuration)
 	}
-	c.interactsh.StopPolling()
-	c.interactsh.Close()
+	if c.interactsh != nil {
+		c.interactsh.StopPolling()
+		c.interactsh.Close()
+	}
 	return c.matched
 }
 
@@ -252,3 +280,20 @@ func HasMatchers(op *operators.Operators) bool {
 	}
 	return false
 }
+
+func debugPrintInteraction(interaction *server.Interaction) {
+	builder := &bytes.Buffer{}
+
+	switch interaction.Protocol {
+	case "dns":
+		builder.WriteString(fmt.Sprintf("[%s] Received DNS interaction (%s) from %s at %s", interaction.FullId, interaction.QType, interaction.RemoteAddress, interaction.Timestamp.Format("2006-01-02 15:04:05")))
+		builder.WriteString(fmt.Sprintf("\n-----------\nDNS Request\n-----------\n\n%s\n\n------------\nDNS Response\n------------\n\n%s\n\n", interaction.RawRequest, interaction.RawResponse))
+	case "http":
+		builder.WriteString(fmt.Sprintf("[%s] Received HTTP interaction from %s at %s", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp.Format("2006-01-02 15:04:05")))
+		builder.WriteString(fmt.Sprintf("\n------------\nHTTP Request\n------------\n\n%s\n\n-------------\nHTTP Response\n-------------\n\n%s\n\n", interaction.RawRequest, interaction.RawResponse))
+	case "smtp":
+		builder.WriteString(fmt.Sprintf("[%s] Received SMTP interaction from %s at %s", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp.Format("2006-01-02 15:04:05")))
+		builder.WriteString(fmt.Sprintf("\n------------\nSMTP Interaction\n------------\n\n%s\n\n", interaction.RawRequest))
+	}
+	fmt.Fprint(os.Stderr, builder.String())
+}

v2/pkg/protocols/dns/dns.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/miekg/dns"
 	"github.com/pkg/errors"
+
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
@@ -18,18 +19,49 @@ type Request struct {
 	// Operators for the current request go here.
 	operators.Operators `yaml:",inline"`
 
-	ID string `yaml:"id"`
+	// ID is the optional id of the request
+	ID string `yaml:"id,omitempty" jsonschema:"title=id of the dns request,description=ID is the optional ID of the DNS Request"`
 
-	// Path contains the path/s for the request
-	Name string `yaml:"name"`
-	// Type is the type of DNS request to make
-	Type string `yaml:"type"`
-	// Class is the class of the DNS request
-	Class string `yaml:"class"`
-	// Retries is the number of retries for the DNS request
-	Retries int `yaml:"retries"`
+	// description: |
+	//   Name is the Hostname to make DNS request for.
+	//
+	//   Generally, it is set to {{FQDN}} which is the domain we get from input.
+	// examples:
+	//   - value: "\"{{FQDN}}\""
+	Name string `yaml:"name,omitempty" jsonschema:"title=hostname to make dns request for,description=Name is the Hostname to make DNS request for"`
+	// description: |
+	//   Type is the type of DNS request to make.
+	// values:
+	//   - "A"
+	//   - "NS"
+	//   - "DS"
+	//   - "CNAME"
+	//   - "SOA"
+	//   - "PTR"
+	//   - "MX"
+	//   - "TXT"
+	//   - "AAAA"
+	Type string `yaml:"type,omitempty" jsonschema:"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"`
+	// description: |
+	//   Class is the class of the DNS request.
+	//
+	//   Usually it's enough to just leave it as INET.
+	// values:
+	//   - "inet"
+	//   - "csnet"
+	//   - "chaos"
+	//   - "hesiod"
+	//   - "none"
+	//   - "any"
+	Class string `yaml:"class,omitempty" jsonschema:"title=class of DNS request,description=Class is the class of the DNS request,enum=inet,enum=csnet,enum=chaos,enum=hesiod,enum=none,enum=any"`
+	// description: |
+	//   Retries is the number of retries for the DNS request
+	// examples:
+	//   - name: Use a retry of 3 to 5 generally
+	//     value: 5
+	Retries int `yaml:"retries,omitempty" jsonschema:"title=retries for dns request,description=Retries is the number of retries for the DNS request"`
 
-	CompiledOperators *operators.Operators
+	CompiledOperators *operators.Operators `yaml:"-"`
 	dnsClient         *retryabledns.Client
 	options           *protocols.ExecuterOptions
 
@@ -37,8 +69,9 @@ type Request struct {
 	class    uint16
 	question uint16
 
-	// Recursion specifies whether to recurse all the answers.
-	Recursion bool `yaml:"recursion"`
+	// description: |
+	//   Recursion determines if resolver should recurse all records to get fresh results.
+	Recursion bool `yaml:"recursion,omitempty" jsonschema:"title=recurse all servers,description=Recursion determines if resolver should recurse all records to get fresh results"`
 }
 
 // GetID returns the unique ID of the request if any.
@@ -126,7 +159,7 @@ func questionTypeToInt(questionType string) uint16 {
 	return question
 }
 
-// classToInt converts a dns class name to it's internal representation
+// classToInt converts a dns class name to its internal representation
 func classToInt(class string) uint16 {
 	class = strings.TrimSpace(strings.ToUpper(class))
 	result := dns.ClassINET

v2/pkg/protocols/dns/dns_test.go

@@ -3,8 +3,11 @@ package dns
 import (
 	"testing"
 
-	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
 	"github.com/stretchr/testify/require"
+
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 )
 
 func TestDNSCompileMake(t *testing.T) {
@@ -22,7 +25,7 @@ func TestDNSCompileMake(t *testing.T) {
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
-		Info: map[string]interface{}{"severity": "low", "name": "test"},
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
 	})
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile dns request")

v2/pkg/protocols/dns/dnsclientpool/clientpool.go

@@ -25,7 +25,7 @@ var defaultResolvers = []string{
 
 // Init initializes the clientpool implementation
 func Init(options *types.Options) error {
-	// Don't create clients if already created in past.
+	// Don't create clients if already created in the past.
 	if normalClient != nil {
 		return nil
 	}

v2/pkg/protocols/dns/operators.go

@@ -5,6 +5,8 @@ import (
 	"time"
 
 	"github.com/miekg/dns"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
@@ -41,7 +43,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher)
 	return false
 }
 
-// Extract performs extracting operation for a extractor on model and returns true or false.
+// Extract performs extracting operation for an extractor on model and returns true or false.
 func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
 	part := extractor.Part
 	switch part {
@@ -139,16 +141,14 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),
 		TemplatePath:     types.ToString(wrapped.InternalEvent["template-path"]),
-		Info:             wrapped.InternalEvent["template-info"].(map[string]interface{}),
+		Info:             wrapped.InternalEvent["template-info"].(model.Info),
 		Type:             "dns",
 		Host:             types.ToString(wrapped.InternalEvent["host"]),
 		Matched:          types.ToString(wrapped.InternalEvent["matched"]),
 		ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
 		Timestamp:        time.Now(),
-	}
-	if r.options.Options.JSONRequests {
-		data.Request = types.ToString(wrapped.InternalEvent["request"])
-		data.Response = types.ToString(wrapped.InternalEvent["raw"])
+		Request:          types.ToString(wrapped.InternalEvent["request"]),
+		Response:         types.ToString(wrapped.InternalEvent["raw"]),
 	}
 	return data
 }

v2/pkg/protocols/dns/operators_test.go

@@ -6,12 +6,15 @@ import (
 	"testing"
 
 	"github.com/miekg/dns"
+	"github.com/stretchr/testify/require"
+
 	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/stretchr/testify/require"
 )
 
 func TestResponseToDSLMap(t *testing.T) {
@@ -29,7 +32,7 @@ func TestResponseToDSLMap(t *testing.T) {
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
-		Info: map[string]interface{}{"severity": "low", "name": "test"},
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
 	})
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile dns request")
@@ -61,7 +64,7 @@ func TestDNSOperatorMatch(t *testing.T) {
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
-		Info: map[string]interface{}{"severity": "low", "name": "test"},
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
 	})
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile dns request")
@@ -144,7 +147,7 @@ func TestDNSOperatorExtract(t *testing.T) {
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
-		Info: map[string]interface{}{"severity": "low", "name": "test"},
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
 	})
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile dns request")
@@ -214,7 +217,7 @@ func TestDNSMakeResult(t *testing.T) {
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
-		Info: map[string]interface{}{"severity": "low", "name": "test"},
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
 	})
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile dns request")

v2/pkg/protocols/dns/request.go

@@ -12,7 +12,7 @@ import (
 var _ protocols.Request = &Request{}
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
-func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
+func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	// Parse the URL and return domain if URL.
 	var domain string
 	if isURL(input) {
@@ -71,8 +71,7 @@ func (r *Request) ExecuteWithResults(input string, metadata, previous output.Int
 
 // isURL tests a string to determine if it is a well-structured url or not.
 func isURL(toTest string) bool {
-	_, err := url.ParseRequestURI(toTest)
-	if err != nil {
+	if _, err := url.ParseRequestURI(toTest); err != nil {
 		return false
 	}
 	u, err := url.Parse(toTest)

v2/pkg/protocols/dns/request_test.go

@@ -3,12 +3,15 @@ package dns
 import (
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/stretchr/testify/require"
 )
 
 func TestDNSExecuteWithResults(t *testing.T) {
@@ -39,7 +42,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
-		Info: map[string]interface{}{"severity": "low", "name": "test"},
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
 	})
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile dns request")

v2/pkg/protocols/file/file.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 )
@@ -12,26 +13,41 @@ import (
 type Request struct {
 	// Operators for the current request go here.
 	operators.Operators `yaml:",inline"`
-	// Extensions is the list of extensions to perform matching on.
-	Extensions []string `yaml:"extensions"`
-	// ExtensionDenylist is the list of file extensions to deny during matching.
-	ExtensionDenylist []string `yaml:"denylist"`
+	// description: |
+	//   Extensions is the list of extensions to perform matching on.
+	// examples:
+	//   - value: '[]string{".txt", ".go", ".json"}'
+	Extensions []string `yaml:"extensions,omitempty" jsonschema:"title=extensions to match,description=List of extensions to perform matching on"`
+	// description: |
+	//   ExtensionDenylist is the list of file extensions to deny during matching.
+	//
+	//   By default, it contains some non-interesting extensions that are hardcoded
+	//   in nuclei.
+	// examples:
+	//   - value: '[]string{".avi", ".mov", ".mp3"}'
+	ExtensionDenylist []string `yaml:"denylist,omitempty" jsonschema:"title=extensions to deny match,description=List of file extensions to deny during matching"`
 
-	ID string `yaml:"id"`
+	// ID is the optional id of the request
+	ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=ID is the optional ID for the request"`
 
-	// MaxSize is the maximum size of the file to run request on.
-	// By default, nuclei will process 5MB files and not go more than that.
-	// It can be set to much lower or higher depending on use.
-	MaxSize           int `yaml:"max-size"`
-	CompiledOperators *operators.Operators
+	// description: |
+	//   MaxSize is the maximum size of the file to run request on.
+	//
+	//   By default, nuclei will process 5 MB files and not go more than that.
+	//   It can be set to much lower or higher depending on use.
+	// examples:
+	//   - value: 2048
+	MaxSize           int                  `yaml:"max-size,omitempty" jsonschema:"title=max size data to run request on,description=Maximum size of the file to run request on"`
+	CompiledOperators *operators.Operators `yaml:"-"`
 
 	// cache any variables that may be needed for operation.
 	options           *protocols.ExecuterOptions
 	extensions        map[string]struct{}
 	extensionDenylist map[string]struct{}
 
-	// NoRecursive specifies whether to not do recursive checks if folders are provided.
-	NoRecursive bool `yaml:"no-recursive"`
+	// description: |
+	//   NoRecursive specifies whether to not do recursive checks if folders are provided.
+	NoRecursive bool `yaml:"no-recursive,omitempty" jsonschema:"title=do not perform recursion,description=Specifies whether to not do recursive checks if folders are provided"`
 
 	allExtensions bool
 }
@@ -53,7 +69,7 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
 		}
 		r.CompiledOperators = compiled
 	}
-	// By default use 5mb as max size to read.
+	// By default, use 5 MB as max size to read.
 	if r.MaxSize == 0 {
 		r.MaxSize = 5 * 1024 * 1024
 	}