Bug fix

v2.3.1 bug fixes

.github/dependabot.yml

@@ -0,0 +1,34 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "chore"
+      include: "scope"
+
+  # Maintain dependencies for go modules
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "chore"
+      include: "scope"
+
+  # Maintain dependencies for docker
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "chore"
+      include: "scope"

.github/workflows/build.yaml

@@ -13,22 +13,13 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v1
+        uses: golangci/golangci-lint-action@v2
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.31
+          version: v1.33
           args: --timeout 5m
           working-directory: v2/
-
-          # Optional: working directory, useful for monorepos
-          # working-directory: somedir
-
-          # Optional: golangci-lint command line arguments.
-          # args: --issues-exit-code=0
-
-          # Optional: show only new issues if it's a pull request. The default value is `false`.
-          # only-new-issues: true
-
+        
   build:
     name: Build
     runs-on: ubuntu-latest
@@ -36,14 +27,18 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.14
+          go-version: 1.15
 
       - name: Check out code
         uses: actions/checkout@v2
 
       - name: Test
-        run: go test .
-        working-directory: v2/cmd/nuclei/
+        run: go test ./...
+        working-directory: v2/
+
+      - name: Integration Tests
+        run: bash run.sh
+        working-directory: integration_tests/
 
       - name: Build
         run: go build .

.github/workflows/dockerhub-push.yml

@@ -5,13 +5,15 @@ on:
   release:
     types: [published]
 jobs:
-  build:
+  update:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@master
     - name: Publish to Dockerhub Registry
+      #pre: echo ::save-state name=RELEASE_VERSION::$(echo ${GITHUB_REF:10})
       uses: elgohr/Publish-Docker-Github-Action@master
       with:
         name: projectdiscovery/nuclei
         username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        tags: "latest" #"latest,${{ env.STATE_RELEASE_VERSION }}"

.github/workflows/release.yml

@@ -17,7 +17,7 @@ jobs:
         name: "Set up Go"
         uses: actions/setup-go@v2
         with: 
-          go-version: 1.14
+          go-version: 1.15
       - 
         env: 
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
@@ -25,4 +25,6 @@ jobs:
         uses: goreleaser/goreleaser-action@v2
         with: 
           args: "release --rm-dist"
-          version: latest
+          version: latest
+          workdir: v2/
+        

.gitignore

@@ -1,3 +1,6 @@
 cmd/nuclei/nuclei*
 v2/cmd/nuclei/nuclei
 .idea
+integration_tests/integration-test
+integration_tests/nuclei
+v2/cmd/integration-test/integration-test

.golangci.yml

@@ -6,9 +6,9 @@ linters-settings:
   # funlen:
   #   lines: 100
   #   statements: 50
-  goconst:
-    min-len: 2
-    min-occurrences: 2
+  #goconst:
+  #  min-len: 2
+  #  min-occurrences: 2
   gocritic:
     enabled-tags:
       - diagnostic
@@ -56,18 +56,18 @@ linters:
   # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
   disable-all: true
   enable:
-    - bodyclose
+    #- bodyclose
     - deadcode
     - dogsled
     - errcheck
     - exhaustive
     - gochecknoinits
-    - goconst
+    #- goconst
     - gocritic
     - gofmt
     - goimports
     - golint
-    - gomnd
+    #- gomnd
     - goprintffuncname
     - gosimple
     - govet
@@ -110,6 +110,6 @@ linters:
 # golangci.com configuration
 # https://github.com/golangci/golangci/wiki/Configuration
 service:
-  golangci-lint-version: 1.31.x # use the fixed version to not introduce new linters unexpectedly
+  golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly
   prepare:
     - echo "here I can run custom commands, but no preparation needed for this repo"

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at contact@projectdiscovery.io. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

Dockerfile

@@ -1,19 +1,7 @@
-FROM golang:alpine as builder
+FROM golang:1.15-alpine as build-env
+RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
 
-RUN mkdir -p /app
-WORKDIR /app
-COPY ./v2/go.mod .
-RUN go mod download
-
-COPY . .
-RUN cd ./v2/cmd/nuclei && go build -o nuclei .
-
-FROM alpine
-
-RUN mkdir /app
-RUN adduser --home /app --shell /bin/sh --disabled-password appuser
-COPY --from=builder /app/v2/cmd/nuclei/nuclei /app
-USER appuser
-
-WORKDIR /app
-ENTRYPOINT ["/app/nuclei"]
+FROM alpine:latest
+RUN apk add --no-cache bind-tools ca-certificates
+COPY --from=build-env /go/bin/nuclei /usr/local/bin/nuclei
+ENTRYPOINT ["nuclei"]

LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Exposed Atoms
+Copyright (c) 2021 ProjectDiscovery, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,248 +1,165 @@
 <h1 align="center">
-  <img src="static/nuclei-logo.png" alt="nuclei" width="200px"></a>
   <br>
+  <a href="https://nuclei.projectdiscovery.io"><img src="static/nuclei-logo.png" width="200px" alt="Nuclei"></a>
 </h1>
 
-[![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT)
-[![Go Report Card](https://goreportcard.com/badge/github.com/projectdiscovery/nuclei)](https://goreportcard.com/report/github.com/projectdiscovery/nuclei)
-[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectdiscovery/nuclei/issues)
-[![GitHub Release](https://img.shields.io/github/release/projectdiscovery/nuclei)](https://github.com/projectdiscovery/nuclei/releases)
-[![Follow on Twitter](https://img.shields.io/twitter/follow/pdnuclei.svg?logo=twitter)](https://twitter.com/pdnuclei)
-[![Docker Images](https://img.shields.io/docker/pulls/projectdiscovery/nuclei.svg)](https://hub.docker.com/r/projectdiscovery/nuclei)
-[![Chat on Discord](https://img.shields.io/discord/695645237418131507.svg?logo=discord)](https://discord.gg/KECAGdH)
+<h4 align="center">Fast and customisable vulnerability scanner based on simple YAML based DSL.</h4>
+
 
 <p align="center">
-<a href="https://nuclei.projectdiscovery.io/templating-guide/" target="_blank"><img src="static/read-the-docs-button.png" height="42px"/></center></a>  <a href="https://github.com/projectdiscovery/nuclei-templates" target="_blank"><img src="static/download-templates-button.png" height="42px"/></a>
+<a href="https://goreportcard.com/report/github.com/projectdiscovery/nuclei"><img src="https://goreportcard.com/badge/github.com/projectdiscovery/nuclei"></a>
+<a href="https://github.com/projectdiscovery/nuclei/issues"><img src="https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat"></a>
+<a href="https://github.com/projectdiscovery/nuclei/releases"><img src="https://img.shields.io/github/release/projectdiscovery/nuclei"></a>
+<a href="https://twitter.com/pdnuclei"><img src="https://img.shields.io/twitter/follow/pdnuclei.svg?logo=twitter"></a>
+<a href="https://discord.gg/KECAGdH"><img src="https://img.shields.io/discord/695645237418131507.svg?logo=discord"></a>
+<a href="https://github.com/projectdiscovery/nuclei/actions/workflows/build.yaml"><img src="https://github.com/projectdiscovery/nuclei/actions/workflows/build.yaml/badge.svg?branch=master"></a>
+</p>
+      
+<p align="center">
+  <a href="#how-it-works">How</a> •
+  <a href="#install-nuclei">Install</a> •
+  <a href="#for-security-engineers">For Security Engineers</a> •
+  <a href="#for-developers-and-organisations">For Developers</a> •
+  <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">Documentation</a> •
+  <a href="#credits">Credits</a> •
+  <a href="#license">License</a> •
+  <a href="https://discord.gg/KECAGdH">Join Discord</a>
 </p>
 
+---
 
-Nuclei is a fast tool for configurable targeted scanning based on templates offering massive extensibility and ease of use.
+Nuclei is used to send requests across targets based on a template leading to zero false positives and providing fast scanning on large number of hosts. Nuclei offers scanning for a variety of protocols including TCP, DNS, HTTP, File, etc. With powerful and flexible templating, all kinds of security checks can be modelled with Nuclei.
 
-Nuclei is used to send requests across targets based on a template leading to zero false positives and providing effective scanning for known paths. Main use cases for nuclei are during initial reconnaissance phase to quickly check for low hanging fruits or CVEs across targets that are known and easily detectable. It uses [retryablehttp-go library](https://github.com/projectdiscovery/retryablehttp-go) designed to handle various errors and retries in case of blocking by WAFs, this is also one of our core modules from custom-queries.
-
-We have also [open-sourced a template repository](https://github.com/projectdiscovery/nuclei-templates) to maintain various type of templates, we hope that you will contribute there too. Templates are provided in hopes that these will be useful and will allow everyone to build their own templates for the scanner. Checkout the templating guide at [**nuclei.projectdiscovery.io**](https://nuclei.projectdiscovery.io/templating-guide/) for a primer on nuclei templates.
-
-## Resources
-
--   [Features](#features)
--   [Usage](#usage)
--   [Installation Instructions](#installation-instructions)
-    -   [From Binary](#from-binary)
-    -   [From Source](#from-source)
-    -   [From Github](#from-github)
--   [Nuclei templates](#nuclei-templates)
--   [Running nuclei](#running-nuclei)
-    -   [Running with a single template.](#running-with-single-template)
-    -   [Running with multiple templates.](#running-with-multiple-templates)
-    -   [Running with subfinder](#running-with-subfinder)
-    -   [Running in Docker](#running-in-docker-container)
--   [Template exclusion](#template-exclusion)
--   [Thanks](#thanks)
+We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-templates) that houses various type of vulnerability templates contributed by **more than 100** security researchers and engineers. It is preloaded with ready to use templates using `-update-templates` flag.
 
 
-## Features
+
+## How it works
+
+
+<h3 align="center">
+  <img src="static/nuclei-flow.jpg" alt="nuclei-flow" width="700px"></a>
+</h3>
+
+
+# Install Nuclei
+
+```sh
+▶ GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+```
+
+**More installation [methods can be found here](https://nuclei.projectdiscovery.io/nuclei/get-started/#nuclei-installation).**
+
+<table>
+<tr>
+<td>  
+
+### Download Templates
+
+You can download and update the nuclei templates using <ins>*update-templates*</ins> flag of nuclei that downloads all the available **nuclei-templates** from [Github project](https://github.com/projectdiscovery/nuclei-templates), a community curated list of templates that are ready to use.
+
+`▶ nuclei -update-templates`
+
+Nuclei is designed to used with custom templates according to the target and workflow, you can write your own checks for your specific workflow and needs, please refer to nuclei [templating guide](https://nuclei.projectdiscovery.io/templating-guide/) to write your own custom templates.
+
+</td>
+</tr>
+</table>
+
+### Running Nuclei
+
+Scanning for CVEs on given list of URLs.
+
+```sh
+▶ nuclei -l target_urls.txt -t cves/
+```
+
+**More detailed examples of running nuclei can be found [here](https://nuclei.projectdiscovery.io/nuclei/get-started/#running-nuclei).**
+
+# For Security Engineers
+
+Nuclei offers great number of features that are helpful for security engineers to customise workflow in their organisation. With the varieties of scan capabilities (like DNS, HTTP, TCP), security engineers can easily create their suite of custom checks with Nuclei.
+
+- Varieties of protocols supported: TCP, DNS, HTTP, File, etc
+- Achieve complex vulnerability steps with workflows and [dynamic requests.](https://blog.projectdiscovery.io/nuclei-unleashed-quickly-write-complex-exploits/)
+- Easy to integrate into CI/CD, designed to be easily integrated into regression cycle to actively check the fix and re-appearance of vulnerability. 
 
 <h1 align="left">
-  <img src="static/nuclei-run.png" alt="nuclei" width="700px"></a>
-  <br>
+  <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/"><img src="static/learn-more-button.png" width="170px" alt="Learn More"></a>
 </h1>
 
--   Simple and modular code base making it easy to contribute.
--   Fast And fully configurable using a template based engine.
--   Handles edge cases doing retries, backoffs etc for handling WAFs.
--   Smart matching functionality for zero false positive scanning.
+<table>
+<tr>
+<td>  
 
-## Usage
+**For bugbounty hunters:**
 
-```sh
-nuclei -h
-```
+Nuclei allows you to customise your testing approach with your own suite of checks and easily run across your bug bounty programs. Moroever, Nuclei can be easily integrated into any continuous scanning workflow.
 
-This will display help for the tool. Here are all the switches it supports.
+- Designed to be easily integrated into other tool workflow.
+- Can process thousands of hosts in few minutes.
+- Easily automate your custom testing approach with our simple YAML DSL.
 
-|        Flag       |                      Description                      |                     Example                     |
-| :---------------: | :---------------------------------------------------: | :---------------------------------------------: |
-|         -c        |       Number of concurrent requests (default 10)      |                  nuclei -c 100                  |
-|         -l        |             List of urls to run templates             |                nuclei -l urls.txt               |
-|      -target      |             Target to scan using templates            |        nuclei -target hxxps://example.com       |
-|         -t        |    Templates input file/files to check across hosts   |             nuclei -t git-core.yaml             |
-|         -t        |    Templates input file/files to check across hosts   |         nuclei -t nuclei-templates/cves/        |
-|        -nC        |               Don't Use colors in output              |                    nuclei -nC                   |
-|       -json       |         Prints and write output in json format        |                   nuclei -json                  |
-|   -json-requests  |  Write requests/responses for matches in JSON output  |           nuclei -json -json-requests           |
-|         -o        |         File to save output result (optional)         |               nuclei -o output.txt              |
-|       -pbar       |           Enable the progress bar (optional)          |                   nuclei -pbar                  |
-|      -silent      |           Show only found results in output           |                  nuclei -silent                 |
-|                   |             (except when using with pbar)             |                                                 |
-|      -retries     | Number of times to retry a failed request (default 1) |                nuclei -retries 1                |
-|      -timeout     |       Seconds to wait before timeout (default 5)      |                nuclei -timeout 5                |
-|      -rl          |       Rate-Limit of requests per specified target     |                nuclei -rl 100                   |
-|      -severity    |Run templates based on severity                        |                nuclei -severity critical, low                |
-|      -exclude     |Template input dir/file/files to exclude               |                nuclei -exclude panels -exclude tokens           |
-|       -debug      |         Allow debugging of request/responses.         |                  nuclei -debug                  |
-| -update-templates |         Download and updates nuclei templates         |             nuclei -update-templates            |
-| -update-directory |    Directory for storing nuclei-templates(optional)   |        nuclei -update-directory templates       |
-|        -tl        |                List available templates               |                    nuclei -tl                   |
-|         -v        |       Shows verbose output of all sent requests       |                    nuclei -v                    |
-|      -version     |                 Show version of nuclei                |                 nuclei -version                 |
-|     -proxy-url    |                       Proxy URL                       |     nuclei -proxy-url hxxp://127.0.0.1:8080     |
-|  -proxy-socks-url |                    Socks proxy  URL                   | nuclei -proxy-socks-url socks5://127.0.0.1:8080 |
-|         -H        |                     Custom Header                     |         nuclei -H "x-bug-bounty: hacker"        |
+Please check our other open-source projects that might fit into your bug bounty workflow: [github.com/projectdiscovery](http://github.com/projectdiscovery), we also host daily [refresh of DNS data at Chaos](http://chaos.projectdiscovery.io).
 
-## Installation Instructions
+</td>
+</tr>
+</table>
 
-### From Binary
+<table>
+<tr>
+<td>
+  
+**For pentesters:**
 
-The installation is easy. You can download the pre-built binaries for your platform from the [Releases](https://github.com/projectdiscovery/nuclei/releases/) page. Extract them using tar, move it to your `$PATH`and you're ready to go.
+Nuclei immensely improve how you approach security assessment by augmenting the manual repetitve processes. Consultancies are already converting their manual assessment steps with Nuclei, it allows them to run set of their custom assessment approach across thousands of hosts in an automated manner. 
 
-```sh
-Download latest binary from https://github.com/projectdiscovery/nuclei/releases
+Pen-testers get the full power of our public templates and customization capabilities to speed-up their assessment process, and specifically with the regression cycle where you can easily verify the fix.
 
-▶ tar -xzvf nuclei-linux-amd64.tar.gz
-▶ mv nuclei /usr/local/bin/
-▶ nuclei -version
-```
+- Easily create your compliance, standards suite (e.g. OWASP Top 10) checklist.
+- With capabilities like [fuzz](https://nuclei.projectdiscovery.io/templating-guide/#advance-fuzzing) and [workflows](https://nuclei.projectdiscovery.io/templating-guide/#workflows), complex manual steps and repetitive assessment can be easily automated with Nuclei.
+- Easy to re-test vulnerability-fix by just re-running the template.
 
-### From Source
-
-nuclei requires **go1.14+** to install successfully. Run the following command to get the repo -
-
-```sh
-▶ GO111MODULE=on go get -u -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
-```
-
-### From Github
-
-```sh
-▶ git clone https://github.com/projectdiscovery/nuclei.git; cd nuclei/v2/cmd/nuclei/; go build; mv nuclei /usr/local/bin/; nuclei -version
-```
-
-## Nuclei templates
-
-You can download or update the nuclei templates using `update-templates` flag.
-
-```sh
-▶ nuclei -update-templates
-```
-
-or download it from [nuclei templates](https://github.com/projectdiscovery/nuclei-templates) Github project.
-
-```sh
-▶ git clone https://github.com/projectdiscovery/nuclei-templates.git
-```
-
-**Please refer to nuclei [templating guide](https://nuclei.projectdiscovery.io/templating-guide/) to writing your own custom templates.**
-
-## Running nuclei
-
-### Running with single template.
-
-This will run `git-core.yaml` template against all the hosts in `urls.txt` and returns the matched results.
-
-```sh
-▶ nuclei -l urls.txt -t files/git-core.yaml -o results.txt
-```
-
-You can also pass the list of urls at standard input (STDIN). This allows for easy integration in automation pipelines.
-
-```sh
-▶ cat urls.txt | nuclei -t files/git-core.yaml -o results.txt
-```
-
-💡 Nuclei accepts list of URLs as input, for example here is how `urls.txt` looks like:- 
-
-```
-https://test.some-site.com
-http://vuls-testing.com
-https://test.com
-```
-### Running with multiple templates.
-
-This will run the tool against all the urls in `urls.txt` with all the templates in the `cves` and `files` directory and returns the matched results.
-
-```sh
-▶ nuclei -l urls.txt -t cves/ -t files/ -o results.txt
-```
-
-### Running with subfinder.
-
-```sh
-▶ subfinder -d hackerone.com -silent | httpx -silent | nuclei -t cves/ -o results.txt
-```
-
-### Running in Docker container
-
-You can use the [nuclei dockerhub image](https://hub.docker.com/r/projectdiscovery/nuclei). Simply run -
-
-```sh
-▶ docker pull projectdiscovery/nuclei
-```
-
-After downloading or building the container, run the following:
-
-```sh
-▶ docker run -it projectdiscovery/nuclei
-```
-
-For example, this will run the tool against all the hosts in `urls.txt` and output the results to your host file system:
-
-```sh
-▶ cat urls.txt | docker run -v /path/to/nuclei-templates:/app/nuclei-templates -v /path/to/nuclei/config:/app/.nuclei-config.json -i projectdiscovery/nuclei -t ./files/git-config.yaml > results.txt
-```
-
-Remember to change `/path-to-nuclei-templates` to the real path on your host file system.
-
-### Template Exclusion
-
-[Nuclei-templates](https://github.com/projectdiscovery/nuclei-templates) includes multiple checks including many that are useful for attack surface mapping and not necessarily a security issue, in cases where you only looking to scan few specific templates or directory, here are few options / flags to filter or exclude them from running. 
-
-#### Running templates based on severity
-
-You can run the templates based on the specific severity of the template, single and multiple severity can be used for scan. 
-
-```sh
-nuclei -l urls.txt -t cves/ -severity critical, medium
-```
-
-The above example will run all the templates under `cves` directory with `critical` and `medium` severity. 
-
-```sh
-nuclei -l urls.txt -t panels/ -t technologies -severity info
-```
-
-The above example will run all the templates under `panels` and `technologies` directory with **severity** marked as `info`
-
-#### Running templates with exclusion
-
-We do not suggest running all the nuclei-templates directory at once, in case of doing so, one can make use of `exclude` flag to exclude specific directory or templates to ignore from scanning. 
-
-```sh
-nuclei -l urls.txt -t nuclei-templates -exclude panels/ -exclude technologies -exclude files/wp-xmlrpc.yaml
-```
-
-Note:- both directory and specific templates case be excluded from scan as shared in the above example.
-
-#### Using `.nuclei-ignore` file for template exclusion
-
-Since release of nuclei [v2.1.1](https://github.com/projectdiscovery/nuclei/releases/tag/v2.1.1), we have added support of `.nuclei-ignore` file that works along with `update-templates` flag of nuclei, in **.nuclei-ignore** file, you can define all the template directory or template path that you wanted to exclude from all the nuclei scans, to start using this feature, make sure you installed nuclei templates using `nuclei -update-templates` flag, now you can add/update/remove templates in the file that you wanted to exclude from running. 
-
-```
-nano ~/nuclei-templates/.nuclei-ignore
-```
-
-Default **nuclei-ignore** list can be accessed from [here](https://github.com/projectdiscovery/nuclei-templates/blob/master/.nuclei-ignore), in case you don't want to exclude anything, simply remove the `.nuclei-ignore` file.
-
-* * *
-
-# 📋 Notes
-- Progress bar is experimental feature, might not work in few cases. 
-- Progress bar doesn't work with workflows, numbers are not accurate due to conditional execution.
+</td>
+</tr>
+</table>
 
 
-## Thanks
+# For Developers and Organisations
 
-nuclei is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)** file for more details.
+Nuclei is built with simplicity in mind, with the community backed templates by hundreds of security researchers, it allows you to stay updated with latest security threats using continuous Nuclei scanning on the hosts. It is designed to be easily integrated into regression tests cycle, to verify the fixes and  eliminate vulnerabilities from occuring in future.
 
-Do also check out these similar awesome projects that may fit in your workflow:
+- **CI/CD:** Engineers are already utilising Nuclei within their CI/CD pipeline, it allows them to constantly monitor their staging and production environments with customised templates.
+- **Continuous Regression Cycle:** With Nuclei, you can create your custom template on every new identified vulnerability and put into Nuclei engine to eliminate in the continuous regression cycle.
 
-[Burp Suite](https://portswigger.net/burp), [FFuF](https://github.com/ffuf/ffuf), [Jaeles](https://github.com/jaeles-project/jaeles), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [ChopChop](https://github.com/michelin/ChopChop)
+We have [a discussion thread around this](https://github.com/projectdiscovery/nuclei-templates/discussions/693), there are already some bug bounty programs giving incentives to hackers on writing nuclei templates with every submission, that helps them to eliminate the vulnerability across all their assets, as well as to eliminate future risk in reappearing on productions. If you're interested in implementing it in your organisation, feel free to [reach out to us](mailto:[email protected]). We will be more than happy to help you in the getting started process, or you can also post into the [discussion thread for any help](https://github.com/projectdiscovery/nuclei-templates/discussions/693).
+
+<h3 align="center">
+  <img src="static/regression-with-nuclei.jpg" alt="regression-cycle-with-nuclei" width="1100px"></a>
+</h3>
+
+<h1 align="left">
+  <a href="https://github.com/projectdiscovery/nuclei-action"><img src="static/learn-more-button.png" width="170px" alt="Learn More"></a>
+</h1>
+
+### Resources
+
+- [Community Powered Scanning with Nuclei](https://blog.projectdiscovery.io/community-powered-scanning-with-nuclei/)
+- [Nuclei Unleashed - Quickly write complex exploits](https://blog.projectdiscovery.io/nuclei-unleashed-quickly-write-complex-exploits/)
+- [Nuclei - Fuzz all the things](https://blog.projectdiscovery.io/nuclei-fuzz-all-the-things/)
+- [Weaponizes nuclei Workflows to Pwn All the Things](https://medium.com/@dwisiswant0/weaponizes-nuclei-workflows-to-pwn-all-the-things-cd01223feb77) by [@dwisiswant0](https://github.com/dwisiswant0)
+- [How to Scan Continuously with Nuclei?](https://medium.com/@dwisiswant0/how-to-scan-continuously-with-nuclei-fcb7e9d8b8b9) by [@dwisiswant0](https://github.com/dwisiswant0)
+
+### Credits
+
+Thanks to all the amazing community [contributors for sending PRs](https://github.com/projectdiscovery/nuclei/graphs/contributors). Do also check out the below similar open-source projects that may fit in your workflow:
+
+[FFuF](https://github.com/ffuf/ffuf), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [Jaeles](https://github.com/jaeles-project/jaeles), [ChopChop](https://github.com/michelin/ChopChop)
+
+### License
+
+Nuclei is distributed under [MIT License](https://github.com/projectdiscovery/nuclei/blob/master/LICENSE.md)
+
+<h1 align="left">
+  <a href="https://discord.gg/KECAGdH"><img src="static/Join-Discord.png" width="380" alt="Join Discord"></a> <a href="https://nuclei.projectdiscovery.io"><img src="static/check-nuclei-documentation.png" width="380" alt="Check Nuclei Documentation"></a>
+</h1>

README_CN.md

@@ -0,0 +1,253 @@
+<h1 align="center">
+  <img src="static/nuclei-logo.png" alt="nuclei" width="200px"></a>
+  <br>
+</h1>
+
+[![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT)
+[![Go Report Card](https://goreportcard.com/badge/github.com/projectdiscovery/nuclei)](https://goreportcard.com/report/github.com/projectdiscovery/nuclei)
+[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectdiscovery/nuclei/issues)
+[![GitHub Release](https://img.shields.io/github/release/projectdiscovery/nuclei)](https://github.com/projectdiscovery/nuclei/releases)
+[![Follow on Twitter](https://img.shields.io/twitter/follow/pdnuclei.svg?logo=twitter)](https://twitter.com/pdnuclei)
+[![Docker Images](https://img.shields.io/docker/pulls/projectdiscovery/nuclei.svg)](https://hub.docker.com/r/projectdiscovery/nuclei)
+[![Chat on Discord](https://img.shields.io/discord/695645237418131507.svg?logo=discord)](https://discord.gg/KECAGdH)
+
+<p align="center">
+<a href="https://nuclei.projectdiscovery.io/templating-guide/" target="_blank"><img src="static/read-the-docs-button.png" height="42px"/></center></a>  <a href="https://github.com/projectdiscovery/nuclei-templates" target="_blank"><img src="static/download-templates-button.png" height="42px"/></a>
+</p>
+
+
+Nuclei是一个基于模板的、可配置攻击目标的扫描快速工具，同时还提供了强大的可扩展性和易用性。
+
+基于模板的nuclei被用来发送请求给目标，有着实现零误报的优点，并且可以对已知的路径进行有效的扫描。nuclei的主要用于在初期的探测阶段快速地对已知的且易于检测的漏洞或者CVE进行扫描。如果存在WAF的话，nuclei使用[retryablehttp-go库](https://github.com/projectdiscovery/retryablehttp-go)来处理各种错误，并且重新尝试攻击，这也是我们自定义功能的核心模块之一。
+
+我们也维护一个具有各个类型的模板的[开源库](https://github.com/projectdiscovery/nuclei-templates)，我们希望你也能贡献一些模板，贡献的这些模板最好是有效的，并且能允许每个人基于你的模板重新构建。查看[**nuclei.projectdiscovery.io**](https://nuclei.projectdiscovery.io/templating-guide/)这个网站去学习制作模板的入门知识。
+
+## 目录
+
+- [目录](#目录)
+- [功能](#功能)
+- [安装](#安装)
+- [Nuclei模板](#nuclei模板)
+- [用法](#用法)
+- [运行Nuclei](#运行nuclei)
+- [排除模板](#排除模板)
+- [致谢](#致谢)
+
+## 功能
+
+<h1 align="left">
+  <img src="static/nuclei-run.png" alt="nuclei" width="700px"></a>
+  <br>
+</h1>
+
+- 有着易于开发的、简单的、模块化的代码库
+- 使用了基于模板的引擎，运行速度极快，可以修改所以配置
+- 可以对特殊情况处理、重试、绕过等，可以绕过WAF
+- 智能匹配，零误报
+
+## 安装
+
+### 二进制文件安装
+
+二进制文件安装很简单，你可以从[Releases](https://github.com/projectdiscovery/nuclei/releases/)页面下载已经构建好的二进制文件压缩包，使用解压工具提取下载的压缩包，并将解压的文件夹移动到$PATH目录，就可以直接使用了。
+
+```sh
+Download latest binary from https://github.com/projectdiscovery/nuclei/releases
+
+▶ tar -xzvf nuclei-linux-amd64.tar.gz
+▶ mv nuclei /usr/local/bin/
+▶ nuclei -version
+```
+
+### 源码安装
+
+nuclei需要**go1.14+**才能成功安装，运行以下命令获取repo
+
+```sh
+▶ GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+```
+
+### GitHub安装
+
+```sh
+▶ git clone https://github.com/projectdiscovery/nuclei.git; cd nuclei/v2/cmd/nuclei/; go build; mv nuclei /usr/local/bin/; nuclei -version
+```
+
+## Nuclei模板
+
+你可以使用`update-templates`来下载和更新nuclei模板，该命令会从unclei的[模板库]()中下载最新版本，这个由社区来维护的库是可以随时使用的。
+
+```sh
+▶ nuclei -update-templates
+```
+
+此外您可以根据自己的工作情况或者需求编写模板，请参阅**nuclei[模板向导](https://nuclei.projectdiscovery.io/templating-guide/)去编写自定义模板**
+
+## 用法
+
+```sh
+nuclei -h
+```
+
+这条命令会显示帮助，以下是nuclei支持的所有命令
+
+|命令|描述|例子|
+|-----|-----|-----|
+|bulk-size|每个模板最大并行的主机数(默认25)|nuclei -bulk-size 25|
+|burp-collaborator-biid|使用burp-collaborator插件|nuclei -burp-collaborator-biid XXXX|
+|c|并行的最大模板数量(默认10)|nuclei -c 10|
+|l|对URL列表进行测试|nuclei -l urls.txt|
+|target|对目标进行测试|nuclei -target hxxps://example.com|
+|t|要检测的模板种类|nuclei -t git-core.yaml -t cves/|
+|no-color|输出不显示颜色|nuclei -no-color|
+|no-meta|不显示匹配的元数据|nuclei -no-meta|
+|json|输出为json格式|nuclei -json|
+|include-rr|json输出格式中包含请求和响应数据|nuclei -json -include-rr|
+|o|输出为文件|nuclei -o output.txt|
+|project|避免发送相同的请求|nuclei -project|
+|stats|使用进度条|nuclei -stats|
+|silent|只输出测试成功的结果|nuclei -silent|
+|retries|失败后的重试次数|nuclei -retries 1|
+|timeout|超时时间(默认为5秒)|nuclei -timeout 5|
+|trace-log|输出日志到log文件|nuclei -trace-log logs|
+|rate-limit|每秒最大请求数(默认150)|nuclei -rate-limit 150|
+|severity|根据严重性选择模板|nuclei  -severity critical,high|
+|stop-at-first-match|第一次匹配不要处理HTTP请求|nuclei -stop-at-frst-match|
+|exclude|排除的模板或文件夹|nuclei -exclude panels -exclude tokens|
+|debug|调试请求或者响应|nuclei -debug|
+|update-templates|下载或者升级模板|nuclei -update-templates|
+|update-directory|选择储存模板的目录(可选)|nuclei -update-directory templates|
+|tl|列出可用的模板|nuclei -tl|
+|templates-version|显示已安装的模板版本|nuclei -templates-version|
+|v|显示发送请求的详细信息|nuclei -v|
+|version|显示nuclei的版本号|nuclei -version|
+|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080|
+|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080|
+|random-agent|使用随机的UA|nuclei -random-agent|
+|H|自定义请求头|nuclei -H “x-bug-bounty:hacker”|
+
+## 运行Nuclei
+
+### 运行单个模板
+
+这将对`urls.txt`中所有的主机运行`git-core.yaml`并返回结果到`results.txt`
+
+```sh
+▶ nuclei -l urls.txt -t files/git-core.yaml -o results.txt
+```
+
+你可以轻松的通过管道使用标准的输入(STDIN)传递URL列表。
+
+```sh
+▶ cat urls.txt | nuclei -t files/git-core.yaml -o results.txt
+```
+
+💡 Nuclei可以接受如下列表的URL作为输入，例如以下URL：
+
+```
+https://test.some-site.com
+http://vuls-testing.com
+https://test.com
+```
+### 运行多个模板
+
+这将会对`urls.txt`中所有的URL运行`cves`和`files`模板检查，并返回输出到`results.txt`
+
+```sh
+▶ nuclei -l urls.txt -t cves/ -t files/ -o results.txt
+```
+
+### 使用subfinder运行
+
+```sh
+▶ subfinder -d hackerone.com -silent | httpx -silent | nuclei -t cves/ -o results.txt
+```
+
+### 在docker中运行
+
+你需要使用[nuclei的docker镜像](https://hub.docker.com/r/projectdiscovery/nuclei)来运行
+
+```sh
+▶ docker pull projectdiscovery/nuclei
+```
+
+下载并构建完成后，运行以下命令：
+
+```sh
+▶ docker run -it projectdiscovery/nuclei
+```
+
+这将会对`urls.txt`中的URL通过docker中的nuclei进行检测，并将结果输出到本机的`results.txt`文件的：
+
+```sh
+▶ cat urls.txt | docker run -v /path/to/nuclei-templates:/app/nuclei-templates -v /path/to/nuclei/config:/app/.nuclei-config.json -i projectdiscovery/nuclei -t /app/nuclei-templates/files/git-config.yaml > results.txt
+```
+记住更改的模板路径到本机
+
+### 速率限制
+
+Nuclei有多种控制速率的方法，包括并行执行多个模板、并行检查多个主机，以及使nuclei限制全局的请求速率，下面就是示例。
+
+- `-c`参数 => 限制并行的模板数
+- `-bulk-size`参数 => 限制并行的主机数
+- `-rate-limit`参数 => 全局速率限制
+
+如果你想快速扫描或者控制扫描，请使用这些标志并输入限制数，`速率限制`只保证控制传出的请求，与其他参数无关。
+
+### 排除模板
+
+[Nuclei模板](https://github.com/projectdiscovery/nuclei-templates)包含多种检查，其中有许多对攻击有用的检查，但并不是都有用的。如果您只希望扫描少数特定的模板或目录，则可以使用如下的参数筛选模板，或将某些模板排除。
+
+#### 排除模板运行
+
+我们不建议同时运行所有的nuclei模板，如果要排除模板，可以使用`exclude`参数来排除特定的目录或模板。
+
+```sh
+nuclei -l urls.txt -t nuclei-templates -exclude panels/ -exclude technologies -exclude files/wp-xmlrpc.yaml
+```
+
+注意：如上述示例中显示的那样，目录和特定模板都将不会扫描
+
+#### 基于严重性运行模板
+
+您可以根据模板的严重性运行模板，扫描时可以选择单个严重性或多个严重性。
+
+```sh
+nuclei -l urls.txt -t cves/ -severity critical,medium
+```
+
+上面的例子将运行`cves`目录下所有`严重`和`中等`的模板。
+
+```sh
+nuclei -l urls.txt -t panels/ -t technologies -severity info
+```
+
+上面的例子将运行`panels`和`technologies`目录下严重性标记为`info`的模板
+
+#### 使用`.nuclei-ignore`文件排除模板
+
+自从nuclei的[v2.1.1版本](https://github.com/projectdiscovery/nuclei/releases/tag/v2.1.1)以来，我们添加了对`.nuclei-ignore`文件的支持，该文件与`update-templates`参数一起使用，在 **.nuclei-ignore** 文件中，您可以定义要从nuclei扫描中排除的所有模板目录或者模板路径，要开始使用此功能，请确保使用`nuclei-update-templates`参数安装nuclei模板，现在可以根据`.nuclei-ignore`的文件来添加、更新、删除模板文件。
+
+```
+nano ~/nuclei-templates/.nuclei-ignore
+```
+
+默认的**nuclei忽略**列表可以访问[这里]((https://github.com/projectdiscovery/nuclei-templates/blob/master/.nuclei-ignore)，如果不想排除任何内容，只需要删除`.nuclei-ignore`文件。
+
+* * *
+
+### 📋 笔记
+
+- 进度条是实验性功能，在某些情况下可能无法使用。
+- 进度条不适用于工作流，因为是条件执行，所以不准确。
+
+
+## 致谢
+
+也要看看这些类似的好项目，或许它们也适合你：
+
+[Burp Suite](https://portswigger.net/burp), [FFuF](https://github.com/ffuf/ffuf), [Jaeles](https://github.com/jaeles-project/jaeles), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [ChopChop](https://github.com/michelin/ChopChop)
+
+--------
+
+Nuclei是由[projectdiscovery](https://projectdiscovery.io)团队用🖤制作的，当然社区也贡献了很多，通过 **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)**文件以获取更多详细信息。

integration_tests/dns/basic.yaml

@@ -0,0 +1,17 @@
+id: basic-dns-example
+
+info:
+  name: Test DNS Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: A
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        words:
+          - "1.1.1.1"

integration_tests/http/get-headers.yaml

@@ -0,0 +1,17 @@
+id: basic-get-headers
+
+info:
+  name: Basic GET Headers Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    headers:
+      test: nuclei
+    matchers:
+      - type: word
+        words:
+          - "This is test headers matcher text"

integration_tests/http/get-query-string.yaml

@@ -0,0 +1,15 @@
+id: basic-get-querystring
+
+info:
+  name: Basic GET QueryString Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}?test=nuclei"
+    matchers:
+      - type: word
+        words:
+          - "This is test querystring matcher text"

integration_tests/http/get-redirects.yaml

@@ -0,0 +1,17 @@
+id: basic-get-redirects
+
+info:
+  name: Basic GET Redirects Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    redirects: true
+    max-redirects: 3
+    matchers:
+      - type: word
+        words:
+          - "This is test redirects matcher text"

integration_tests/http/get.yaml

@@ -0,0 +1,15 @@
+id: basic-get
+
+info:
+  name: Basic GET Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "This is test matcher text"

integration_tests/http/post-body.yaml

@@ -0,0 +1,19 @@
+id: basic-post-body
+
+info:
+  name: Basic POST Body Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: POST
+    path:
+      - "{{BaseURL}}"
+    headers: 
+      Content-Type: application/x-www-form-urlencoded
+      Content-Length: 1 # as long as there is a value, nuclei will auto-recalculate it.
+    body: username=test&password=nuclei
+    matchers:
+      - type: word
+        words:
+          - "This is test post-body matcher text"

integration_tests/http/post-json-body.yaml

@@ -0,0 +1,19 @@
+id: basic-post-json-body
+
+info:
+  name: Basic POST JSON Body Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: POST
+    path:
+      - "{{BaseURL}}"
+    headers: 
+      Content-Type: application/json
+      Content-Length: 1
+    body: '{"username":"test","password":"nuclei"}'
+    matchers:
+      - type: word
+        words:
+          - "This is test post-json-body matcher text"

integration_tests/http/post-multipart-body.yaml

@@ -0,0 +1,29 @@
+id: basic-post-multipart-body
+
+info:
+  name: Basic POST Multipart Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: POST
+    path:
+      - "{{BaseURL}}"
+    headers: 
+      Content-Type: multipart/form-data; boundary=d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8
+      Content-Length: 1
+    body: |
+      --d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8
+      Content-Disposition: form-data; name="username"; filename="username"
+      Content-Type: application/octet-stream
+
+      test
+      --d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8
+      Content-Disposition: form-data; name="password"
+
+      nuclei
+      --d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8--
+    matchers:
+      - type: word
+        words:
+          - "This is test post-multipart matcher text"

integration_tests/http/raw-cookie-reuse.yaml

@@ -0,0 +1,34 @@
+id: cookiereuse-raw-example
+info:
+  name: Test CookieReuse RAW Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        POST / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+
+        testing=parameter
+      - |
+        GET / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+        
+    cookie-reuse: true
+    matchers:
+      - type: word
+        words:
+          - "Test is test-cookie-reuse matcher text"

integration_tests/http/raw-dynamic-extractor.yaml

@@ -0,0 +1,43 @@
+id: dynamic-extractor-raw-example
+
+info:
+  name: Test Dynamic Extractor RAW Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        POST / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+
+        testing=parameter
+      - |
+        GET /?username={{randkey}} HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+    extractors:
+      - type: regex
+        name: randkey
+        part: body
+        group: 1
+        internal: true
+        regex:
+          - "Token: '([A-Za-z0-9]+)'"
+
+    cookie-reuse: true
+    matchers:
+      - type: word
+        words:
+          - "Test is test-dynamic-extractor-raw matcher text"

integration_tests/http/raw-get-query.yaml

@@ -0,0 +1,18 @@
+id: basic-raw-query-example
+
+info:
+  name: Test RAW GET Query Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        GET ?test=nuclei HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+
+    matchers:
+      - type: word
+        words:
+          - "Test is test raw-get-query-matcher text"

integration_tests/http/raw-get.yaml

@@ -0,0 +1,18 @@
+id: basic-raw-http-example
+
+info:
+  name: Test RAW GET Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        GET / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+
+    matchers:
+      - type: word
+        words:
+          - "Test is test raw-get-matcher text"

integration_tests/http/raw-payload.yaml

@@ -0,0 +1,29 @@
+id: payload-raw-example
+info:
+  name: Test RAW With Payload Template
+  author: pdteam
+  severity: info
+
+requests:
+  - payloads:
+      username:
+        - test
+      password:
+        - nuclei
+        - guest
+    attack: clusterbomb
+    raw:
+      - |
+        POST / HTTP/1.1
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5)
+        Host: {{Hostname}}
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+        another_header: {{base64('§password§')}}
+        Accept: */*
+
+        username=§username§&password={{password}}
+    matchers:
+      - type: word
+        words:
+          - "Test is raw-payload matcher text"

integration_tests/http/raw-post-body.yaml

@@ -0,0 +1,21 @@
+id: basic-raw-http-body-example
+
+info:
+  name: Test RAW POST Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        POST / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+
+        username=test&password=nuclei
+    matchers:
+      - type: word
+        words:
+          - "Test is test raw-post-body-matcher text"

integration_tests/http/raw-unsafe-request.yaml

@@ -0,0 +1,20 @@
+id: basic-raw-unsafe-request-example
+
+info:
+  name: Test RAW Unsafe Request Template
+  author: pd-team
+  severity: info
+
+requests:
+  - raw:
+      - |
+        GET / HTTP/1.1
+        Host: 
+        Content-Length: 4
+
+    unsafe: true
+    matchers-condition: and
+    matchers:
+      - type: word
+        words:
+          - "This is test-raw-unsafe request matcher."

integration_tests/http/request-condition-new.yaml

@@ -0,0 +1,19 @@
+id: request-condition-new
+
+info:
+  name: request-condition-new
+  author: pd-team
+  severity: info
+
+requests:
+  - method: GET
+    id: first
+    path:
+      - "{{BaseURL}}/200"
+  - method: GET
+    path:
+      - "{{BaseURL}}/400"
+    matchers:
+      - type: dsl
+        dsl:
+          - "first_status_code==200 && status_code==400"

integration_tests/http/request-condition.yaml

@@ -0,0 +1,18 @@
+id: request-condition
+
+info:
+  name: request-condition
+  author: pd-team
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}/200"
+      - "{{BaseURL}}/400"
+ 
+    req-condition: true
+    matchers:
+      - type: dsl
+        dsl:
+          - "status_code_1==200 && status_code_2==400"

integration_tests/network/basic.yaml

@@ -0,0 +1,18 @@
+id: basic-network-request
+
+info:
+  name: Basic Network Request
+  author: pdteam
+  severity: info
+
+network:
+  - host: 
+      - "{{Hostname}}"
+    inputs:
+      - data: "PING\r\n"
+    read-size: 4
+    matchers:
+      - type: word
+        part: data
+        words:
+          - "PONG"

integration_tests/network/hex.yaml

@@ -0,0 +1,22 @@
+id: hex-network-request
+
+info:
+  name: Hex Input Network Request
+  author: pdteam
+  severity: info
+
+network:
+  - host: 
+      - "{{Hostname}}"
+    inputs:
+      - data: "50494e47"
+        type: hex
+      - data: "\r\n"
+
+    read-size: 4
+    matchers:
+      - type: word
+        part: data
+        encoding: hex
+        words:
+          - "504f4e47"

integration_tests/network/multi-step.yaml

@@ -0,0 +1,32 @@
+id: multi-step
+
+info:
+  name: Multi-Step Network Template
+  author: pd-team
+  severity: info
+
+network:
+  - inputs:
+      - data: "FIRST"
+        read: 4
+        name: first
+      - data: "SECOND"
+        read: 4
+        name: second
+    host:
+      - "{{Hostname}}"
+    read-size: 6
+    matchers:
+      - type: word
+        part: first
+        words:
+          - "PING" 
+      - type: word
+        part: second
+        words:
+          - "PONG" 
+      - type: word
+        part: data
+        words:
+          - "NUCLEI" 
+    matchers-condition: and

integration_tests/run.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+cd ../v2/cmd/nuclei
+go build
+cp nuclei ../../../integration_tests/nuclei 
+cd ../integration-test
+go build
+cp integration-test ../../../integration_tests/integration-test 
+cd ../../../integration_tests
+./integration-test
+if [ $? -eq 0 ]
+then
+  exit 0
+else
+  exit 1
+fi

integration_tests/workflow/basic.yaml

@@ -0,0 +1,10 @@
+id: workflow-example
+
+info:
+  name: Test Workflow Template
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-1.yaml
+  - template: workflow/match-2.yaml

integration_tests/workflow/condition-matched.yaml

@@ -0,0 +1,11 @@
+id: condition-matched-workflow
+
+info:
+  name: Condition Matched Workflow
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-1.yaml
+    subtemplates:
+      - template: workflow/match-2.yaml

integration_tests/workflow/condition-unmatched.yaml

@@ -0,0 +1,11 @@
+id: condition-unmatched-workflow
+
+info:
+  name: Condition UnMatched Workflow
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/nomatch-1.yaml
+    subtemplates:
+      - template: workflow/match-2.yaml

integration_tests/workflow/match-1.yaml

@@ -0,0 +1,15 @@
+id: basic-get
+
+info:
+  name: Basic GET Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "This is test matcher text"

integration_tests/workflow/match-2.yaml

@@ -0,0 +1,16 @@
+id: basic-get-another
+
+info:
+  name: Basic Another GET Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        name: test-matcher
+        words:
+          - "This is test matcher text"

integration_tests/workflow/matcher-name.yaml

@@ -0,0 +1,13 @@
+id: matcher-name-workflow
+
+info:
+  name: Matcher Name Workflow
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-2.yaml
+    matchers: 
+      - name: test-matcher
+        subtemplates:
+          - template: workflow/match-1.yaml

integration_tests/workflow/nomatch-1.yaml

@@ -0,0 +1,15 @@
+id: basic-get-nomatch
+
+info:
+  name: Basic GET Request NoMatch
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "Random"

static/regression-cycle.md

@@ -0,0 +1 @@
+

v2/.goreleaser.yml

@@ -1,6 +1,10 @@
+before:
+  hooks:
+    - go mod tidy
+
 builds:
     - binary: nuclei
-      main: v2/cmd/nuclei/main.go
+      main: cmd/nuclei/main.go
       goos:
         - linux
         - windows
@@ -18,4 +22,4 @@ archives:
           darwin: macOS
       format_overrides:
           - goos: windows
-            format: zip
+            format: zip

v2/cmd/integration-test/dns.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var dnsTestCases = map[string]testutils.TestCase{
+	"dns/basic.yaml": &dnsBasic{},
+}
+
+type dnsBasic struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *dnsBasic) Execute(filePath string) error {
+	var routerErr error
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, "one.one.one.one", debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/integration-test/http.go

@@ -0,0 +1,524 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"strings"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var httpTestcases = map[string]testutils.TestCase{
+	"http/get-headers.yaml":           &httpGetHeaders{},
+	"http/get-query-string.yaml":      &httpGetQueryString{},
+	"http/get-redirects.yaml":         &httpGetRedirects{},
+	"http/get.yaml":                   &httpGet{},
+	"http/post-body.yaml":             &httpPostBody{},
+	"http/post-json-body.yaml":        &httpPostJSONBody{},
+	"http/post-multipart-body.yaml":   &httpPostMultipartBody{},
+	"http/raw-cookie-reuse.yaml":      &httpRawCookieReuse{},
+	"http/raw-dynamic-extractor.yaml": &httpRawDynamicExtractor{},
+	"http/raw-get-query.yaml":         &httpRawGetQuery{},
+	"http/raw-get.yaml":               &httpRawGet{},
+	"http/raw-payload.yaml":           &httpRawPayload{},
+	"http/raw-post-body.yaml":         &httpRawPostBody{},
+	"http/raw-unsafe-request.yaml":    &httpRawUnsafeRequest{},
+	"http/request-condition.yaml":     &httpRequestCondition{},
+	"http/request-condition-new.yaml": &httpRequestCondition{},
+}
+
+func httpDebugRequestDump(r *http.Request) {
+	if debug {
+		if dump, err := httputil.DumpRequest(r, true); err == nil {
+			fmt.Printf("\nRequest dump: \n%s\n\n", string(dump))
+		}
+	}
+}
+
+type httpGetHeaders struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGetHeaders) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.Header.Get("test"), "nuclei") {
+			fmt.Fprintf(w, "This is test headers matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpGetQueryString struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGetQueryString) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") {
+			fmt.Fprintf(w, "This is test querystring matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpGetRedirects struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGetRedirects) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		http.Redirect(w, r, "/redirected", http.StatusFound)
+	}))
+	router.GET("/redirected", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test redirects matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpGet struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGet) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpPostBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpPostBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("username"), "test") && strings.EqualFold(r.Form.Get("password"), "nuclei") {
+			fmt.Fprintf(w, "This is test post-body matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpPostJSONBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpPostJSONBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+
+		type doc struct {
+			Username string `json:"username"`
+			Password string `json:"password"`
+		}
+		obj := &doc{}
+		if err := json.NewDecoder(r.Body).Decode(obj); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(obj.Username, "test") && strings.EqualFold(obj.Password, "nuclei") {
+			fmt.Fprintf(w, "This is test post-json-body matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpPostMultipartBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpPostMultipartBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseMultipartForm(1 * 1024); err != nil {
+			routerErr = err
+			return
+		}
+		password, ok := r.MultipartForm.Value["password"]
+		if !ok || len(password) != 1 {
+			routerErr = errors.New("no password in request")
+			return
+		}
+		file := r.MultipartForm.File["username"]
+		if len(file) != 1 {
+			routerErr = errors.New("no file in request")
+			return
+		}
+		if strings.EqualFold(password[0], "nuclei") && strings.EqualFold(file[0].Filename, "username") {
+			fmt.Fprintf(w, "This is test post-multipart matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawDynamicExtractor struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawDynamicExtractor) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("testing"), "parameter") {
+			fmt.Fprintf(w, "Token: 'nuclei'")
+		}
+	}))
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.URL.Query().Get("username"), "nuclei") {
+			fmt.Fprintf(w, "Test is test-dynamic-extractor-raw matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawGetQuery struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawGetQuery) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") {
+			fmt.Fprintf(w, "Test is test raw-get-query-matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawGet struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawGet) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+
+		fmt.Fprintf(w, "Test is test raw-get-matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawPayload struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawPayload) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if !(strings.EqualFold(r.Header.Get("another_header"), "bnVjbGVp") || strings.EqualFold(r.Header.Get("another_header"), "Z3Vlc3Q=")) {
+			return
+		}
+		if strings.EqualFold(r.Form.Get("username"), "test") && (strings.EqualFold(r.Form.Get("password"), "nuclei") || strings.EqualFold(r.Form.Get("password"), "guest")) {
+			fmt.Fprintf(w, "Test is raw-payload matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 2 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawPostBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawPostBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("username"), "test") && strings.EqualFold(r.Form.Get("password"), "nuclei") {
+			fmt.Fprintf(w, "Test is test raw-post-body-matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawCookieReuse struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawCookieReuse) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("testing"), "parameter") {
+			http.SetCookie(w, &http.Cookie{Name: "nuclei", Value: "test"})
+		}
+	}))
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		cookie, err := r.Cookie("nuclei")
+		if err != nil {
+			routerErr = err
+			return
+		}
+
+		if strings.EqualFold(cookie.Value, "test") {
+			fmt.Fprintf(w, "Test is test-cookie-reuse matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawUnsafeRequest struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawUnsafeRequest) Execute(filePath string) error {
+	var routerErr error
+
+	ts := testutils.NewTCPServer(func(conn net.Conn) {
+		defer conn.Close()
+
+		_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 40\r\nContent-Type: text/plain; charset=utf-8\r\nDate: Thu, 25 Feb 2021 17:17:28 GMT\r\n\r\nThis is test-raw-unsafe request matcher.\r\n"))
+	})
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, "http://"+ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRequestCondition struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRequestCondition) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.GET("/200", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		w.WriteHeader(200)
+	}))
+	router.GET("/400", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		w.WriteHeader(400)
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/integration-test/integration-test.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var (
+	debug      = os.Getenv("DEBUG") == "true"
+	customTest = os.Getenv("TEST")
+	protocol   = os.Getenv("PROTO")
+)
+
+func main() {
+	success := aurora.Green("[✓]").String()
+	failed := aurora.Red("[✘]").String()
+
+	protocolTests := map[string]map[string]testutils.TestCase{
+		"http":     httpTestcases,
+		"network":  networkTestcases,
+		"dns":      dnsTestCases,
+		"workflow": workflowTestcases,
+	}
+	for proto, tests := range protocolTests {
+		if protocol == "" || protocol == proto {
+			fmt.Printf("Running test cases for \"%s\" protocol\n", aurora.Blue(proto))
+
+			for file, test := range tests {
+				if customTest != "" && !strings.Contains(file, customTest) {
+					continue // only run tests user asked
+				}
+				err := test.Execute(file)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, file, err)
+					os.Exit(1)
+				} else {
+					fmt.Printf("%s Test \"%s\" passed!\n", success, file)
+				}
+			}
+		}
+	}
+}
+
+func errIncorrectResultsCount(results []string) error {
+	return fmt.Errorf("incorrect number of results %s", strings.Join(results, "\n\t"))
+}

v2/cmd/integration-test/network.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"net"
+
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var networkTestcases = map[string]testutils.TestCase{
+	"network/basic.yaml":      &networkBasic{},
+	"network/hex.yaml":        &networkBasic{},
+	"network/multi-step.yaml": &networkMultiStep{},
+}
+
+type networkBasic struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *networkBasic) Execute(filePath string) error {
+	var routerErr error
+
+	ts := testutils.NewTCPServer(func(conn net.Conn) {
+		defer conn.Close()
+
+		data := make([]byte, 4)
+		if _, err := conn.Read(data); err != nil {
+			routerErr = err
+			return
+		}
+		if string(data) == "PING" {
+			_, _ = conn.Write([]byte("PONG"))
+		}
+	})
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type networkMultiStep struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *networkMultiStep) Execute(filePath string) error {
+	var routerErr error
+
+	ts := testutils.NewTCPServer(func(conn net.Conn) {
+		defer conn.Close()
+
+		data := make([]byte, 5)
+		if _, err := conn.Read(data); err != nil {
+			routerErr = err
+			return
+		}
+		if string(data) == "FIRST" {
+			_, _ = conn.Write([]byte("PING"))
+		}
+
+		data = make([]byte, 6)
+		if _, err := conn.Read(data); err != nil {
+			routerErr = err
+			return
+		}
+		if string(data) == "SECOND" {
+			_, _ = conn.Write([]byte("PONG"))
+		}
+		_, _ = conn.Write([]byte("NUCLEI"))
+	})
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/integration-test/workflow.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var workflowTestcases = map[string]testutils.TestCase{
+	"workflow/basic.yaml":               &workflowBasic{},
+	"workflow/condition-matched.yaml":   &workflowConditionMatched{},
+	"workflow/condition-unmatched.yaml": &workflowConditionUnmatch{},
+	"workflow/matcher-name.yaml":        &workflowMatcherName{},
+}
+
+type workflowBasic struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowBasic) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 2 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type workflowConditionMatched struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowConditionMatched) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type workflowConditionUnmatch struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowConditionUnmatch) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 0 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type workflowMatcherName struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowMatcherName) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/nuclei/issue-tracker-config.yaml

@@ -0,0 +1,47 @@
+#allow-list:
+#  severity: high,critical
+#deny-list:
+#  severity: low
+
+# github contains configuration options for github issue tracker
+#github: 
+#  # base-url is the optional self-hosted github application url
+#  base-url: ""
+#  # username is the username of the github user
+#  username: ""
+#  # owner is the owner name of the repository for issues.
+#  owner: ""
+#  # token is the token for github account.
+#  token: ""
+#  # project-name is the name of the repository.
+#  project-name: ""
+#  # issue-label is the label of the created issue type
+#  issue-label: ""
+
+# gitlab contains configuration options for gitlab issue tracker
+#gitlab: 
+#  # base-url is the optional self-hosted gitlab application url
+#  base-url: ""
+#  # username is the username of the gitlab user
+#  username: ""
+#  # token is the token for gitlab account.
+#  token: ""
+#  # project-id is the ID of the repository.
+#  project-id: ""
+#  # issue-label is the label of the created issue type
+#  issue-label: ""
+
+# jira contains configuration options for jira issue tracker
+#jira: 
+#  # URL is the jira application url
+#  url: ""
+#  # account-id is the account-id of the jira user
+#  account-id: ""
+#  # email is the email of the user for jira instance
+#  email: ""
+#  # token is the token for jira instance.
+#  token: ""
+#  # project-name is the name of the project.
+#  project-name: ""
+#  # issue-type is the name of the created issue type
+#  issue-type: ""

v2/cmd/nuclei/main.go

@@ -1,19 +1,96 @@
 package main
 
 import (
+	"os"
+	"path"
+
+	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/internal/runner"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+var (
+	cfgFile string
+	options = &types.Options{}
 )
 
 func main() {
-	// Parse the command line flags and read config files
-	options := runner.ParseOptions()
+	readConfig()
+
+	runner.ParseOptions(options)
 
 	nucleiRunner, err := runner.New(options)
 	if err != nil {
-		gologger.Fatalf("Could not create runner: %s\n", err)
+		gologger.Fatal().Msgf("Could not create runner: %s\n", err)
 	}
-
 	nucleiRunner.RunEnumeration()
-	nucleiRunner.Close()
+}
+
+func readConfig() {
+	home, _ := os.UserHomeDir()
+	templatesDirectory := path.Join(home, "nuclei-templates")
+
+	set := goflags.New()
+	set.SetDescription(`Nuclei is a fast tool for configurable targeted scanning 
+based on templates offering massive extensibility and ease of use.`)
+	set.StringVar(&cfgFile, "config", "", "Nuclei configuration file")
+	set.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port")
+	set.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on")
+	set.StringVarP(&options.Target, "target", "u", "", "URL to scan with nuclei")
+	set.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "Templates to run, supports single and multiple templates using directory.")
+	set.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "Workflows to run for nuclei")
+	set.StringSliceVarP(&options.ExcludedTemplates, "exclude", "et", []string{}, "Templates to exclude, supports single and multiple templates using directory.")
+	set.StringSliceVarP(&options.Severity, "severity", "impact", []string{}, "Templates to run based on severity, supports single and multiple severity.")
+	set.StringVarP(&options.Targets, "list", "l", "", "List of URLs to run templates on")
+	set.StringVarP(&options.Output, "output", "o", "", "File to write output to (optional)")
+	set.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server")
+	set.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the proxy socks server")
+	set.BoolVar(&options.Silent, "silent", false, "Show only results in output")
+	set.BoolVar(&options.Version, "version", false, "Show version of nuclei")
+	set.BoolVarP(&options.Verbose, "verbose", "v", false, "Show verbose output")
+	set.BoolVarP(&options.NoColor, "no-color", "nc", false, "Disable colors in output")
+	set.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout")
+	set.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
+	set.BoolVarP(&options.RandomAgent, "random-agent", "ra", false, "Use randomly selected HTTP User-Agent header value")
+	set.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "Custom Header.")
+	set.BoolVar(&options.Debug, "debug", false, "Debugging request and responses")
+	set.BoolVar(&options.DebugRequests, "debug-req", false, "Debugging request")
+	set.BoolVar(&options.DebugResponse, "debug-resp", false, "Debugging response")
+	set.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "Download / updates nuclei community templates")
+	set.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log")
+	set.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "Directory storing nuclei-templates")
+	set.BoolVar(&options.JSON, "json", false, "Write json output to files")
+	set.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "Write requests/responses for matches in JSON output")
+	set.BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan")
+	set.BoolVar(&options.TemplateList, "tl", false, "List available templates")
+	set.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "Maximum requests to send per second")
+	set.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "Stop processing http requests at first match (this may break template/workflow logic)")
+	set.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "Maximum Number of hosts analyzed in parallel per template")
+	set.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "Maximum Number of templates executed in parallel")
+	set.BoolVar(&options.Project, "project", false, "Use a project folder to avoid sending same request multiple times")
+	set.StringVar(&options.ProjectPath, "project-path", "", "Use a user defined project folder, temporary folder is used if not specified but enabled")
+	set.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "Don't display metadata for the matches")
+	set.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "Shows the installed nuclei-templates version")
+	set.BoolVar(&options.OfflineHTTP, "passive", false, "Enable Passive HTTP response processing mode")
+	set.StringVarP(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "biid", "", "Burp Collaborator BIID")
+	set.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "Nuclei Reporting Module configuration file")
+	set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database (Always use this to persistent report data)")
+	set.StringSliceVar(&options.Tags, "tags", []string{}, "Tags to execute templates for")
+	set.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "Exclude templates with the provided tags")
+	set.StringVarP(&options.ResolversFile, "resolvers", "r", "", "File containing resolver list for nuclei")
+	set.BoolVar(&options.Headless, "headless", false, "Enable headless browser based templates support")
+	set.BoolVar(&options.ShowBrowser, "show-browser", false, "Show the browser on the screen")
+	set.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "Number of seconds between each stats line")
+	set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback")
+	set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless")
+	set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates")
+	set.StringVarP(&options.DiskExportDirectory, "disk-export", "de", "", "Directory on disk to export reports in markdown to")
+	_ = set.Parse()
+
+	if cfgFile != "" {
+		if err := set.MergeConfigFile(cfgFile); err != nil {
+			gologger.Fatal().Msgf("Could not read config: %s\n", err)
+		}
+	}
 }

v2/go.mod

@@ -1,30 +1,58 @@
 module github.com/projectdiscovery/nuclei/v2
 
-go 1.14
+go 1.15
 
 require (
 	github.com/Knetic/govaluate v3.0.0+incompatible
+	github.com/andygrunwald/go-jira v1.13.0
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/d5/tengo/v2 v2.6.2
+	github.com/corpix/uarand v0.1.1
+	github.com/fatih/structs v1.1.0 // indirect
+	github.com/go-rod/rod v0.91.1
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-github/v32 v32.1.0
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
 	github.com/json-iterator/go v1.1.10
+	github.com/julienschmidt/httprouter v1.3.0
 	github.com/karrick/godirwalk v1.16.1
 	github.com/logrusorgru/aurora v2.0.3+incompatible
-	github.com/miekg/dns v1.1.35
+	github.com/mattn/go-runewidth v0.0.10 // indirect
+	github.com/miekg/dns v1.1.38
+	github.com/mitchellh/go-ps v1.0.0
+	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/projectdiscovery/clistats v0.0.5
-	github.com/projectdiscovery/collaborator v0.0.1
-	github.com/projectdiscovery/fastdialer v0.0.1
-	github.com/projectdiscovery/gologger v1.0.1
+	github.com/projectdiscovery/clistats v0.0.8
+	github.com/projectdiscovery/collaborator v0.0.2
+	github.com/projectdiscovery/fastdialer v0.0.8
+	github.com/projectdiscovery/goflags v0.0.4
+	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/hmap v0.0.1
-	github.com/projectdiscovery/rawhttp v0.0.4
-	github.com/projectdiscovery/retryabledns v1.0.5
+	github.com/projectdiscovery/rawhttp v0.0.6
+	github.com/projectdiscovery/retryabledns v1.0.10
 	github.com/projectdiscovery/retryablehttp-go v1.0.1
 	github.com/remeh/sizedwaitgroup v1.0.0
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/rs/xid v1.2.1
 	github.com/segmentio/ksuid v1.0.3
 	github.com/spaolacci/murmur3 v1.1.0
-	github.com/stretchr/testify v1.5.1
+	github.com/spf13/cast v1.3.1
+	github.com/stretchr/testify v1.7.0
+	github.com/syndtr/goleveldb v1.0.0
+	github.com/trivago/tgo v1.0.7 // indirect
+	github.com/valyala/fasttemplate v1.2.1
+	github.com/xanzy/go-gitlab v0.44.0
+	go.uber.org/atomic v1.7.0
+	go.uber.org/multierr v1.6.0
 	go.uber.org/ratelimit v0.1.0
-	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
-	gopkg.in/yaml.v2 v2.3.0
+	golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect
+	golang.org/x/net v0.0.0-20210119194325-5f4716e94777
+	golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
+	golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b // indirect
+	golang.org/x/text v0.3.4 // indirect
+	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	gopkg.in/yaml.v2 v2.4.0
 )

v2/go.sum

@@ -1,116 +1,557 @@
-github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
-github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg=
+github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
+github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
+github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE=
+github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
-github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc=
-github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
+github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4=
+github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
+github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
+github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
+github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-rod/rod v0.91.1 h1:7xIlC/bXCXosZqZUl2x6GVB8tv4yMQ4W/ZVdGVa1qYI=
+github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
+github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
+github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
 github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
+github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
+github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
 github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
+github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs=
 github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw=
+github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
+github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/projectdiscovery/clistats v0.0.5 h1:vcvOR9PrFRawO/7FWD6pER9nYVSoSTD2F+/fkRs73a0=
-github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
-github.com/projectdiscovery/collaborator v0.0.1 h1:dbQ5BCL/a3c+BB9cGtrGgiLs23+EfSzoaTzX/pxqiTI=
-github.com/projectdiscovery/collaborator v0.0.1/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
-github.com/projectdiscovery/fastdialer v0.0.1 h1:MgBkJ/zkciFu/PcbAz0DYGiZn2aqv6b39NvfXxfN8qg=
-github.com/projectdiscovery/fastdialer v0.0.1/go.mod h1:d24GUzSb93wOY7lu4gJmXAzfomqAGEcRrInEVrM6zbc=
-github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ=
-github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
+github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20=
+github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
+github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5zmiJ6tI7+6mB3s=
+github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
+github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0=
+github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4=
+github.com/projectdiscovery/goflags v0.0.4 h1:fWKLMAr3KmPlZxE1b54pfei+vGIUJn9q6aM7woZIbCY=
+github.com/projectdiscovery/goflags v0.0.4/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
+github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
+github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
 github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
-github.com/projectdiscovery/rawhttp v0.0.4 h1:O5IreNGk83d4xTD9e6SpkKbX0sHTs8K1Q33Bz4eYl2E=
-github.com/projectdiscovery/rawhttp v0.0.4/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
-github.com/projectdiscovery/retryabledns v1.0.5 h1:bQivGy5CuqKlwcxRkgA5ENincqIed/BR2sA6t2gdwuI=
-github.com/projectdiscovery/retryabledns v1.0.5/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
+github.com/projectdiscovery/rawhttp v0.0.6 h1:HbgPB1eKXQVV5F9sq0Uxflm95spWFyZYD8dgFpeOC9M=
+github.com/projectdiscovery/rawhttp v0.0.6/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
+github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
+github.com/projectdiscovery/retryabledns v1.0.10 h1:xJZ2aKoqrNg/OZEw1+4+QIOH40V/WkZDYY1ZZc+uphE=
+github.com/projectdiscovery/retryabledns v1.0.10/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
 github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY=
 github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/trivago/tgo v1.0.1/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
+github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM=
+github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
+github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
+github.com/xanzy/go-gitlab v0.44.0 h1:cEiGhqu7EpFGuei2a2etAwB+x6403E5CvpLn35y+GPs=
+github.com/xanzy/go-gitlab v0.44.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
+github.com/ysmood/goob v0.3.0 h1:XZ51cZJ4W3WCoCiUktixzMIQF86W7G5VFL4QQ/Q2uS0=
+github.com/ysmood/goob v0.3.0/go.mod h1:S3lq113Y91y1UBf1wj1pFOxeahvfKkCk6mTWTWbDdWs=
+github.com/ysmood/got v0.9.3 h1:qx51X49jL/WAiqZzPTkPZ0zp5pTmrWJa4zYFTYo0gHI=
+github.com/ysmood/got v0.9.3/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
+github.com/ysmood/gotrace v0.2.0 h1:IkTC6rJREwXSaG8yWK+NFwIJGIsxA1DjC6/gxYyQttE=
+github.com/ysmood/gotrace v0.2.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
+github.com/ysmood/gson v0.6.3 h1:4cU+5oOdsyundXHy00t99H0rLXLthuseD3x6W+xmCiU=
+github.com/ysmood/gson v0.6.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
+github.com/ysmood/leakless v0.6.12 h1:XxtRYl97bJklfv4BZVdyGnd/y42p6w8lu1hUzfCkT4M=
+github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
 go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9 h1:umElSU9WZirRdgu2yFHY0ayQkEnKiOC1TtM3fWXFnoU=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df h1:y7QZzfUiTwWam+xBn29Ulb8CBwVN5UdzmMDavl9Whlw=
+golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 h1:5vD4XjIc0X5+kHZjx4UecYdjA6mJo+XXNoaW0EjU5Os=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba h1:xmhUJGQGbxlod18iJGqVEp9cHIPLl7QiX2aA3to708s=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b h1:lAZ0/chPUDWwjqosYR0X4M490zQhMsiJ4K3DbA7o+3g=
+golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE=
+golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

v2/internal/bufwriter/bufwriter.go

@@ -1,74 +0,0 @@
-package bufwriter
-
-import (
-	"bufio"
-	"os"
-	"sync"
-)
-
-// Writer is a mutex protected buffered writer
-type Writer struct {
-	file   *os.File
-	writer *bufio.Writer
-	mutex  *sync.Mutex
-}
-
-// New creates a new mutex protected buffered writer for a file
-func New(file string) (*Writer, error) {
-	output, err := os.Create(file)
-	if err != nil {
-		return nil, err
-	}
-	return &Writer{file: output, writer: bufio.NewWriter(output), mutex: &sync.Mutex{}}, nil
-}
-
-// Write writes a byte slice to the underlying file
-//
-// It also writes a newline if the last byte isn't a newline character.
-func (w *Writer) Write(data []byte) error {
-	if len(data) == 0 {
-		return nil
-	}
-	w.mutex.Lock()
-	defer w.mutex.Unlock()
-
-	_, err := w.writer.Write(data)
-	if err != nil {
-		return err
-	}
-	if data[len(data)-1] != '\n' {
-		_, err = w.writer.WriteRune('\n')
-	}
-	return err
-}
-
-// WriteString writes a string to the underlying file
-//
-// It also writes a newline if the last byte isn't a newline character.
-func (w *Writer) WriteString(data string) error {
-	if data == "" {
-		return nil
-	}
-	w.mutex.Lock()
-	defer w.mutex.Unlock()
-
-	_, err := w.writer.WriteString(data)
-	if err != nil {
-		return err
-	}
-	if data[len(data)-1] != '\n' {
-		_, err = w.writer.WriteRune('\n')
-	}
-	return err
-}
-
-// Close closes the underlying writer flushing everything to disk
-func (w *Writer) Close() error {
-	w.mutex.Lock()
-	defer w.mutex.Unlock()
-
-	w.writer.Flush()
-	//nolint:errcheck // we don't care whether sync failed or succeeded.
-	w.file.Sync()
-	return w.file.Close()
-}

v2/internal/collaborator/collaborator.go

@@ -0,0 +1,72 @@
+package collaborator
+
+import (
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/projectdiscovery/collaborator"
+)
+
+var (
+	// PollSeconds is the seconds to poll at.
+	PollSeconds = 5
+	// DefaultMaxBufferLimit is the default request buffer limit
+	DefaultMaxBufferLimit = 150
+	// DefaultPollInterval is the default poll interval for burp collabortor polling.
+	DefaultPollInterval time.Duration = time.Second * time.Duration(PollSeconds)
+	// DefaultCollaborator is the default burp collaborator instance
+	DefaultCollaborator = &Collaborator{Collab: collaborator.NewBurpCollaborator()}
+)
+
+// Collaborator is a client for recording burp collaborator interactions
+type Collaborator struct {
+	sync.RWMutex
+	options *Options // unused
+	Collab  *collaborator.BurpCollaborator
+}
+
+// Options contains configuration options for collaborator client
+type Options struct {
+	BIID           string
+	PollInterval   time.Duration
+	MaxBufferLimit int
+}
+
+// New creates a new collaborator client
+func New(options *Options) *Collaborator {
+	collab := collaborator.NewBurpCollaborator()
+	collab.AddBIID(options.BIID)
+	collab.MaxBufferLimit = options.MaxBufferLimit
+	return &Collaborator{Collab: collab, options: options}
+}
+
+// Poll initiates collaborator polling if any BIIDs were provided
+func (b *Collaborator) Poll() {
+	// if no valid biids were provided just return
+	if len(b.Collab.BIIDs) > 0 {
+		go b.Collab.PollEach(DefaultPollInterval)
+	}
+}
+
+// Has checks if a collabrator hit was found for a URL
+func (b *Collaborator) Has(s string) bool {
+	for _, r := range b.Collab.RespBuffer {
+		for i := 0; i < len(r.Responses); i++ {
+			// search in dns - http - smtp
+			b.RLock()
+			found := strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) ||
+				strings.Contains(r.Responses[i].Data.RequestDecoded, s) ||
+				strings.Contains(r.Responses[i].Data.MessageDecoded, s)
+			b.RUnlock()
+
+			if found {
+				b.Lock()
+				r.Responses = append(r.Responses[:i], r.Responses[i+1:]...)
+				b.Unlock()
+				return true
+			}
+		}
+	}
+	return false
+}

v2/internal/colorizer/colorizer.go

@@ -0,0 +1,24 @@
+package colorizer
+
+import "github.com/logrusorgru/aurora"
+
+// Colorizer returns a colorized severity printer
+type Colorizer struct {
+	Data map[string]string
+}
+
+const (
+	fgOrange uint8 = 208
+)
+
+// New returns a new severity based colorizer
+func New(colorizer aurora.Aurora) *Colorizer {
+	severityMap := map[string]string{
+		"info":     colorizer.Blue("info").String(),
+		"low":      colorizer.Green("low").String(),
+		"medium":   colorizer.Yellow("medium").String(),
+		"high":     colorizer.Index(fgOrange, "high").String(),
+		"critical": colorizer.Red("critical").String(),
+	}
+	return &Colorizer{Data: severityMap}
+}

v2/internal/progress/progress.go

@@ -1,146 +0,0 @@
-package progress
-
-import (
-	"fmt"
-	"os"
-	"strings"
-	"time"
-
-	"github.com/projectdiscovery/clistats"
-	"github.com/projectdiscovery/gologger"
-)
-
-// Progress is a progress instance for showing program stats
-type Progress struct {
-	active       bool
-	stats        clistats.StatisticsClient
-	tickDuration time.Duration
-}
-
-// NewProgress creates and returns a new progress tracking object.
-func NewProgress(active bool) *Progress {
-	var tickDuration time.Duration
-	if active {
-		tickDuration = 5 * time.Second
-	} else {
-		tickDuration = -1
-	}
-
-	var progress Progress
-	if active {
-		stats, err := clistats.New()
-		if err != nil {
-			gologger.Warningf("Couldn't create progress engine: %s\n", err)
-		}
-		progress.active = active
-		progress.stats = stats
-		progress.tickDuration = tickDuration
-	}
-
-	return &progress
-}
-
-// Init initializes the progress display mechanism by setting counters, etc.
-func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) {
-	if p.active {
-		p.stats.AddStatic("templates", rulesCount)
-		p.stats.AddStatic("hosts", hostCount)
-		p.stats.AddStatic("startedAt", time.Now())
-		p.stats.AddCounter("requests", uint64(0))
-		p.stats.AddCounter("errors", uint64(0))
-		p.stats.AddCounter("total", uint64(requestCount))
-		if err := p.stats.Start(makePrintCallback(), p.tickDuration); err != nil {
-			gologger.Warningf("Couldn't start statistics: %s\n", err)
-		}
-	}
-}
-
-// AddToTotal adds a value to the total request count
-func (p *Progress) AddToTotal(delta int64) {
-	if p.active {
-		p.stats.IncrementCounter("total", int(delta))
-	}
-}
-
-// Update progress tracking information and increments the request counter by one unit.
-func (p *Progress) Update() {
-	if p.active {
-		p.stats.IncrementCounter("requests", 1)
-	}
-}
-
-// Drop drops the specified number of requests from the progress bar total.
-// This may be the case when uncompleted requests are encountered and shouldn't be part of the total count.
-func (p *Progress) Drop(count int64) {
-	if p.active {
-		// mimic dropping by incrementing the completed requests
-		p.stats.IncrementCounter("errors", int(count))
-	}
-}
-
-const bufferSize = 128
-
-func makePrintCallback() func(stats clistats.StatisticsClient) {
-	builder := &strings.Builder{}
-	builder.Grow(bufferSize)
-
-	return func(stats clistats.StatisticsClient) {
-		builder.WriteRune('[')
-		startedAt, _ := stats.GetStatic("startedAt")
-		duration := time.Since(startedAt.(time.Time))
-		builder.WriteString(fmtDuration(duration))
-		builder.WriteRune(']')
-
-		templates, _ := stats.GetStatic("templates")
-		builder.WriteString(" | Templates: ")
-		builder.WriteString(clistats.String(templates))
-		hosts, _ := stats.GetStatic("hosts")
-		builder.WriteString(" | Hosts: ")
-		builder.WriteString(clistats.String(hosts))
-
-		requests, _ := stats.GetCounter("requests")
-		total, _ := stats.GetCounter("total")
-
-		builder.WriteString(" | RPS: ")
-		builder.WriteString(clistats.String(uint64(float64(requests) / duration.Seconds())))
-
-		errors, _ := stats.GetCounter("errors")
-		builder.WriteString(" | Errors: ")
-		builder.WriteString(clistats.String(errors))
-
-		builder.WriteString(" | Requests: ")
-		builder.WriteString(clistats.String(requests))
-		builder.WriteRune('/')
-		builder.WriteString(clistats.String(total))
-		builder.WriteRune(' ')
-		builder.WriteRune('(')
-		//nolint:gomnd // this is not a magic number
-		builder.WriteString(clistats.String(uint64(float64(requests) / float64(total) * 100.0)))
-		builder.WriteRune('%')
-		builder.WriteRune(')')
-		builder.WriteRune('\n')
-
-		fmt.Fprintf(os.Stderr, "%s", builder.String())
-		builder.Reset()
-	}
-}
-
-// fmtDuration formats the duration for the time elapsed
-func fmtDuration(d time.Duration) string {
-	d = d.Round(time.Second)
-	h := d / time.Hour
-	d -= h * time.Hour
-	m := d / time.Minute
-	d -= m * time.Minute
-	s := d / time.Second
-	return fmt.Sprintf("%d:%02d:%02d", h, m, s)
-}
-
-// Stop stops the progress bar execution
-func (p *Progress) Stop() {
-	if p.active {
-		if err := p.stats.Stop(); err != nil {
-			gologger.Warningf("Couldn't stop statistics: %s\n", err)
-		}
-	}
-}

v2/internal/runner/banner.go

@@ -7,17 +7,17 @@ const banner = `
      ____  __  _______/ /__  (_)
     / __ \/ / / / ___/ / _ \/ /
    / / / / /_/ / /__/ /  __/ /
-  /_/ /_/\__,_/\___/_/\___/_/   v2.2.0
+  /_/ /_/\__,_/\___/_/\___/_/   v2.3.2
 `
 
 // Version is the current version of nuclei
-const Version = `2.2.0`
+const Version = `2.3.2`
 
 // showBanner is used to show the banner to the user
 func showBanner() {
-	gologger.Printf("%s\n", banner)
-	gologger.Printf("\t\tprojectdiscovery.io\n\n")
+	gologger.Print().Msgf("%s\n", banner)
+	gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n")
 
-	gologger.Labelf("Use with caution. You are responsible for your actions\n")
-	gologger.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n")
+	gologger.Warning().Msgf("Use with caution. You are responsible for your actions\n")
+	gologger.Warning().Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
 }

v2/internal/runner/config.go

@@ -16,13 +16,15 @@ type nucleiConfig struct {
 	TemplatesDirectory string    `json:"templates-directory,omitempty"`
 	CurrentVersion     string    `json:"current-version,omitempty"`
 	LastChecked        time.Time `json:"last-checked,omitempty"`
-
+	IgnoreURL          string    `json:"ignore-url,omitempty"`
+	NucleiVersion      string    `json:"nuclei-version,omitempty"`
+	LastCheckedIgnore  time.Time `json:"last-checked-ignore,omitempty"`
 	// IgnorePaths ignores all the paths listed unless specified manually
 	IgnorePaths []string `json:"ignore-paths,omitempty"`
 }
 
 // nucleiConfigFilename is the filename of nuclei configuration file.
-const nucleiConfigFilename = ".nuclei-config.json"
+const nucleiConfigFilename = ".templates-config.json"
 
 var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
 
@@ -32,8 +34,10 @@ func readConfiguration() (*nucleiConfig, error) {
 	if err != nil {
 		return nil, err
 	}
+	configDir := path.Join(home, "/.config", "/nuclei")
+	_ = os.MkdirAll(configDir, os.ModePerm)
 
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
+	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
 	file, err := os.Open(templatesConfigFile)
 	if err != nil {
 		return nil, err
@@ -54,9 +58,16 @@ func (r *Runner) writeConfiguration(config *nucleiConfig) error {
 	if err != nil {
 		return err
 	}
+	configDir := path.Join(home, "/.config", "/nuclei")
+	_ = os.MkdirAll(configDir, os.ModePerm)
 
+	if config.IgnoreURL == "" {
+		config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
+	}
 	config.LastChecked = time.Now()
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
+	config.LastCheckedIgnore = time.Now()
+	config.NucleiVersion = Version
+	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
 	file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
 	if err != nil {
 		return err
@@ -93,44 +104,27 @@ func (r *Runner) readNucleiIgnoreFile() {
 	}
 }
 
-// checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules.
-func (r *Runner) checkIfInNucleiIgnore(item string) bool {
-	if r.templatesConfig == nil {
-		return false
-	}
-
-	for _, paths := range r.templatesConfig.IgnorePaths {
-		// If we have a path to ignore, check if it's in the item.
-		if paths[len(paths)-1] == '/' {
-			if strings.Contains(item, paths) {
-				return true
-			}
-
-			continue
-		}
-		// Check for file based extension in ignores
-		if strings.HasSuffix(item, paths) {
-			return true
-		}
-	}
-
-	return false
-}
-
+// getIgnoreFilePath returns the ignore file path for the runner
 func (r *Runner) getIgnoreFilePath() string {
-	defIgnoreFilePath := path.Join(r.templatesConfig.TemplatesDirectory, nucleiIgnoreFile)
+	var defIgnoreFilePath string
+
+	home, err := os.UserHomeDir()
+	if err == nil {
+		configDir := path.Join(home, "/.config", "/nuclei")
+		_ = os.MkdirAll(configDir, os.ModePerm)
+
+		defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile)
+	}
 
 	cwd, err := os.Getwd()
 	if err != nil {
 		return defIgnoreFilePath
 	}
-
 	cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile)
 
 	cwdIfpInfo, err := os.Stat(cwdIgnoreFilePath)
 	if os.IsNotExist(err) || cwdIfpInfo.IsDir() {
 		return defIgnoreFilePath
 	}
-
 	return cwdIgnoreFilePath
 }

v2/internal/runner/options.go

@@ -1,136 +1,67 @@
 package runner
 
 import (
+	"bufio"
 	"errors"
-	"flag"
 	"net/url"
 	"os"
+	"strings"
 
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
+	"github.com/projectdiscovery/gologger/formatter"
+	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
-// Options contains the configuration options for tuning
-// the template requesting process.
-// nolint // false positive, options are allocated once and are necessary as is
-type Options struct {
-	Debug                bool                   // Debug mode allows debugging request/responses for the engine
-	Silent               bool                   // Silent suppresses any extra text and only writes found URLs on screen.
-	Version              bool                   // Version specifies if we should just show version and exit
-	Verbose              bool                   // Verbose flag indicates whether to show verbose output or not
-	NoColor              bool                   // No-Color disables the colored output.
-	UpdateTemplates      bool                   // UpdateTemplates updates the templates installed at startup
-	JSON                 bool                   // JSON writes json output to files
-	JSONRequests         bool                   // write requests/responses for matches in JSON output
-	EnableProgressBar    bool                   // Enable progrss bar
-	TemplatesVersion     bool                   // Show the templates installed version
-	TemplateList         bool                   // List available templates
-	Stdin                bool                   // Stdin specifies whether stdin input was given to the process
-	StopAtFirstMatch     bool                   // Stop processing template at first full match (this may break chained requests)
-	NoMeta               bool                   // Don't display metadata for the matches
-	BulkSize             int                    // Number of targets analyzed in parallel for each template
-	TemplateThreads      int                    // Number of templates executed in parallel
-	Project              bool                   // Nuclei uses project folder to avoid sending same HTTP request multiple times
-	ProjectPath          string                 // Nuclei uses a user defined project folder
-	Timeout              int                    // Timeout is the seconds to wait for a response from the server.
-	Retries              int                    // Retries is the number of times to retry the request
-	RateLimit            int                    // Rate-Limit of requests per specified target
-	Severity             string                 // Filter templates based on their severity and only run the matching ones.
-	Target               string                 // Target is a single URL/Domain to scan usng a template
-	Targets              string                 // Targets specifies the targets to scan using templates.
-	Output               string                 // Output is the file to write found subdomains to.
-	ProxyURL             string                 // ProxyURL is the URL for the proxy server
-	ProxySocksURL        string                 // ProxySocksURL is the URL for the proxy socks server
-	TemplatesDirectory   string                 // TemplatesDirectory is the directory to use for storing templates
-	TraceLogFile         string                 // TraceLogFile specifies a file to write with the trace of all requests
-	Templates            multiStringFlag        // Signature specifies the template/templates to use
-	ExcludedTemplates    multiStringFlag        // Signature specifies the template/templates to exclude
-	CustomHeaders        requests.CustomHeaders // Custom global headers
-	Threads              int                    // Thread controls the number of concurrent requests to make.
-	BurpCollaboratorBiid string                 // Burp Collaborator BIID for polling
-}
-
-type multiStringFlag []string
-
-func (m *multiStringFlag) String() string {
-	return ""
-}
-
-func (m *multiStringFlag) Set(value string) error {
-	*m = append(*m, value)
-	return nil
-}
-
 // ParseOptions parses the command line flags provided by a user
-func ParseOptions() *Options {
-	options := &Options{}
-
-	flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template")
-	flag.Var(&options.Templates, "t", "Template input dir/file/files to run on host. Can be used multiple times. Supports globbing.")
-	flag.Var(&options.ExcludedTemplates, "exclude", "Template input dir/file/files to exclude. Can be used multiple times. Supports globbing.")
-	flag.StringVar(&options.Severity, "severity", "", "Filter templates based on their severity and only run the matching ones. Comma-separated values can be used to specify multiple severities.")
-	flag.StringVar(&options.Targets, "l", "", "List of URLs to run templates on")
-	flag.StringVar(&options.Output, "o", "", "File to write output to (optional)")
-	flag.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server")
-	flag.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the proxy socks server")
-	flag.BoolVar(&options.Silent, "silent", false, "Show only results in output")
-	flag.BoolVar(&options.Version, "version", false, "Show version of nuclei")
-	flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output")
-	flag.BoolVar(&options.NoColor, "no-color", false, "Disable colors in output")
-	flag.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout")
-	flag.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
-	flag.Var(&options.CustomHeaders, "H", "Custom Header.")
-	flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses")
-	flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)")
-	flag.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log")
-	flag.StringVar(&options.TemplatesDirectory, "update-directory", "", "Directory to use for storing nuclei-templates")
-	flag.BoolVar(&options.JSON, "json", false, "Write json output to files")
-	flag.BoolVar(&options.JSONRequests, "include-rr", false, "Write requests/responses for matches in JSON output")
-	flag.BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan")
-	flag.BoolVar(&options.TemplateList, "tl", false, "List available templates")
-	flag.IntVar(&options.RateLimit, "rate-limit", 150, "Rate-Limit (maximum requests/second")
-	flag.BoolVar(&options.StopAtFirstMatch, "stop-at-first-match", false, "Stop processing http requests at first match (this may break template/workflow logic)")
-	flag.IntVar(&options.BulkSize, "bulk-size", 25, "Maximum Number of hosts analyzed in parallel per template")
-	flag.IntVar(&options.TemplateThreads, "c", 10, "Maximum Number of templates executed in parallel")
-	flag.BoolVar(&options.Project, "project", false, "Use a project folder to avoid sending same request multiple times")
-	flag.StringVar(&options.ProjectPath, "project-path", "", "Use a user defined project folder, temporary folder is used if not specified but enabled")
-	flag.BoolVar(&options.NoMeta, "no-meta", false, "Don't display metadata for the matches")
-	flag.BoolVar(&options.TemplatesVersion, "templates-version", false, "Shows the installed nuclei-templates version")
-	flag.StringVar(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "", "Burp Collaborator BIID")
-	flag.Parse()
-
+func ParseOptions(options *types.Options) {
 	// Check if stdin pipe was given
 	options.Stdin = hasStdin()
 
 	// Read the inputs and configure the logging
-	options.configureOutput()
+	configureOutput(options)
 
 	// Show the user the banner
 	showBanner()
 
+	options.ExcludeTags = append(options.ExcludeTags, "dos")
 	if options.Version {
-		gologger.Infof("Current Version: %s\n", Version)
+		gologger.Info().Msgf("Current Version: %s\n", Version)
 		os.Exit(0)
 	}
 	if options.TemplatesVersion {
 		config, err := readConfiguration()
 		if err != nil {
-			gologger.Fatalf("Could not read template configuration: %s\n", err)
+			gologger.Fatal().Msgf("Could not read template configuration: %s\n", err)
 		}
-		gologger.Infof("Current nuclei-templates version: %s (%s)\n", config.CurrentVersion, config.TemplatesDirectory)
+		gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", config.CurrentVersion, config.TemplatesDirectory)
 		os.Exit(0)
 	}
 
 	// Validate the options passed by the user and if any
 	// invalid options have been used, exit.
-	err := options.validateOptions()
-	if err != nil {
-		gologger.Fatalf("Program exiting: %s\n", err)
+	if err := validateOptions(options); err != nil {
+		gologger.Fatal().Msgf("Program exiting: %s\n", err)
 	}
 
-	return options
+	// Auto adjust rate limits when using headless mode if the user
+	// hasn't specified any custom limits.
+	if options.Headless && options.BulkSize == 25 && options.TemplateThreads == 10 {
+		options.BulkSize = 2
+		options.TemplateThreads = 2
+	}
+
+	// Load the resolvers if user asked for them
+	loadResolvers(options)
+
+	err := protocolinit.Init(options)
+	if err != nil {
+		gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err)
+	}
 }
 
+// hasStdin returns true if we have stdin input
 func hasStdin() bool {
 	stat, err := os.Stdin.Stat()
 	if err != nil {
@@ -144,7 +75,7 @@ func hasStdin() bool {
 }
 
 // validateOptions validates the configuration options passed
-func (options *Options) validateOptions() error {
+func validateOptions(options *types.Options) error {
 	// Both verbose and silent flags were used
 	if options.Verbose && options.Silent {
 		return errors.New("both verbose and silent mode specified")
@@ -152,32 +83,21 @@ func (options *Options) validateOptions() error {
 
 	if !options.TemplateList {
 		// Check if a list of templates was provided and it exists
-		if len(options.Templates) == 0 && !options.UpdateTemplates {
+		if len(options.Templates) == 0 && !options.NewTemplates && len(options.Workflows) == 0 && len(options.Tags) == 0 && !options.UpdateTemplates {
 			return errors.New("no template/templates provided")
 		}
-
-		if options.Targets == "" && !options.Stdin && options.Target == "" && !options.UpdateTemplates {
-			return errors.New("no target input provided")
-		}
 	}
 
 	// Validate proxy options if provided
-	err := validateProxyURL(
-		options.ProxyURL,
-		"invalid http proxy format (It should be http://username:password@host:port)",
-	)
+	err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)")
 	if err != nil {
 		return err
 	}
 
-	err = validateProxyURL(
-		options.ProxySocksURL,
-		"invalid socks proxy format (It should be socks5://username:password@host:port)",
-	)
+	err = validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)")
 	if err != nil {
 		return err
 	}
-
 	return nil
 }
 
@@ -191,22 +111,48 @@ func validateProxyURL(proxyURL, message string) error {
 
 func isValidURL(urlString string) bool {
 	_, err := url.Parse(urlString)
-
 	return err == nil
 }
 
 // configureOutput configures the output on the screen
-func (options *Options) configureOutput() {
+func configureOutput(options *types.Options) {
 	// If the user desires verbose output, show verbose output
 	if options.Verbose {
-		gologger.MaxLevel = gologger.Verbose
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
+	}
+	if options.Debug {
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
 	}
-
 	if options.NoColor {
-		gologger.UseColors = false
+		gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true))
 	}
-
 	if options.Silent {
-		gologger.MaxLevel = gologger.Silent
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
+	}
+}
+
+// loadResolvers loads resolvers from both user provided flag and file
+func loadResolvers(options *types.Options) {
+	if options.ResolversFile == "" {
+		return
+	}
+
+	file, err := os.Open(options.ResolversFile)
+	if err != nil {
+		gologger.Fatal().Msgf("Could not open resolvers file: %s\n", err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		part := scanner.Text()
+		if part == "" {
+			continue
+		}
+		if strings.Contains(part, ":") {
+			options.InternalResolversList = append(options.InternalResolversList, part)
+		} else {
+			options.InternalResolversList = append(options.InternalResolversList, part+":53")
+		}
 	}
 }

v2/internal/runner/paths.go

@@ -1,48 +0,0 @@
-package runner
-
-import (
-	"fmt"
-	"os"
-	"path"
-	"strings"
-
-	"github.com/projectdiscovery/gologger"
-)
-
-// isRelative checks if a given path is a relative path
-func isRelative(filePath string) bool {
-	if strings.HasPrefix(filePath, "/") || strings.Contains(filePath, ":\\") {
-		return false
-	}
-
-	return true
-}
-
-// resolvePath gets the absolute path to the template by either
-// looking in the current directory or checking the nuclei templates directory.
-//
-// Current directory is given preference over the nuclei-templates directory.
-func (r *Runner) resolvePath(templateName string) (string, error) {
-	curDirectory, err := os.Getwd()
-	if err != nil {
-		return "", err
-	}
-
-	templatePath := path.Join(curDirectory, templateName)
-	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
-		gologger.Debugf("Found template in current directory: %s\n", templatePath)
-
-		return templatePath, nil
-	}
-
-	if r.templatesConfig != nil {
-		templatePath := path.Join(r.templatesConfig.TemplatesDirectory, templateName)
-		if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
-			gologger.Debugf("Found template in nuclei-templates directory: %s\n", templatePath)
-
-			return templatePath, nil
-		}
-	}
-
-	return "", fmt.Errorf("no such path found: %s", templateName)
-}

v2/internal/runner/processor.go

@@ -1,332 +1,50 @@
 package runner
 
 import (
-	"context"
-	"fmt"
-	"net/http/cookiejar"
-	"os"
-	"path"
-	"path/filepath"
-	"strings"
-
-	tengo "github.com/d5/tengo/v2"
-	"github.com/d5/tengo/v2/stdlib"
-	"github.com/karrick/godirwalk"
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/internal/progress"
-	"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
-	"github.com/projectdiscovery/nuclei/v2/pkg/executer"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
 	"github.com/remeh/sizedwaitgroup"
+	"go.uber.org/atomic"
 )
 
-// workflowTemplates contains the initialized workflow templates per template group
-type workflowTemplates struct {
-	Name      string
-	Templates []*workflows.Template
-}
-
-// processTemplateWithList processes a template and runs the enumeration on all the targets
-func (r *Runner) processTemplateWithList(p *progress.Progress, template *templates.Template, request interface{}) bool {
-	var httpExecuter *executer.HTTPExecuter
-	var dnsExecuter *executer.DNSExecuter
-	var err error
-
-	// Create an executer based on the request type.
-	switch value := request.(type) {
-	case *requests.DNSRequest:
-		dnsExecuter = executer.NewDNSExecuter(&executer.DNSOptions{
-			TraceLog:      r.traceLog,
-			Debug:         r.options.Debug,
-			Template:      template,
-			DNSRequest:    value,
-			Writer:        r.output,
-			JSON:          r.options.JSON,
-			JSONRequests:  r.options.JSONRequests,
-			NoMeta:        r.options.NoMeta,
-			ColoredOutput: !r.options.NoColor,
-			Colorizer:     r.colorizer,
-			Decolorizer:   r.decolorizer,
-			RateLimiter:   r.ratelimiter,
-		})
-	case *requests.BulkHTTPRequest:
-		httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{
-			TraceLog:         r.traceLog,
-			Debug:            r.options.Debug,
-			Template:         template,
-			BulkHTTPRequest:  value,
-			Writer:           r.output,
-			Timeout:          r.options.Timeout,
-			Retries:          r.options.Retries,
-			ProxyURL:         r.options.ProxyURL,
-			ProxySocksURL:    r.options.ProxySocksURL,
-			CustomHeaders:    r.options.CustomHeaders,
-			JSON:             r.options.JSON,
-			JSONRequests:     r.options.JSONRequests,
-			NoMeta:           r.options.NoMeta,
-			CookieReuse:      value.CookieReuse,
-			ColoredOutput:    !r.options.NoColor,
-			Colorizer:        &r.colorizer,
-			Decolorizer:      r.decolorizer,
-			StopAtFirstMatch: r.options.StopAtFirstMatch,
-			PF:               r.pf,
-			Dialer:           r.dialer,
-			RateLimiter:      r.ratelimiter,
-		})
-	}
-
-	if err != nil {
-		p.Drop(request.(*requests.BulkHTTPRequest).GetRequestCount())
-		gologger.Warningf("Could not create http client: %s\n", err)
-
-		return false
-	}
-
-	var globalresult atomicboolean.AtomBool
-
+// processTemplateWithList process a template on the URL list
+func (r *Runner) processTemplateWithList(template *templates.Template) bool {
+	results := &atomic.Bool{}
 	wg := sizedwaitgroup.New(r.options.BulkSize)
-
-	r.hm.Scan(func(k, _ []byte) error {
+	r.hostMap.Scan(func(k, _ []byte) error {
 		URL := string(k)
+
 		wg.Add()
 		go func(URL string) {
 			defer wg.Done()
 
-			var result *executer.Result
-
-			if httpExecuter != nil {
-				result = httpExecuter.ExecuteHTTP(p, URL)
-				globalresult.Or(result.GotResults)
-			}
-
-			if dnsExecuter != nil {
-				result = dnsExecuter.ExecuteDNS(p, URL)
-				globalresult.Or(result.GotResults)
-			}
-
-			if result.Error != nil {
-				gologger.Warningf("[%s] Could not execute step: %s\n", r.colorizer.Colorizer.BrightBlue(template.ID), result.Error)
+			match, err := template.Executer.Execute(URL)
+			if err != nil {
+				gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err)
 			}
+			results.CAS(false, match)
 		}(URL)
-
 		return nil
 	})
-
 	wg.Wait()
-
-	// See if we got any results from the executers
-	return globalresult.Get()
+	return results.Load()
 }
 
-// ProcessWorkflowWithList coming from stdin or list of targets
-func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflows.Workflow) bool {
-	result := false
-
-	workflowTemplatesList, err := r.preloadWorkflowTemplates(p, workflow)
-	if err != nil {
-		gologger.Warningf("Could not preload templates for workflow %s: %s\n", workflow.ID, err)
-		return result
-	}
-
-	logicBytes := []byte(workflow.Logic)
-
+// processTemplateWithList process a template on the URL list
+func (r *Runner) processWorkflowWithList(template *templates.Template) bool {
+	results := &atomic.Bool{}
 	wg := sizedwaitgroup.New(r.options.BulkSize)
 
-	r.hm.Scan(func(k, _ []byte) error {
-		targetURL := string(k)
+	r.hostMap.Scan(func(k, _ []byte) error {
+		URL := string(k)
 		wg.Add()
-
-		go func(targetURL string) {
+		go func(URL string) {
 			defer wg.Done()
-
-			script := tengo.NewScript(logicBytes)
-			script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
-
-			variables := make(map[string]*workflows.NucleiVar)
-
-			for _, workflowTemplate := range *workflowTemplatesList {
-				name := workflowTemplate.Name
-				variable := &workflows.NucleiVar{Templates: workflowTemplate.Templates, URL: targetURL}
-				err := script.Add(name, variable)
-				if err != nil {
-					gologger.Errorf("Could not initialize script for workflow '%s': %s\n", workflow.ID, err)
-					continue
-				}
-				variables[name] = variable
-			}
-
-			_, err := script.RunContext(context.Background())
-			if err != nil {
-				gologger.Errorf("Could not execute workflow '%s': %s\n", workflow.ID, err)
-			}
-
-			for _, variable := range variables {
-				result = !variable.IsFalsy()
-				if result {
-					break
-				}
-			}
-		}(targetURL)
+			match := template.CompiledWorkflow.RunWorkflow(URL)
+			results.CAS(false, match)
+		}(URL)
 		return nil
 	})
-
 	wg.Wait()
-
-	return result
-}
-
-func (r *Runner) preloadWorkflowTemplates(p *progress.Progress, workflow *workflows.Workflow) (*[]workflowTemplates, error) {
-	var jar *cookiejar.Jar
-
-	if workflow.CookieReuse {
-		var err error
-		jar, err = cookiejar.New(nil)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// Single yaml provided
-	var wflTemplatesList []workflowTemplates
-
-	for name, value := range workflow.Variables {
-		// Check if the template is an absolute path or relative path.
-		// If the path is absolute, use it. Otherwise,
-		if isRelative(value) {
-			newPath, err := r.resolvePath(value)
-			if err != nil {
-				newPath, err = resolvePathWithBaseFolder(filepath.Dir(workflow.GetPath()), value)
-				if err != nil {
-					return nil, err
-				}
-			}
-
-			value = newPath
-		}
-
-		var wtlst []*workflows.Template
-
-		if strings.HasSuffix(value, ".yaml") {
-			t, err := templates.Parse(value)
-			if err != nil {
-				return nil, err
-			}
-
-			template := &workflows.Template{Progress: p}
-			if len(t.BulkRequestsHTTP) > 0 {
-				template.HTTPOptions = &executer.HTTPOptions{
-					TraceLog:         r.traceLog,
-					Debug:            r.options.Debug,
-					Writer:           r.output,
-					Template:         t,
-					Timeout:          r.options.Timeout,
-					Retries:          r.options.Retries,
-					ProxyURL:         r.options.ProxyURL,
-					ProxySocksURL:    r.options.ProxySocksURL,
-					CustomHeaders:    r.options.CustomHeaders,
-					JSON:             r.options.JSON,
-					JSONRequests:     r.options.JSONRequests,
-					CookieJar:        jar,
-					ColoredOutput:    !r.options.NoColor,
-					Colorizer:        &r.colorizer,
-					Decolorizer:      r.decolorizer,
-					PF:               r.pf,
-					RateLimiter:      r.ratelimiter,
-					NoMeta:           r.options.NoMeta,
-					StopAtFirstMatch: r.options.StopAtFirstMatch,
-					Dialer:           r.dialer,
-				}
-			} else if len(t.RequestsDNS) > 0 {
-				template.DNSOptions = &executer.DNSOptions{
-					TraceLog:      r.traceLog,
-					Debug:         r.options.Debug,
-					Template:      t,
-					Writer:        r.output,
-					JSON:          r.options.JSON,
-					JSONRequests:  r.options.JSONRequests,
-					ColoredOutput: !r.options.NoColor,
-					Colorizer:     r.colorizer,
-					Decolorizer:   r.decolorizer,
-					NoMeta:        r.options.NoMeta,
-					RateLimiter:   r.ratelimiter,
-				}
-			}
-
-			if template.DNSOptions != nil || template.HTTPOptions != nil {
-				wtlst = append(wtlst, template)
-			}
-		} else {
-			matches := []string{}
-
-			err := godirwalk.Walk(value, &godirwalk.Options{
-				Callback: func(path string, d *godirwalk.Dirent) error {
-					if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
-						matches = append(matches, path)
-					}
-
-					return nil
-				},
-				ErrorCallback: func(path string, err error) godirwalk.ErrorAction {
-					return godirwalk.SkipNode
-				},
-				Unsorted: true,
-			})
-
-			if err != nil {
-				return nil, err
-			}
-
-			// 0 matches means no templates were found in directory
-			if len(matches) == 0 {
-				return nil, fmt.Errorf("no match found in the directory %s", value)
-			}
-
-			for _, match := range matches {
-				t, err := templates.Parse(match)
-				if err != nil {
-					return nil, err
-				}
-				template := &workflows.Template{Progress: p}
-				if len(t.BulkRequestsHTTP) > 0 {
-					template.HTTPOptions = &executer.HTTPOptions{
-						Debug:         r.options.Debug,
-						Writer:        r.output,
-						Template:      t,
-						Timeout:       r.options.Timeout,
-						Retries:       r.options.Retries,
-						ProxyURL:      r.options.ProxyURL,
-						ProxySocksURL: r.options.ProxySocksURL,
-						CustomHeaders: r.options.CustomHeaders,
-						CookieJar:     jar,
-						TraceLog:      r.traceLog,
-					}
-				} else if len(t.RequestsDNS) > 0 {
-					template.DNSOptions = &executer.DNSOptions{
-						Debug:    r.options.Debug,
-						Template: t,
-						Writer:   r.output,
-						TraceLog: r.traceLog,
-					}
-				}
-				if template.DNSOptions != nil || template.HTTPOptions != nil {
-					wtlst = append(wtlst, template)
-				}
-			}
-		}
-		wflTemplatesList = append(wflTemplatesList, workflowTemplates{Name: name, Templates: wtlst})
-	}
-
-	return &wflTemplatesList, nil
-}
-
-func resolvePathWithBaseFolder(baseFolder, templateName string) (string, error) {
-	templatePath := path.Join(baseFolder, templateName)
-	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
-		gologger.Debugf("Found template in current directory: %s\n", templatePath)
-		return templatePath, nil
-	}
-
-	return "", fmt.Errorf("no such path found: %s", templateName)
+	return results.Load()
 }

v2/internal/runner/runner.go

@@ -2,102 +2,119 @@ package runner
 
 import (
 	"bufio"
+	"fmt"
 	"os"
-	"regexp"
+	"path"
 	"strings"
 
 	"github.com/logrusorgru/aurora"
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/hmap/store/hybrid"
-	"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
-	"github.com/projectdiscovery/nuclei/v2/internal/progress"
-	"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
-	"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
-	"github.com/projectdiscovery/nuclei/v2/pkg/collaborator"
-	"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
+	"github.com/projectdiscovery/nuclei/v2/internal/collaborator"
+	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/remeh/sizedwaitgroup"
+	"github.com/rs/xid"
+	"go.uber.org/atomic"
 	"go.uber.org/ratelimit"
+	"gopkg.in/yaml.v2"
 )
 
 // Runner is a client for running the enumeration process.
 type Runner struct {
-	inputCount int64
-
-	traceLog tracelog.Log
-
-	// output is the output file to write if any
-	output *bufwriter.Writer
-
+	hostMap         *hybrid.HybridMap
+	output          output.Writer
+	inputCount      int64
 	templatesConfig *nucleiConfig
-	// options contains configuration options for runner
-	options *Options
-
-	pf *projectfile.ProjectFile
-
-	// progress tracking
-	progress *progress.Progress
-
-	// output coloring
-	colorizer   colorizer.NucleiColorizer
-	decolorizer *regexp.Regexp
-
-	// rate limiter
-	ratelimiter ratelimit.Limiter
-
-	// input deduplication
-	hm     *hybrid.HybridMap
-	dialer *fastdialer.Dialer
+	options         *types.Options
+	projectFile     *projectfile.ProjectFile
+	catalog         *catalog.Catalog
+	progress        progress.Progress
+	colorizer       aurora.Aurora
+	issuesClient    *reporting.Client
+	severityColors  *colorizer.Colorizer
+	browser         *engine.Browser
+	ratelimiter     ratelimit.Limiter
 }
 
 // New creates a new client for running enumeration process.
-func New(options *Options) (*Runner, error) {
+func New(options *types.Options) (*Runner, error) {
 	runner := &Runner{
-		traceLog: &tracelog.NoopLogger{},
-		options:  options,
+		options: options,
 	}
-	if options.TraceLogFile != "" {
-		fileLog, err := tracelog.NewFileLogger(options.TraceLogFile)
+	if options.Headless {
+		browser, err := engine.New(options)
 		if err != nil {
-			return nil, errors.Wrap(err, "could not create file trace logger")
+			return nil, err
 		}
-		runner.traceLog = fileLog
+		runner.browser = browser
 	}
-
 	if err := runner.updateTemplates(); err != nil {
-		gologger.Labelf("Could not update templates: %s\n", err)
+		gologger.Warning().Msgf("Could not update templates: %s\n", err)
+	}
+	// Read nucleiignore file if given a templateconfig
+	if runner.templatesConfig != nil {
+		runner.readNucleiIgnoreFile()
+	}
+	runner.catalog = catalog.New(runner.options.TemplatesDirectory)
+
+	var reportingOptions *reporting.Options
+	if options.ReportingConfig != "" {
+		file, err := os.Open(options.ReportingConfig)
+		if err != nil {
+			gologger.Fatal().Msgf("Could not open reporting config file: %s\n", err)
+		}
+
+		reportingOptions = &reporting.Options{}
+		if parseErr := yaml.NewDecoder(file).Decode(reportingOptions); parseErr != nil {
+			file.Close()
+			gologger.Fatal().Msgf("Could not parse reporting config file: %s\n", parseErr)
+		}
+		file.Close()
+	}
+	if options.DiskExportDirectory != "" {
+		if reportingOptions != nil {
+			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
+		} else {
+			reportingOptions = &reporting.Options{}
+			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
+		}
+	}
+	if reportingOptions != nil {
+		if client, err := reporting.New(reportingOptions, options.ReportingDB); err != nil {
+			gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err)
+		} else {
+			runner.issuesClient = client
+		}
 	}
 
 	// output coloring
 	useColor := !options.NoColor
-	runner.colorizer = *colorizer.NewNucleiColorizer(aurora.NewAurora(useColor))
-
-	if useColor {
-		// compile a decolorization regex to cleanup file output messages
-		runner.decolorizer = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
-	}
+	runner.colorizer = aurora.NewAurora(useColor)
+	runner.severityColors = colorizer.New(runner.colorizer)
 
 	if options.TemplateList {
 		runner.listAvailableTemplates()
 		os.Exit(0)
 	}
 
-	if (len(options.Templates) == 0 || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
+	if (len(options.Templates) == 0 || !options.NewTemplates || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
 		os.Exit(0)
 	}
-	// Read nucleiignore file if given a templateconfig
-	if runner.templatesConfig != nil {
-		runner.readNucleiIgnoreFile()
-	}
-
 	if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil {
-		gologger.Fatalf("Could not create temporary input file: %s\n", err)
+		gologger.Fatal().Msgf("Could not create temporary input file: %s\n", err)
 	} else {
-		runner.hm = hm
+		runner.hostMap = hm
 	}
 
 	runner.inputCount = 0
@@ -107,7 +124,7 @@ func New(options *Options) (*Runner, error) {
 	if options.Target != "" {
 		runner.inputCount++
 		// nolint:errcheck // ignoring error
-		runner.hm.Set(options.Target, nil)
+		runner.hostMap.Set(options.Target, nil)
 	}
 
 	// Handle stdin
@@ -115,20 +132,16 @@ func New(options *Options) (*Runner, error) {
 		scanner := bufio.NewScanner(os.Stdin)
 		for scanner.Scan() {
 			url := strings.TrimSpace(scanner.Text())
-			// skip empty lines
 			if url == "" {
 				continue
 			}
-
-			// skip dupes
-			if _, ok := runner.hm.Get(url); ok {
+			if _, ok := runner.hostMap.Get(url); ok {
 				dupeCount++
 				continue
 			}
-
 			runner.inputCount++
 			// nolint:errcheck // ignoring error
-			runner.hm.Set(url, nil)
+			runner.hostMap.Set(url, nil)
 		}
 	}
 
@@ -136,49 +149,47 @@ func New(options *Options) (*Runner, error) {
 	if options.Targets != "" {
 		input, err := os.Open(options.Targets)
 		if err != nil {
-			gologger.Fatalf("Could not open targets file '%s': %s\n", options.Targets, err)
+			gologger.Fatal().Msgf("Could not open targets file '%s': %s\n", options.Targets, err)
 		}
 		scanner := bufio.NewScanner(input)
 		for scanner.Scan() {
 			url := strings.TrimSpace(scanner.Text())
-			// skip empty lines
 			if url == "" {
 				continue
 			}
-
-			// skip dupes
-			if _, ok := runner.hm.Get(url); ok {
+			if _, ok := runner.hostMap.Get(url); ok {
 				dupeCount++
 				continue
 			}
-
 			runner.inputCount++
 			// nolint:errcheck // ignoring error
-			runner.hm.Set(url, nil)
+			runner.hostMap.Set(url, nil)
 		}
 		input.Close()
 	}
 
 	if dupeCount > 0 {
-		gologger.Labelf("Supplied input was automatically deduplicated (%d removed).", dupeCount)
+		gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", dupeCount)
 	}
 
 	// Create the output file if asked
-	if options.Output != "" {
-		output, errBufWriter := bufwriter.New(options.Output)
-		if errBufWriter != nil {
-			gologger.Fatalf("Could not create output file '%s': %s\n", options.Output, errBufWriter)
-		}
-		runner.output = output
+	outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.JSON, options.Output, options.TraceLogFile)
+	if err != nil {
+		gologger.Fatal().Msgf("Could not create output file '%s': %s\n", options.Output, err)
 	}
+	runner.output = outputWriter
 
 	// Creates the progress tracking object
-	runner.progress = progress.NewProgress(options.EnableProgressBar)
+	var progressErr error
+	runner.progress, progressErr = progress.NewStatsTicker(options.StatsInterval, options.EnableProgressBar, options.Metrics, options.MetricsPort)
+	if progressErr != nil {
+		return nil, progressErr
+	}
 
 	// create project file if requested or load existing one
 	if options.Project {
 		var projectFileErr error
-		runner.pf, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""})
+		runner.projectFile, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""})
 		if projectFileErr != nil {
 			return nil, projectFileErr
 		}
@@ -189,19 +200,11 @@ func New(options *Options) (*Runner, error) {
 		collaborator.DefaultCollaborator.Collab.AddBIID(options.BurpCollaboratorBiid)
 	}
 
-	// Create Dialer
-	var err error
-	runner.dialer, err = fastdialer.NewDialer(fastdialer.DefaultOptions)
-	if err != nil {
-		return nil, err
-	}
-
 	if options.RateLimit > 0 {
 		runner.ratelimiter = ratelimit.New(options.RateLimit)
 	} else {
 		runner.ratelimiter = ratelimit.NewUnlimited()
 	}
-
 	return runner, nil
 }
 
@@ -210,18 +213,30 @@ func (r *Runner) Close() {
 	if r.output != nil {
 		r.output.Close()
 	}
-	r.hm.Close()
-	if r.pf != nil {
-		r.pf.Close()
+	r.hostMap.Close()
+	if r.projectFile != nil {
+		r.projectFile.Close()
 	}
 }
 
 // RunEnumeration sets up the input layer for giving input nuclei.
 // binary and runs the actual enumeration
 func (r *Runner) RunEnumeration() {
-	// resolves input templates definitions and any optional exclusion
-	includedTemplates := r.getTemplatesFor(r.options.Templates)
-	excludedTemplates := r.getTemplatesFor(r.options.ExcludedTemplates)
+	defer r.Close()
+
+	// If we have no templates, run on whole template directory with provided tags
+	if len(r.options.Templates) == 0 && len(r.options.Workflows) == 0 && !r.options.NewTemplates && (len(r.options.Tags) > 0 || len(r.options.ExcludeTags) > 0) {
+		r.options.Templates = append(r.options.Templates, r.options.TemplatesDirectory)
+	}
+	if r.options.NewTemplates {
+		templatesLoaded, err := r.readNewTemplatesFile()
+		if err != nil {
+			gologger.Warning().Msgf("Could not get newly added templates: %s\n", err)
+		}
+		r.options.Templates = append(r.options.Templates, templatesLoaded...)
+	}
+	includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates)
+	excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates)
 	// defaults to all templates
 	allTemplates := includedTemplates
 
@@ -237,79 +252,138 @@ func (r *Runner) RunEnumeration() {
 			if _, found := excludedMap[incl]; !found {
 				allTemplates = append(allTemplates, incl)
 			} else {
-				gologger.Warningf("Excluding '%s'", incl)
+				gologger.Warning().Msgf("Excluding '%s'", incl)
 			}
 		}
 	}
 
 	// pre-parse all the templates, apply filters
-	availableTemplates, workflowCount := r.getParsedTemplatesFor(allTemplates, r.options.Severity)
-	templateCount := len(availableTemplates)
-	hasWorkflows := workflowCount > 0
+	finalTemplates := []*templates.Template{}
+
+	workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows)
+	availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false)
+	availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true)
+
+	var unclusteredRequests int64 = 0
+	for _, template := range availableTemplates {
+		// workflows will dynamically adjust the totals while running, as
+		// it can't be know in advance which requests will be called
+		if len(template.Workflows) > 0 {
+			continue
+		}
+		unclusteredRequests += int64(template.TotalRequests) * r.inputCount
+	}
+
+	originalTemplatesCount := len(availableTemplates)
+	clusterCount := 0
+	clusters := clusterer.Cluster(availableTemplates)
+	for _, cluster := range clusters {
+		if len(cluster) > 1 && !r.options.OfflineHTTP {
+			executerOpts := protocols.ExecuterOptions{
+				Output:       r.output,
+				Options:      r.options,
+				Progress:     r.progress,
+				Catalog:      r.catalog,
+				RateLimiter:  r.ratelimiter,
+				IssuesClient: r.issuesClient,
+				Browser:      r.browser,
+				ProjectFile:  r.projectFile,
+			}
+			clusterID := fmt.Sprintf("cluster-%s", xid.New().String())
+
+			finalTemplates = append(finalTemplates, &templates.Template{
+				ID:            clusterID,
+				RequestsHTTP:  cluster[0].RequestsHTTP,
+				Executer:      clusterer.NewExecuter(cluster, &executerOpts),
+				TotalRequests: len(cluster[0].RequestsHTTP),
+			})
+			clusterCount += len(cluster)
+		} else {
+			finalTemplates = append(finalTemplates, cluster...)
+		}
+	}
+	for _, workflows := range availableWorkflows {
+		finalTemplates = append(finalTemplates, workflows)
+	}
+
+	var totalRequests int64 = 0
+	for _, t := range finalTemplates {
+		if len(t.Workflows) > 0 {
+			continue
+		}
+		totalRequests += int64(t.TotalRequests) * r.inputCount
+	}
+	if totalRequests < unclusteredRequests {
+		gologger.Info().Msgf("Reduced %d requests to %d (%d templates clustered)", unclusteredRequests, totalRequests, clusterCount)
+	}
+	templateCount := originalTemplatesCount + len(availableWorkflows)
 
 	// 0 matches means no templates were found in directory
 	if templateCount == 0 {
-		gologger.Fatalf("Error, no templates were found.\n")
+		gologger.Fatal().Msgf("Error, no templates were found.\n")
 	}
 
-	gologger.Infof("Using %s rules (%s templates, %s workflows)",
-		r.colorizer.Colorizer.Bold(templateCount).String(),
-		r.colorizer.Colorizer.Bold(templateCount-workflowCount).String(),
-		r.colorizer.Colorizer.Bold(workflowCount).String())
+	gologger.Info().Msgf("Using %s rules (%s templates, %s workflows)",
+		r.colorizer.Bold(templateCount).String(),
+		r.colorizer.Bold(templateCount-workflowCount).String(),
+		r.colorizer.Bold(workflowCount).String())
 
-	// precompute total request count
-	var totalRequests int64 = 0
-
-	for _, t := range availableTemplates {
-		switch av := t.(type) {
-		case *templates.Template:
-			totalRequests += (av.GetHTTPRequestCount() + av.GetDNSRequestCount()) * r.inputCount
-		case *workflows.Workflow:
-			// workflows will dynamically adjust the totals while running, as
-			// it can't be know in advance which requests will be called
-		} // nolint:wsl // comment
-	}
-
-	results := atomicboolean.New()
+	results := &atomic.Bool{}
 	wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
 	// Starts polling or ignore
 	collaborator.DefaultCollaborator.Poll()
 
-	if r.inputCount == 0 {
-		gologger.Errorf("Could not find any valid input URLs.")
-	} else if totalRequests > 0 || hasWorkflows {
-		// tracks global progress and captures stdout/stderr until p.Wait finishes
-		p := r.progress
-		p.Init(r.inputCount, templateCount, totalRequests)
+	// tracks global progress and captures stdout/stderr until p.Wait finishes
+	r.progress.Init(r.inputCount, templateCount, totalRequests)
 
-		for _, t := range availableTemplates {
-			wgtemplates.Add()
-			go func(template interface{}) {
-				defer wgtemplates.Done()
-				switch tt := template.(type) {
-				case *templates.Template:
-					for _, request := range tt.RequestsDNS {
-						results.Or(r.processTemplateWithList(p, tt, request))
-					}
-					for _, request := range tt.BulkRequestsHTTP {
-						results.Or(r.processTemplateWithList(p, tt, request))
-					}
-				case *workflows.Workflow:
-					results.Or(r.processWorkflowWithList(p, template.(*workflows.Workflow)))
-				}
-			}(t)
-		}
+	for _, t := range finalTemplates {
+		wgtemplates.Add()
+		go func(template *templates.Template) {
+			defer wgtemplates.Done()
 
-		wgtemplates.Wait()
-		p.Stop()
+			if len(template.Workflows) > 0 {
+				results.CAS(false, r.processWorkflowWithList(template))
+			} else {
+				results.CAS(false, r.processTemplateWithList(template))
+			}
+		}(t)
 	}
+	wgtemplates.Wait()
+	r.progress.Stop()
 
-	if !results.Get() {
+	if r.issuesClient != nil {
+		r.issuesClient.Close()
+	}
+	if !results.Load() {
 		if r.output != nil {
 			r.output.Close()
 			os.Remove(r.options.Output)
 		}
+		gologger.Info().Msgf("No results found. Better luck next time!")
+	}
 
-		gologger.Infof("No results found. Happy hacking!")
+	if r.browser != nil {
+		r.browser.Close()
 	}
 }
+
+// readNewTemplatesFile reads newly added templates from directory if it exists
+func (r *Runner) readNewTemplatesFile() ([]string, error) {
+	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	file, err := os.Open(additionsFile)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	templatesList := []string{}
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text == "" {
+			continue
+		}
+		templatesList = append(templatesList, text)
+	}
+	return templatesList, nil
+}

v2/internal/runner/templates.go

@@ -1,209 +1,98 @@
 package runner
 
 import (
-	"errors"
 	"fmt"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/karrick/godirwalk"
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
-// getTemplatesFor parses the specified input template definitions and returns a list of unique, absolute template paths.
-func (r *Runner) getTemplatesFor(definitions []string) []string {
-	// keeps track of processed dirs and files
-	processed := make(map[string]bool)
-	allTemplates := []string{}
-
-	// parses user input, handle file/directory cases and produce a list of unique templates
-	for _, t := range definitions {
-		var absPath string
-
-		var err error
-
-		if strings.Contains(t, "*") {
-			dirs := strings.Split(t, "/")
-			priorDir := strings.Join(dirs[:len(dirs)-1], "/")
-			absPath, err = r.resolvePathIfRelative(priorDir)
-			absPath += "/" + dirs[len(dirs)-1]
-		} else {
-			// resolve and convert relative to absolute path
-			absPath, err = r.resolvePathIfRelative(t)
-		}
-
-		if err != nil {
-			gologger.Errorf("Could not find template file '%s': %s\n", t, err)
-			continue
-		}
-
-		// Template input includes a wildcard
-		if strings.Contains(absPath, "*") {
-			var matches []string
-			matches, err = filepath.Glob(absPath)
-
-			if err != nil {
-				gologger.Labelf("Wildcard found, but unable to glob '%s': %s\n", absPath, err)
-
-				continue
-			}
-
-			// couldn't find templates in directory
-			if len(matches) == 0 {
-				gologger.Labelf("Error, no templates were found with '%s'.\n", absPath)
-				continue
-			} else {
-				gologger.Labelf("Identified %d templates\n", len(matches))
-			}
-
-			for _, match := range matches {
-				if !r.checkIfInNucleiIgnore(match) {
-					processed[match] = true
-
-					allTemplates = append(allTemplates, match)
-				}
-			}
-		} else {
-			// determine file/directory
-			isFile, err := isFilePath(absPath)
-			if err != nil {
-				gologger.Errorf("Could not stat '%s': %s\n", absPath, err)
-				continue
-			}
-			// test for uniqueness
-			if !isNewPath(absPath, processed) {
-				continue
-			}
-			// mark this absolute path as processed
-			// - if it's a file, we'll never process it again
-			// - if it's a dir, we'll never walk it again
-			processed[absPath] = true
-
-			if isFile {
-				allTemplates = append(allTemplates, absPath)
-			} else {
-				matches := []string{}
-
-				// Recursively walk down the Templates directory and run all the template file checks
-				err := directoryWalker(
-					absPath,
-					func(path string, d *godirwalk.Dirent) error {
-						if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
-							if !r.checkIfInNucleiIgnore(path) && isNewPath(path, processed) {
-								matches = append(matches, path)
-								processed[path] = true
-							}
-						}
-						return nil
-					},
-				)
-
-				// directory couldn't be walked
-				if err != nil {
-					gologger.Labelf("Could not find templates in directory '%s': %s\n", absPath, err)
-					continue
-				}
-
-				// couldn't find templates in directory
-				if len(matches) == 0 {
-					gologger.Labelf("Error, no templates were found in '%s'.\n", absPath)
-					continue
-				}
-
-				allTemplates = append(allTemplates, matches...)
-			}
-		}
-	}
-
-	return allTemplates
-}
-
 // getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered
 // by severity, along with a flag indicating if workflows are present.
-func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities string) (parsedTemplates []interface{}, workflowCount int) {
-	workflowCount = 0
-	severities = strings.ToLower(severities)
-	allSeverities := strings.Split(severities, ",")
+func (r *Runner) getParsedTemplatesFor(templatePaths, severities []string, workflows bool) (parsedTemplates map[string]*templates.Template, workflowCount int) {
 	filterBySeverity := len(severities) > 0
 
-	gologger.Infof("Loading templates...")
-
-	for _, match := range templatePaths {
-		t, err := r.parseTemplateFile(match)
-		switch tp := t.(type) {
-		case *templates.Template:
-			// only include if severity matches or no severity filtering
-			sev := strings.ToLower(tp.Info["severity"])
-			if !filterBySeverity || hasMatchingSeverity(sev, allSeverities) {
-				parsedTemplates = append(parsedTemplates, tp)
-				gologger.Infof("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"]))
-			} else {
-				gologger.Warningf("Excluding template %s due to severity filter (%s not in [%s])", tp.ID, sev, severities)
-			}
-		case *workflows.Workflow:
-			parsedTemplates = append(parsedTemplates, tp)
-			gologger.Infof("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"]))
-			workflowCount++
-		default:
-			gologger.Errorf("Could not parse file '%s': %s\n", match, err)
-		}
+	if !workflows {
+		gologger.Info().Msgf("Loading templates...")
+	} else {
+		gologger.Info().Msgf("Loading workflows...")
 	}
 
+	parsedTemplates = make(map[string]*templates.Template)
+	for _, match := range templatePaths {
+		t, err := r.parseTemplateFile(match)
+		if err != nil {
+			gologger.Warning().Msgf("Could not parse file '%s': %s\n", match, err)
+			continue
+		}
+		if t == nil {
+			continue
+		}
+		if len(t.Workflows) == 0 && workflows {
+			continue // don't print if user only wants to run workflows
+		}
+		if len(t.Workflows) > 0 && !workflows {
+			continue // don't print workflow if user only wants to run templates
+		}
+		if len(t.Workflows) > 0 {
+			workflowCount++
+		}
+		sev := strings.ToLower(types.ToString(t.Info["severity"]))
+		if !filterBySeverity || hasMatchingSeverity(sev, severities) {
+			parsedTemplates[t.ID] = t
+			gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), sev))
+		} else {
+			gologger.Warning().Msgf("Excluding template %s due to severity filter (%s not in [%s])", t.ID, sev, severities)
+		}
+	}
 	return parsedTemplates, workflowCount
 }
 
-func (r *Runner) parseTemplateFile(file string) (interface{}, error) {
-	// check if it's a template
-	template, errTemplate := templates.Parse(file)
-	if errTemplate == nil {
-		return template, nil
+// parseTemplateFile returns the parsed template file
+func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
+	executerOpts := protocols.ExecuterOptions{
+		Output:       r.output,
+		Options:      r.options,
+		Progress:     r.progress,
+		Catalog:      r.catalog,
+		IssuesClient: r.issuesClient,
+		RateLimiter:  r.ratelimiter,
+		ProjectFile:  r.projectFile,
+		Browser:      r.browser,
 	}
-
-	// check if it's a workflow
-	workflow, errWorkflow := workflows.Parse(file)
-	if errWorkflow == nil {
-		return workflow, nil
+	template, err := templates.Parse(file, executerOpts)
+	if err != nil {
+		return nil, err
 	}
-
-	if errTemplate != nil {
-		return nil, errTemplate
+	if template == nil {
+		return nil, nil
 	}
-
-	if errWorkflow != nil {
-		return nil, errWorkflow
-	}
-
-	return nil, errors.New("unknown error occurred")
+	return template, nil
 }
 
 func (r *Runner) templateLogMsg(id, name, author, severity string) string {
 	// Display the message for the template
 	message := fmt.Sprintf("[%s] %s (%s)",
-		r.colorizer.Colorizer.BrightBlue(id).String(),
-		r.colorizer.Colorizer.Bold(name).String(),
-		r.colorizer.Colorizer.BrightYellow("@"+author).String())
-
+		r.colorizer.BrightBlue(id).String(),
+		r.colorizer.Bold(name).String(),
+		r.colorizer.BrightYellow("@"+author).String())
 	if severity != "" {
-		message += " [" + r.colorizer.GetColorizedSeverity(severity) + "]"
+		message += " [" + r.severityColors.Data[severity] + "]"
 	}
-
 	return message
 }
 
 func (r *Runner) logAvailableTemplate(tplPath string) {
 	t, err := r.parseTemplateFile(tplPath)
-	if t != nil {
-		switch tp := t.(type) {
-		case *templates.Template:
-			gologger.Silentf("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"]))
-		case *workflows.Workflow:
-			gologger.Silentf("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"]))
-		default:
-			gologger.Errorf("Could not parse file '%s': %s\n", tplPath, err)
-		}
+	if err != nil {
+		gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
+	} else {
+		gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
 	}
 }
 
@@ -214,11 +103,11 @@ func (r *Runner) listAvailableTemplates() {
 	}
 
 	if _, err := os.Stat(r.templatesConfig.TemplatesDirectory); os.IsNotExist(err) {
-		gologger.Errorf("%s does not exists", r.templatesConfig.TemplatesDirectory)
+		gologger.Error().Msgf("%s does not exists", r.templatesConfig.TemplatesDirectory)
 		return
 	}
 
-	gologger.Silentf(
+	gologger.Print().Msgf(
 		"\nListing available v.%s nuclei templates for %s",
 		r.templatesConfig.CurrentVersion,
 		r.templatesConfig.TemplatesDirectory,
@@ -227,42 +116,35 @@ func (r *Runner) listAvailableTemplates() {
 		r.templatesConfig.TemplatesDirectory,
 		func(path string, d *godirwalk.Dirent) error {
 			if d.IsDir() && path != r.templatesConfig.TemplatesDirectory {
-				gologger.Silentf("\n%s:\n\n", r.colorizer.Colorizer.Bold(r.colorizer.Colorizer.BgBrightBlue(d.Name())).String())
+				gologger.Print().Msgf("\n%s:\n\n", r.colorizer.Bold(r.colorizer.BgBrightBlue(d.Name())).String())
 			} else if strings.HasSuffix(path, ".yaml") {
 				r.logAvailableTemplate(path)
 			}
-
 			return nil
 		},
 	)
-
 	// directory couldn't be walked
 	if err != nil {
-		gologger.Labelf("Could not find templates in directory '%s': %s\n", r.templatesConfig.TemplatesDirectory, err)
+		gologger.Error().Msgf("Could not find templates in directory '%s': %s\n", r.templatesConfig.TemplatesDirectory, err)
 	}
 }
 
-func (r *Runner) resolvePathIfRelative(filePath string) (string, error) {
-	if isRelative(filePath) {
-		newPath, err := r.resolvePath(filePath)
-
-		if err != nil {
-			return "", err
-		}
-
-		return newPath, nil
-	}
-
-	return filePath, nil
-}
-
 func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool {
 	for _, s := range allowedSeverities {
-		if s != "" && strings.HasPrefix(templateSeverity, s) {
-			return true
+		finalSeverities := []string{}
+		if strings.Contains(s, ",") {
+			finalSeverities = strings.Split(s, ",")
+		} else {
+			finalSeverities = append(finalSeverities, s)
+		}
+
+		for _, sev := range finalSeverities {
+			sev = strings.ToLower(sev)
+			if sev != "" && strings.HasPrefix(templateSeverity, sev) {
+				return true
+			}
 		}
 	}
-
 	return false
 }
 
@@ -282,21 +164,3 @@ func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Di
 
 	return nil
 }
-
-func isFilePath(filePath string) (bool, error) {
-	info, err := os.Stat(filePath)
-	if err != nil {
-		return false, err
-	}
-
-	return info.Mode().IsRegular(), nil
-}
-
-func isNewPath(filePath string, pathMap map[string]bool) bool {
-	if _, already := pathMap[filePath]; already {
-		gologger.Warningf("Skipping already specified path '%s'", filePath)
-		return false
-	}
-
-	return true
-}

v2/internal/runner/update.go

@@ -2,9 +2,11 @@ package runner
 
 import (
 	"archive/zip"
+	"bufio"
 	"bytes"
 	"context"
-	"errors"
+	"crypto/md5"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -12,11 +14,14 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/blang/semver"
 	"github.com/google/go-github/v32/github"
+	"github.com/olekukonko/tablewriter"
+	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
 )
 
@@ -36,6 +41,8 @@ func (r *Runner) updateTemplates() error {
 	if err != nil {
 		return err
 	}
+	configDir := path.Join(home, "/.config", "/nuclei")
+	_ = os.MkdirAll(configDir, os.ModePerm)
 
 	templatesConfigFile := path.Join(home, nucleiConfigFilename)
 	if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
@@ -46,43 +53,83 @@ func (r *Runner) updateTemplates() error {
 		r.templatesConfig = config
 	}
 
-	ctx := context.Background()
+	ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
+	if r.templatesConfig == nil {
+		currentConfig := &nucleiConfig{
+			TemplatesDirectory: path.Join(home, "nuclei-templates"),
+			IgnoreURL:          ignoreURL,
+			NucleiVersion:      Version,
+		}
+		if writeErr := r.writeConfiguration(currentConfig); writeErr != nil {
+			return errors.Wrap(writeErr, "could not write template configuration")
+		}
+		r.templatesConfig = currentConfig
+	}
+	// Check if last checked for nuclei-ignore is more than 1 hours.
+	// and if true, run the check.
+	if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
+		if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" {
+			ignoreURL = r.templatesConfig.IgnoreURL
+		}
+		gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
 
-	if r.templatesConfig == nil || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
+		if reqErr == nil {
+			resp, httpGet := http.DefaultClient.Do(req)
+			if httpGet != nil {
+				if resp != nil && resp.Body != nil {
+					resp.Body.Close()
+				}
+				gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
+			} else {
+				data, _ := ioutil.ReadAll(resp.Body)
+				resp.Body.Close()
+
+				if len(data) > 0 {
+					_ = ioutil.WriteFile(path.Join(configDir, nucleiIgnoreFile), data, 0644)
+				}
+				if r.templatesConfig != nil {
+					r.templatesConfig.LastCheckedIgnore = time.Now()
+				}
+			}
+		}
+		cancel()
+	}
+
+	ctx := context.Background()
+	if r.templatesConfig.CurrentVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
 		if !r.options.UpdateTemplates {
-			gologger.Labelf("nuclei-templates are not installed, use update-templates flag.\n")
+			gologger.Warning().Msgf("nuclei-templates are not installed (or indexed), use update-templates flag.\n")
 			return nil
 		}
 
 		// Use custom location if user has given a template directory
-		if r.options.TemplatesDirectory != "" {
-			home = r.options.TemplatesDirectory
+		r.templatesConfig = &nucleiConfig{
+			TemplatesDirectory: path.Join(home, "nuclei-templates"),
+		}
+		if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") {
+			r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
 		}
-
-		r.templatesConfig = &nucleiConfig{TemplatesDirectory: path.Join(home, "nuclei-templates")}
 
 		// Download the repository and also write the revision to a HEAD file.
 		version, asset, getErr := r.getLatestReleaseFromGithub()
 		if getErr != nil {
 			return getErr
 		}
+		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
 
-		gologger.Verbosef("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory)
-
-		err = r.downloadReleaseAndUnzip(ctx, asset.GetZipballURL())
+		_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
 		if err != nil {
 			return err
 		}
-
 		r.templatesConfig.CurrentVersion = version.String()
 
 		err = r.writeConfiguration(r.templatesConfig)
 		if err != nil {
 			return err
 		}
-
-		gologger.Infof("Successfully downloaded nuclei-templates (v%s). Enjoy!\n", version.String())
-
+		gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). Enjoy!\n", version.String())
 		return nil
 	}
 
@@ -95,17 +142,14 @@ func (r *Runner) updateTemplates() error {
 	// Get the configuration currently on disk.
 	verText := r.templatesConfig.CurrentVersion
 	indices := reVersion.FindStringIndex(verText)
-
 	if indices == nil {
 		return fmt.Errorf("invalid release found with tag %s", err)
 	}
-
 	if indices[0] > 0 {
 		verText = verText[indices[0]:]
 	}
 
 	oldVersion, err := semver.Make(verText)
-
 	if err != nil {
 		return err
 	}
@@ -116,38 +160,32 @@ func (r *Runner) updateTemplates() error {
 	}
 
 	if version.EQ(oldVersion) {
-		gologger.Infof("Your nuclei-templates are up to date: v%s\n", oldVersion.String())
+		gologger.Info().Msgf("Your nuclei-templates are up to date: v%s\n", oldVersion.String())
 		return r.writeConfiguration(r.templatesConfig)
 	}
 
 	if version.GT(oldVersion) {
 		if !r.options.UpdateTemplates {
-			gologger.Labelf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
+			gologger.Warning().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
 			return r.writeConfiguration(r.templatesConfig)
 		}
 
 		if r.options.TemplatesDirectory != "" {
-			home = r.options.TemplatesDirectory
-			r.templatesConfig.TemplatesDirectory = path.Join(home, "nuclei-templates")
+			r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
 		}
-
 		r.templatesConfig.CurrentVersion = version.String()
 
-		gologger.Verbosef("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory)
-
-		err = r.downloadReleaseAndUnzip(ctx, asset.GetZipballURL())
+		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
+		_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
 		if err != nil {
 			return err
 		}
-
 		err = r.writeConfiguration(r.templatesConfig)
 		if err != nil {
 			return err
 		}
-
-		gologger.Infof("Successfully updated nuclei-templates (v%s). Enjoy!\n", version.String())
+		gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). Enjoy!\n", version.String())
 	}
-
 	return nil
 }
 
@@ -162,17 +200,13 @@ func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.Repositor
 
 	// Find the most recent version based on semantic versioning.
 	var latestRelease semver.Version
-
 	var latestPublish *github.RepositoryRelease
-
 	for _, release := range rels {
 		verText := release.GetTagName()
 		indices := reVersion.FindStringIndex(verText)
-
 		if indices == nil {
 			return semver.Version{}, nil, fmt.Errorf("invalid release found with tag %s", err)
 		}
-
 		if indices[0] > 0 {
 			verText = verText[indices[0]:]
 		}
@@ -187,85 +221,242 @@ func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.Repositor
 			latestPublish = release
 		}
 	}
-
 	if latestPublish == nil {
 		return semver.Version{}, nil, errors.New("no version found for the templates")
 	}
-
 	return latestRelease, latestPublish, nil
 }
 
 // downloadReleaseAndUnzip downloads and unzips the release in a directory
-func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string) error {
+func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadURL string) (*templateUpdateResults, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
-		return fmt.Errorf("failed to create HTTP request to %s: %s", downloadURL, err)
+		return nil, fmt.Errorf("failed to create HTTP request to %s: %s", downloadURL, err)
 	}
 
 	res, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return fmt.Errorf("failed to download a release file from %s: %s", downloadURL, err)
+		return nil, fmt.Errorf("failed to download a release file from %s: %s", downloadURL, err)
 	}
 	defer res.Body.Close()
-
 	if res.StatusCode != http.StatusOK {
-		return fmt.Errorf("failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode)
+		return nil, fmt.Errorf("failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode)
 	}
 
 	buf, err := ioutil.ReadAll(res.Body)
 	if err != nil {
-		return fmt.Errorf("failed to create buffer for zip file: %s", err)
+		return nil, fmt.Errorf("failed to create buffer for zip file: %s", err)
 	}
 
 	reader := bytes.NewReader(buf)
 	z, err := zip.NewReader(reader, reader.Size())
-
 	if err != nil {
-		return fmt.Errorf("failed to uncompress zip file: %s", err)
+		return nil, fmt.Errorf("failed to uncompress zip file: %s", err)
 	}
 
 	// Create the template folder if it doesn't exists
 	err = os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm)
 	if err != nil {
-		return fmt.Errorf("failed to create template base folder: %s", err)
+		return nil, fmt.Errorf("failed to create template base folder: %s", err)
 	}
 
+	results, err := r.compareAndWriteTemplates(z)
+	if err != nil {
+		return nil, fmt.Errorf("failed to write templates: %s", err)
+	}
+
+	r.printUpdateChangelog(results, version)
+	checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
+	err = writeTemplatesChecksum(checksumFile, results.checksums)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not write checksum")
+	}
+
+	// Write the additions to a cached file for new runs.
+	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	buffer := &bytes.Buffer{}
+	for _, addition := range results.additions {
+		buffer.WriteString(addition)
+		buffer.WriteString("\n")
+	}
+	err = ioutil.WriteFile(additionsFile, buffer.Bytes(), os.ModePerm)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not write new additions file")
+	}
+	return results, err
+}
+
+type templateUpdateResults struct {
+	additions     []string
+	deletions     []string
+	modifications []string
+	totalCount    int
+	checksums     map[string]string
+}
+
+// compareAndWriteTemplates compares and returns the stats of a template
+// update operations.
+func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults, error) {
+	results := &templateUpdateResults{
+		checksums: make(map[string]string),
+	}
+
+	// We use file-checksums that are md5 hashes to store the list of files->hashes
+	// that have been downloaded previously.
+	// If the path isn't found in new update after being read from the previous checksum,
+	// it is removed. This allows us fine-grained control over the download process
+	// as well as solves a long problem with nuclei-template updates.
+	checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
+	previousChecksum, _ := readPreviousTemplatesChecksum(checksumFile)
 	for _, file := range z.File {
 		directory, name := filepath.Split(file.Name)
 		if name == "" {
 			continue
 		}
-
 		paths := strings.Split(directory, "/")
 		finalPath := strings.Join(paths[1:], "/")
 
+		if (!strings.EqualFold(name, ".nuclei-ignore") && strings.HasPrefix(name, ".")) || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") {
+			continue
+		}
+		results.totalCount++
 		templateDirectory := path.Join(r.templatesConfig.TemplatesDirectory, finalPath)
-		err = os.MkdirAll(templateDirectory, os.ModePerm)
-
+		err := os.MkdirAll(templateDirectory, os.ModePerm)
 		if err != nil {
-			return fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err)
+			return nil, fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err)
 		}
 
-		f, err := os.OpenFile(path.Join(templateDirectory, name), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777)
+		templatePath := path.Join(templateDirectory, name)
+
+		isAddition := false
+		if _, statErr := os.Stat(templatePath); os.IsNotExist(statErr) {
+			isAddition = true
+		}
+		f, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777)
 		if err != nil {
 			f.Close()
-			return fmt.Errorf("could not create uncompressed file: %s", err)
+			return nil, fmt.Errorf("could not create uncompressed file: %s", err)
 		}
 
 		reader, err := file.Open()
 		if err != nil {
 			f.Close()
-			return fmt.Errorf("could not open archive to extract file: %s", err)
+			return nil, fmt.Errorf("could not open archive to extract file: %s", err)
 		}
+		hasher := md5.New()
 
-		_, err = io.Copy(f, reader)
+		// Save file and also read into hasher for md5
+		_, err = io.Copy(f, io.TeeReader(reader, hasher))
 		if err != nil {
 			f.Close()
-			return fmt.Errorf("could not write template file: %s", err)
+			return nil, fmt.Errorf("could not write template file: %s", err)
 		}
-
 		f.Close()
+
+		oldChecksum, checksumOK := previousChecksum[templatePath]
+
+		checksum := hex.EncodeToString(hasher.Sum(nil))
+		if isAddition {
+			results.additions = append(results.additions, path.Join(finalPath, name))
+		} else if checksumOK && oldChecksum[0] != checksum {
+			results.modifications = append(results.modifications, path.Join(finalPath, name))
+		}
+		results.checksums[templatePath] = checksum
 	}
 
+	// If we don't find a previous file in new download and it hasn't been
+	// changed on the disk, delete it.
+	for k, v := range previousChecksum {
+		_, ok := results.checksums[k]
+		if !ok && v[0] == v[1] {
+			os.Remove(k)
+			results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(k, r.templatesConfig.TemplatesDirectory), "/"))
+		}
+	}
+	return results, nil
+}
+
+// readPreviousTemplatesChecksum reads the previous checksum file from the disk.
+//
+// It reads two checksums, the first checksum is what we expect and the second is
+// the actual checksum of the file on disk currently.
+func readPreviousTemplatesChecksum(file string) (map[string][2]string, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+
+	checksum := make(map[string][2]string)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text == "" {
+			continue
+		}
+		parts := strings.Split(text, ",")
+		if len(parts) < 2 {
+			continue
+		}
+		values := [2]string{parts[1]}
+
+		f, err := os.Open(parts[0])
+		if err != nil {
+			return nil, err
+		}
+
+		hasher := md5.New()
+		if _, err := io.Copy(hasher, f); err != nil {
+			return nil, err
+		}
+		f.Close()
+
+		values[1] = hex.EncodeToString(hasher.Sum(nil))
+		checksum[parts[0]] = values
+	}
+	return checksum, nil
+}
+
+// writeTemplatesChecksum writes the nuclei-templates checksum data to disk.
+func writeTemplatesChecksum(file string, checksum map[string]string) error {
+	f, err := os.Create(file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	builder := &strings.Builder{}
+	for k, v := range checksum {
+		builder.WriteString(k)
+		builder.WriteString(",")
+		builder.WriteString(v)
+		builder.WriteString("\n")
+
+		if _, checksumErr := f.WriteString(builder.String()); checksumErr != nil {
+			return err
+		}
+		builder.Reset()
+	}
 	return nil
 }
+
+func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version string) {
+	if len(results.additions) > 0 {
+		gologger.Print().Msgf("\nNewly added templates: \n\n")
+
+		for _, addition := range results.additions {
+			gologger.Print().Msgf("%s", addition)
+		}
+	}
+
+	gologger.Print().Msgf("\nNuclei Templates v%s Changelog\n", version)
+	data := [][]string{
+		{strconv.Itoa(results.totalCount), strconv.Itoa(len(results.additions)), strconv.Itoa(len(results.deletions))},
+	}
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Total", "Added", "Removed"})
+	for _, v := range data {
+		table.Append(v)
+	}
+	table.Render()
+}

v2/internal/runner/update_test.go

@@ -0,0 +1,156 @@
+package runner
+
+import (
+	"archive/zip"
+	"context"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
+	gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{})
+
+	baseTemplates, err := ioutil.TempDir("", "old-temp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(baseTemplates)
+
+	err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create write base file")
+
+	err = zipFromDirectory("base.zip", baseTemplates)
+	require.Nil(t, err, "could not create zip from directory")
+	defer os.Remove("base.zip")
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "base.zip")
+	}))
+	defer ts.Close()
+
+	templatesDirectory, err := ioutil.TempDir("", "template-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(templatesDirectory)
+
+	r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}}
+
+	results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
+	require.Nil(t, err, "could not download release and unzip")
+	require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")
+
+	newTempDir, err := ioutil.TempDir("", "new-tmp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(newTempDir)
+
+	err = ioutil.WriteFile(path.Join(newTempDir, "base.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create base file")
+	err = ioutil.WriteFile(path.Join(newTempDir, "new.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create new file")
+
+	err = zipFromDirectory("new.zip", newTempDir)
+	require.Nil(t, err, "could not create new zip from directory")
+	defer os.Remove("new.zip")
+
+	ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "new.zip")
+	}))
+	defer ts2.Close()
+
+	results, err = r.downloadReleaseAndUnzip(context.Background(), "1.0.1", ts2.URL)
+	require.Nil(t, err, "could not download release and unzip")
+
+	require.Equal(t, "new.yaml", results.additions[0], "could not get correct new addition")
+}
+
+func TestDownloadReleaseAndUnzipDeletion(t *testing.T) {
+	gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{})
+
+	baseTemplates, err := ioutil.TempDir("", "old-temp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(baseTemplates)
+
+	err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create write base file")
+
+	err = zipFromDirectory("base.zip", baseTemplates)
+	require.Nil(t, err, "could not create zip from directory")
+	defer os.Remove("base.zip")
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "base.zip")
+	}))
+	defer ts.Close()
+
+	templatesDirectory, err := ioutil.TempDir("", "template-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(templatesDirectory)
+
+	r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}}
+
+	results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
+	require.Nil(t, err, "could not download release and unzip")
+	require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")
+
+	newTempDir, err := ioutil.TempDir("", "new-tmp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(newTempDir)
+
+	err = zipFromDirectory("new.zip", newTempDir)
+	require.Nil(t, err, "could not create new zip from directory")
+	defer os.Remove("new.zip")
+
+	ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "new.zip")
+	}))
+	defer ts2.Close()
+
+	results, err = r.downloadReleaseAndUnzip(context.Background(), "1.0.1", ts2.URL)
+	require.Nil(t, err, "could not download release and unzip")
+
+	require.Equal(t, "base.yaml", results.deletions[0], "could not get correct new deletions")
+}
+
+func zipFromDirectory(zipPath, directory string) error {
+	file, err := os.Create(zipPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	w := zip.NewWriter(file)
+	defer w.Close()
+
+	walker := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		file, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		f, err := w.Create(strings.TrimPrefix(path, directory))
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(f, file)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+	return filepath.Walk(directory, walker)
+}

v2/internal/testutils/integration.go

@@ -0,0 +1,96 @@
+package testutils
+
+import (
+	"net"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+// RunNucleiAndGetResults returns a list of results for a template
+func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
+	cmd := exec.Command("./nuclei", "-t", template, "-target", url)
+	if debug {
+		cmd = exec.Command("./nuclei", "-t", template, "-target", url, "-debug")
+		cmd.Stderr = os.Stderr
+	}
+	cmd.Args = append(cmd.Args, extra...)
+
+	data, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+	parts := []string{}
+	items := strings.Split(string(data), "\n")
+	for _, i := range items {
+		if i != "" {
+			parts = append(parts, i)
+		}
+	}
+	return parts, nil
+}
+
+// RunNucleiWorkflowAndGetResults returns a list of results for a workflow
+func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
+	cmd := exec.Command("./nuclei", "-w", template, "-target", url)
+	if debug {
+		cmd = exec.Command("./nuclei", "-w", template, "-target", url, "-debug")
+		cmd.Stderr = os.Stderr
+	}
+	cmd.Args = append(cmd.Args, extra...)
+
+	data, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+	parts := []string{}
+	items := strings.Split(string(data), "\n")
+	for _, i := range items {
+		if i != "" {
+			parts = append(parts, i)
+		}
+	}
+	return parts, nil
+}
+
+// TestCase is a single integration test case
+type TestCase interface {
+	// Execute executes a test case and returns any errors if occurred
+	Execute(filePath string) error
+}
+
+// TCPServer creates a new tcp server that returns a response
+type TCPServer struct {
+	URL      string
+	listener net.Listener
+}
+
+// NewTCPServer creates a new TCP server from a handler
+func NewTCPServer(handler func(conn net.Conn)) *TCPServer {
+	server := &TCPServer{}
+
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		panic(err)
+	}
+	server.URL = l.Addr().String()
+	server.listener = l
+
+	go func() {
+		for {
+			// Listen for an incoming connection.
+			conn, err := l.Accept()
+			if err != nil {
+				continue
+			}
+			// Handle connections in a new goroutine.
+			go handler(conn)
+		}
+	}()
+	return server
+}
+
+// Close closes the TCP server
+func (s *TCPServer) Close() {
+	s.listener.Close()
+}

v2/internal/testutils/testutils.go

@@ -0,0 +1,127 @@
+package testutils
+
+import (
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"go.uber.org/ratelimit"
+)
+
+// Init initializes the protocols and their configurations
+func Init(options *types.Options) {
+	_ = protocolinit.Init(options)
+}
+
+// DefaultOptions is the default options structure for nuclei during mocking.
+var DefaultOptions = &types.Options{
+	RandomAgent:          false,
+	Metrics:              false,
+	Debug:                false,
+	DebugRequests:        false,
+	DebugResponse:        false,
+	Silent:               false,
+	Version:              false,
+	Verbose:              false,
+	NoColor:              true,
+	UpdateTemplates:      false,
+	JSON:                 false,
+	JSONRequests:         false,
+	EnableProgressBar:    false,
+	TemplatesVersion:     false,
+	TemplateList:         false,
+	Stdin:                false,
+	StopAtFirstMatch:     false,
+	NoMeta:               false,
+	Project:              false,
+	MetricsPort:          0,
+	BulkSize:             25,
+	TemplateThreads:      10,
+	Timeout:              5,
+	Retries:              1,
+	RateLimit:            150,
+	BurpCollaboratorBiid: "",
+	ProjectPath:          "",
+	Severity:             []string{},
+	Target:               "",
+	Targets:              "",
+	Output:               "",
+	ProxyURL:             "",
+	ProxySocksURL:        "",
+	TemplatesDirectory:   "",
+	TraceLogFile:         "",
+	Templates:            []string{},
+	ExcludedTemplates:    []string{},
+	CustomHeaders:        []string{},
+}
+
+// MockOutputWriter is a mocked output writer.
+type MockOutputWriter struct {
+	aurora          aurora.Aurora
+	RequestCallback func(templateID, url, requestType string, err error)
+	WriteCallback   func(o *output.ResultEvent)
+}
+
+// NewMockOutputWriter creates a new mock output writer
+func NewMockOutputWriter() *MockOutputWriter {
+	return &MockOutputWriter{aurora: aurora.NewAurora(false)}
+}
+
+// Close closes the output writer interface
+func (m *MockOutputWriter) Close() {}
+
+// Colorizer returns the colorizer instance for writer
+func (m *MockOutputWriter) Colorizer() aurora.Aurora {
+	return m.aurora
+}
+
+// Write writes the event to file and/or screen.
+func (m *MockOutputWriter) Write(result *output.ResultEvent) error {
+	if m.WriteCallback != nil {
+		m.WriteCallback(result)
+	}
+	return nil
+}
+
+// Request writes a log the requests trace log
+func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) {
+	if m.RequestCallback != nil {
+		m.RequestCallback(templateID, url, requestType, err)
+	}
+}
+
+// TemplateInfo contains info for a mock executed template.
+type TemplateInfo struct {
+	ID   string
+	Info map[string]interface{}
+	Path string
+}
+
+// NewMockExecuterOptions creates a new mock executeroptions struct
+func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecuterOptions {
+	progressImpl, _ := progress.NewStatsTicker(0, false, false, 0)
+	executerOpts := &protocols.ExecuterOptions{
+		TemplateID:   info.ID,
+		TemplateInfo: info.Info,
+		TemplatePath: info.Path,
+		Output:       NewMockOutputWriter(),
+		Options:      options,
+		Progress:     progressImpl,
+		ProjectFile:  nil,
+		IssuesClient: nil,
+		Browser:      nil,
+		Catalog:      catalog.New(options.TemplatesDirectory),
+		RateLimiter:  ratelimit.New(options.RateLimit),
+	}
+	return executerOpts
+}
+
+// NoopWriter is a NooP gologger writer.
+type NoopWriter struct{}
+
+// Write writes the data to an output writer.
+func (n *NoopWriter) Write(data []byte, level levels.Level) {}

v2/internal/tracelog/tracelog.go

@@ -1,76 +0,0 @@
-package tracelog
-
-import (
-	"os"
-	"sync"
-
-	jsoniter "github.com/json-iterator/go"
-)
-
-// Log is an interface for logging trace log of all the requests
-type Log interface {
-	// Close closes the log interface flushing data
-	Close()
-	// Request writes a log the requests trace log
-	Request(templateID, url, requestType string, err error)
-}
-
-// NoopLogger is a noop logger that simply does nothing
-type NoopLogger struct{}
-
-// Close closes the log interface flushing data
-func (n *NoopLogger) Close() {}
-
-// Request writes a log the requests trace log
-func (n *NoopLogger) Request(templateID, url, requestType string, err error) {}
-
-// FileLogger is a trace logger that writes request logs to a file.
-type FileLogger struct {
-	encoder *jsoniter.Encoder
-	file    *os.File
-	mutex   *sync.Mutex
-}
-
-// NewFileLogger creates a new file logger structure
-func NewFileLogger(path string) (*FileLogger, error) {
-	file, err := os.Create(path)
-	if err != nil {
-		return nil, err
-	}
-	return &FileLogger{file: file, encoder: jsoniter.NewEncoder(file), mutex: &sync.Mutex{}}, nil
-}
-
-// Close closes the log interface flushing data
-func (f *FileLogger) Close() {
-	f.mutex.Lock()
-	defer f.mutex.Unlock()
-
-	f.file.Close()
-}
-
-// JSONRequest is a trace log request written to file
-type JSONRequest struct {
-	ID    string `json:"id"`
-	URL   string `json:"url"`
-	Error string `json:"error"`
-	Type  string `json:"type"`
-}
-
-// Request writes a log the requests trace log
-func (f *FileLogger) Request(templateID, url, requestType string, err error) {
-	request := &JSONRequest{
-		ID:   templateID,
-		URL:  url,
-		Type: requestType,
-	}
-	if err != nil {
-		request.Error = err.Error()
-	} else {
-		request.Error = "none"
-	}
-
-	f.mutex.Lock()
-	defer f.mutex.Unlock()
-	//nolint:errcheck // We don't need to do anything here
-	f.encoder.Encode(request)
-}

v2/pkg/atomicboolean/bool.go

@@ -1,42 +0,0 @@
-package atomicboolean
-
-import (
-	"sync"
-)
-
-type AtomBool struct {
-	sync.RWMutex
-	flag bool
-}
-
-func New() *AtomBool {
-	return &AtomBool{}
-}
-
-func (b *AtomBool) Or(value bool) {
-	b.Lock()
-	defer b.Unlock()
-
-	b.flag = b.flag || value
-}
-
-func (b *AtomBool) And(value bool) {
-	b.Lock()
-	defer b.Unlock()
-
-	b.flag = b.flag && value
-}
-
-func (b *AtomBool) Set(value bool) {
-	b.Lock()
-	defer b.Unlock()
-
-	b.flag = value
-}
-
-func (b *AtomBool) Get() bool {
-	b.RLock()
-	defer b.RUnlock() //nolint
-
-	return b.flag
-}

v2/pkg/catalog/catalogue.go

@@ -0,0 +1,14 @@
+package catalog
+
+// Catalog is a template catalog helper implementation
+type Catalog struct {
+	ignoreFiles        []string
+	templatesDirectory string
+}
+
+// New creates a new Catalog structure using provided input items
+func New(directory string) *Catalog {
+	catalog := &Catalog{templatesDirectory: directory}
+	catalog.readNucleiIgnoreFile()
+	return catalog
+}

v2/pkg/catalog/find.go

@@ -0,0 +1,155 @@
+package catalog
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/karrick/godirwalk"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+)
+
+// GetTemplatesPath returns a list of absolute paths for the provided template list.
+func (c *Catalog) GetTemplatesPath(definitions []string) []string {
+	// keeps track of processed dirs and files
+	processed := make(map[string]bool)
+	allTemplates := []string{}
+
+	for _, t := range definitions {
+		paths, err := c.GetTemplatePath(t)
+		if err != nil {
+			gologger.Error().Msgf("Could not find template '%s': %s\n", t, err)
+		}
+		for _, path := range paths {
+			if _, ok := processed[path]; !ok {
+				processed[path] = true
+				allTemplates = append(allTemplates, path)
+			}
+		}
+	}
+	if len(allTemplates) > 0 {
+		gologger.Verbose().Msgf("Identified %d templates", len(allTemplates))
+	}
+	return allTemplates
+}
+
+// GetTemplatePath parses the specified input template path and returns a compiled
+// list of finished absolute paths to the templates evaluating any glob patterns
+// or folders provided as in.
+func (c *Catalog) GetTemplatePath(target string) ([]string, error) {
+	processed := make(map[string]struct{})
+
+	absPath, err := c.convertPathToAbsolute(target)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not find template file")
+	}
+
+	// Template input includes a wildcard
+	if strings.Contains(absPath, "*") {
+		matches, findErr := c.findGlobPathMatches(absPath, processed)
+		if findErr != nil {
+			return nil, errors.Wrap(findErr, "could not find glob matches")
+		}
+		if len(matches) == 0 {
+			return nil, errors.Errorf("no templates found for path")
+		}
+		return matches, nil
+	}
+
+	// Template input is either a file or a directory
+	match, file, err := c.findFileMatches(absPath, processed)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not find file")
+	}
+	if file {
+		if match != "" {
+			return []string{match}, nil
+		}
+		return nil, nil
+	}
+
+	// Recursively walk down the Templates directory and run all
+	// the template file checks
+	matches, err := c.findDirectoryMatches(absPath, processed)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not find directory matches")
+	}
+	if len(matches) == 0 {
+		return nil, errors.Errorf("no templates found in path")
+	}
+	return matches, nil
+}
+
+// convertPathToAbsolute resolves the paths provided to absolute paths
+// before doing any operations on them regardless of them being blob, folders, files, etc.
+func (c *Catalog) convertPathToAbsolute(t string) (string, error) {
+	if strings.Contains(t, "*") {
+		file := path.Base(t)
+		absPath, err := c.ResolvePath(path.Dir(t), "")
+		if err != nil {
+			return "", err
+		}
+		return path.Join(absPath, file), nil
+	}
+	return c.ResolvePath(t, "")
+}
+
+// findGlobPathMatches returns the matched files from a glob path
+func (c *Catalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) {
+	matches, err := filepath.Glob(absPath)
+	if err != nil {
+		return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
+	}
+	results := make([]string, 0, len(matches))
+	for _, match := range matches {
+		if _, ok := processed[match]; !ok {
+			processed[match] = struct{}{}
+			results = append(results, match)
+		}
+	}
+	return results, nil
+}
+
+// findFileMatches finds if a path is an absolute file. If the path
+// is a file, it returns true otherwise false with no errors.
+func (c *Catalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) {
+	info, err := os.Stat(absPath)
+	if err != nil {
+		return "", false, err
+	}
+	if !info.Mode().IsRegular() {
+		return "", false, nil
+	}
+	if _, ok := processed[absPath]; !ok {
+		processed[absPath] = struct{}{}
+		return absPath, true, nil
+	}
+	return "", true, nil
+}
+
+// findDirectoryMatches finds matches for templates from a directory
+func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) {
+	var results []string
+	err := godirwalk.Walk(absPath, &godirwalk.Options{
+		Unsorted: true,
+		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
+			return godirwalk.SkipNode
+		},
+		Callback: func(path string, d *godirwalk.Dirent) error {
+			if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
+				if c.checkIfInNucleiIgnore(path) {
+					return nil
+				}
+
+				if _, ok := processed[path]; !ok {
+					results = append(results, path)
+					processed[path] = struct{}{}
+				}
+			}
+			return nil
+		},
+	})
+	return results, err
+}

v2/pkg/catalog/ignore.go

@@ -0,0 +1,84 @@
+package catalog
+
+import (
+	"bufio"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/projectdiscovery/gologger"
+)
+
+const nucleiIgnoreFile = ".nuclei-ignore"
+
+// readNucleiIgnoreFile reads the nuclei ignore file marking it in map
+func (c *Catalog) readNucleiIgnoreFile() {
+	file, err := os.Open(path.Join(c.templatesDirectory, nucleiIgnoreFile))
+	if err != nil {
+		return
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text == "" {
+			continue
+		}
+		if strings.HasPrefix(text, "#") {
+			continue
+		}
+		c.ignoreFiles = append(c.ignoreFiles, text)
+	}
+}
+
+// checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules.
+func (c *Catalog) checkIfInNucleiIgnore(item string) bool {
+	if c.templatesDirectory == "" {
+		return false
+	}
+
+	matched := false
+	for _, paths := range c.ignoreFiles {
+		if !strings.HasSuffix(paths, ".yaml") {
+			if strings.HasSuffix(strings.TrimSuffix(item, "/"), strings.TrimSuffix(paths, "/")) {
+				matched = true
+				break
+			}
+		} else if strings.HasSuffix(item, paths) {
+			matched = true
+			break
+		}
+	}
+	if matched {
+		gologger.Error().Msgf("Excluding %s due to nuclei-ignore filter", item)
+		return true
+	}
+	return false
+}
+
+// ignoreFilesWithExcludes ignores results with exclude paths
+func (c *Catalog) ignoreFilesWithExcludes(results, excluded []string) []string {
+	var templates []string
+
+	for _, result := range results {
+		matched := false
+		for _, paths := range excluded {
+			if !strings.HasSuffix(paths, ".yaml") {
+				if strings.HasSuffix(strings.TrimSuffix(result, "/"), strings.TrimSuffix(paths, "/")) {
+					matched = true
+					break
+				}
+			} else if strings.HasSuffix(result, paths) {
+				matched = true
+				break
+			}
+		}
+		if !matched {
+			templates = append(templates, result)
+		} else {
+			gologger.Error().Msgf("Excluding %s due to excludes filter", result)
+		}
+	}
+	return templates
+}

v2/pkg/catalog/ignore_test.go

@@ -0,0 +1,40 @@
+package catalog
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestIgnoreFilesIgnore(t *testing.T) {
+	c := &Catalog{
+		ignoreFiles:        []string{"workflows/", "cves/2020/cve-2020-5432.yaml"},
+		templatesDirectory: "test",
+	}
+	tests := []struct {
+		path   string
+		ignore bool
+	}{
+		{"workflows/", true},
+		{"misc", false},
+		{"cves/", false},
+		{"cves/2020/cve-2020-5432.yaml", true},
+		{"/Users/test/nuclei-templates/workflows/", true},
+		{"/Users/test/nuclei-templates/misc", false},
+		{"/Users/test/nuclei-templates/cves/", false},
+		{"/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", true},
+	}
+	for _, test := range tests {
+		require.Equal(t, test.ignore, c.checkIfInNucleiIgnore(test.path), fmt.Sprintf("could not ignore file correctly: %v", test))
+	}
+}
+
+func TestExcludeFilesIgnore(t *testing.T) {
+	c := &Catalog{}
+	excludes := []string{"workflows/", "cves/2020/cve-2020-5432.yaml"}
+	paths := []string{"/Users/test/nuclei-templates/workflows/", "/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", "/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"}
+
+	data := c.ignoreFilesWithExcludes(paths, excludes)
+	require.Equal(t, []string{"/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"}, data, "could not exclude correct files")
+}

v2/pkg/catalog/path.go

@@ -0,0 +1,45 @@
+package catalog
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+// ResolvePath resolves the path to an absolute one in various ways.
+//
+// It checks if the filename is an absolute path, looks in the current directory
+// or checking the nuclei templates directory. If a second path is given,
+// it also tries to find paths relative to that second path.
+func (c *Catalog) ResolvePath(templateName, second string) (string, error) {
+	if strings.HasPrefix(templateName, "/") || strings.Contains(templateName, ":\\") {
+		return templateName, nil
+	}
+
+	if second != "" {
+		secondBasePath := path.Join(filepath.Dir(second), templateName)
+		if _, err := os.Stat(secondBasePath); !os.IsNotExist(err) {
+			return secondBasePath, nil
+		}
+	}
+
+	curDirectory, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+
+	templatePath := path.Join(curDirectory, templateName)
+	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
+		return templatePath, nil
+	}
+
+	if c.templatesDirectory != "" {
+		templatePath := path.Join(c.templatesDirectory, templateName)
+		if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
+			return templatePath, nil
+		}
+	}
+	return "", fmt.Errorf("no such path found: %s", templateName)
+}

v2/pkg/collaborator/collaborator.go

@@ -1,59 +0,0 @@
-package collaborator
-
-import (
-	"strings"
-	"time"
-
-	"github.com/projectdiscovery/collaborator"
-)
-
-const (
-	PollSeconds           = 5
-	DefaultMaxBufferLimit = 150
-)
-
-var DefaultPollInterval time.Duration = time.Second * time.Duration(PollSeconds)
-
-var DefaultCollaborator BurpCollaborator = BurpCollaborator{Collab: collaborator.NewBurpCollaborator()}
-
-type BurpCollaborator struct {
-	options *Options // unused
-	Collab  *collaborator.BurpCollaborator
-}
-
-type Options struct {
-	BIID           string
-	PollInterval   time.Duration
-	MaxBufferLimit int
-}
-
-func New(options *Options) *BurpCollaborator {
-	collab := collaborator.NewBurpCollaborator()
-	collab.AddBIID(options.BIID)
-	collab.MaxBufferLimit = options.MaxBufferLimit
-	return &BurpCollaborator{Collab: collab, options: options}
-}
-
-func (b *BurpCollaborator) Poll() {
-	// if no valid biids were provided just return
-	if len(b.Collab.BIIDs) > 0 {
-		go b.Collab.PollEach(DefaultPollInterval)
-	}
-}
-
-func (b *BurpCollaborator) Has(s string) bool {
-	for _, r := range b.Collab.RespBuffer {
-		for i := 0; i < len(r.Responses); i++ {
-			// search in dns
-			if strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) {
-				return true
-			}
-			// search in http
-			if strings.Contains(r.Responses[i].Data.RequestDecoded, s) {
-				return true
-			}
-		}
-	}
-
-	return false
-}

v2/pkg/colorizer/colorizer.go

@@ -1,42 +0,0 @@
-package colorizer
-
-import (
-	"strings"
-
-	"github.com/logrusorgru/aurora"
-)
-
-const (
-	fgOrange  uint8  = 208
-	undefined string = "undefined"
-)
-
-// NucleiColorizer contains the severity color mapping
-type NucleiColorizer struct {
-	Colorizer   aurora.Aurora
-	SeverityMap map[string]string
-}
-
-// NewNucleiColorizer initializes the new nuclei colorizer
-func NewNucleiColorizer(colorizer aurora.Aurora) *NucleiColorizer {
-	return &NucleiColorizer{
-		Colorizer: colorizer,
-		SeverityMap: map[string]string{
-			"info":     colorizer.Blue("info").String(),
-			"low":      colorizer.Green("low").String(),
-			"medium":   colorizer.Yellow("medium").String(),
-			"high":     colorizer.Index(fgOrange, "high").String(),
-			"critical": colorizer.Red("critical").String(),
-		},
-	}
-}
-
-// GetColorizedSeverity returns the colorized severity string
-func (r *NucleiColorizer) GetColorizedSeverity(severity string) string {
-	sev := r.SeverityMap[strings.ToLower(severity)]
-	if sev == "" {
-		return undefined
-	}
-
-	return sev
-}

v2/pkg/executer/executer_dns.go

@@ -1,177 +0,0 @@
-package executer
-
-import (
-	"fmt"
-	"os"
-	"regexp"
-
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
-	"github.com/projectdiscovery/nuclei/v2/internal/progress"
-	"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
-	"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	retryabledns "github.com/projectdiscovery/retryabledns"
-	"go.uber.org/ratelimit"
-)
-
-// DNSExecuter is a client for performing a DNS request
-// for a template.
-type DNSExecuter struct {
-	// hm            *hybrid.HybridMap // Unused
-	coloredOutput bool
-	debug         bool
-	jsonOutput    bool
-	jsonRequest   bool
-	noMeta        bool
-	Results       bool
-	traceLog      tracelog.Log
-	dnsClient     *retryabledns.Client
-	template      *templates.Template
-	dnsRequest    *requests.DNSRequest
-	writer        *bufwriter.Writer
-	ratelimiter   ratelimit.Limiter
-
-	colorizer   colorizer.NucleiColorizer
-	decolorizer *regexp.Regexp
-}
-
-// DefaultResolvers contains the list of resolvers known to be trusted.
-var DefaultResolvers = []string{
-	"1.1.1.1:53", // Cloudflare
-	"1.0.0.1:53", // Cloudflare
-	"8.8.8.8:53", // Google
-	"8.8.4.4:53", // Google
-}
-
-// DNSOptions contains configuration options for the DNS executer.
-type DNSOptions struct {
-	ColoredOutput bool
-	Debug         bool
-	JSON          bool
-	JSONRequests  bool
-	NoMeta        bool
-	TraceLog      tracelog.Log
-	Template      *templates.Template
-	DNSRequest    *requests.DNSRequest
-	Writer        *bufwriter.Writer
-
-	Colorizer   colorizer.NucleiColorizer
-	Decolorizer *regexp.Regexp
-	RateLimiter ratelimit.Limiter
-}
-
-// NewDNSExecuter creates a new DNS executer from a template
-// and a DNS request query.
-func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
-	dnsClient := retryabledns.New(DefaultResolvers, options.DNSRequest.Retries)
-
-	executer := &DNSExecuter{
-		debug:         options.Debug,
-		noMeta:        options.NoMeta,
-		jsonOutput:    options.JSON,
-		traceLog:      options.TraceLog,
-		jsonRequest:   options.JSONRequests,
-		dnsClient:     dnsClient,
-		template:      options.Template,
-		dnsRequest:    options.DNSRequest,
-		writer:        options.Writer,
-		coloredOutput: options.ColoredOutput,
-		colorizer:     options.Colorizer,
-		decolorizer:   options.Decolorizer,
-		ratelimiter:   options.RateLimiter,
-	}
-
-	return executer
-}
-
-// ExecuteDNS executes the DNS request on a URL
-func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, reqURL string) *Result {
-	result := &Result{}
-
-	// Parse the URL and return domain if URL.
-	var domain string
-	if isURL(reqURL) {
-		domain = extractDomain(reqURL)
-	} else {
-		domain = reqURL
-	}
-
-	// Compile each request for the template based on the URL
-	compiledRequest, err := e.dnsRequest.MakeDNSRequest(domain)
-	if err != nil {
-		e.traceLog.Request(e.template.ID, domain, "dns", err)
-		result.Error = errors.Wrap(err, "could not make dns request")
-		p.Drop(1)
-		return result
-	}
-	e.traceLog.Request(e.template.ID, domain, "dns", nil)
-
-	if e.debug {
-		gologger.Infof("Dumped DNS request for %s (%s)\n\n", reqURL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s\n", compiledRequest.String())
-	}
-
-	// Send the request to the target servers
-	resp, err := e.dnsClient.Do(compiledRequest)
-	if err != nil {
-		result.Error = errors.Wrap(err, "could not send dns request")
-		p.Drop(1)
-		return result
-	}
-	p.Update()
-
-	gologger.Verbosef("Sent for [%s] to %s\n", "dns-request", e.template.ID, reqURL)
-
-	if e.debug {
-		gologger.Infof("Dumped DNS response for %s (%s)\n\n", reqURL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s\n", resp.String())
-	}
-
-	matcherCondition := e.dnsRequest.GetMatchersCondition()
-
-	for _, matcher := range e.dnsRequest.Matchers {
-		// Check if the matcher matched
-		if !matcher.MatchDNS(resp) {
-			// If the condition is AND we haven't matched, return.
-			if matcherCondition == matchers.ANDCondition {
-				return result
-			}
-		} else {
-			// If the matcher has matched, and its an OR
-			// write the first output then move to next matcher.
-			if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 {
-				e.writeOutputDNS(domain, compiledRequest, resp, matcher, nil)
-				result.GotResults = true
-			}
-		}
-	}
-
-	// All matchers have successfully completed so now start with the
-	// next task which is extraction of input from matchers.
-	var extractorResults []string
-
-	for _, extractor := range e.dnsRequest.Extractors {
-		for match := range extractor.ExtractDNS(resp) {
-			if !extractor.Internal {
-				extractorResults = append(extractorResults, match)
-			}
-		}
-	}
-
-	// Write a final string of output if matcher type is
-	// AND or if we have extractors for the mechanism too.
-	if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
-		e.writeOutputDNS(domain, compiledRequest, resp, nil, extractorResults)
-
-		result.GotResults = true
-	}
-
-	return result
-}
-
-// Close closes the dns executer for a template.
-func (e *DNSExecuter) Close() {}

v2/pkg/executer/executer_http.go

@@ -1,710 +0,0 @@
-package executer
-
-import (
-	"bytes"
-	"context"
-	"crypto/tls"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net"
-	"net/http"
-	"net/http/cookiejar"
-	"net/http/httputil"
-	"net/url"
-	"os"
-	"regexp"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/fastdialer/fastdialer"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
-	"github.com/projectdiscovery/nuclei/v2/internal/progress"
-	"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
-	"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
-	"github.com/projectdiscovery/nuclei/v2/pkg/generators"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	projetctfile "github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/rawhttp"
-	"github.com/projectdiscovery/retryablehttp-go"
-	"github.com/remeh/sizedwaitgroup"
-	"go.uber.org/ratelimit"
-	"golang.org/x/net/proxy"
-)
-
-const (
-	two                   = 2
-	ten                   = 10
-	defaultMaxWorkers     = 150
-	defaultMaxHistorydata = 150
-)
-
-// HTTPExecuter is client for performing HTTP requests
-// for a template.
-type HTTPExecuter struct {
-	pf               *projetctfile.ProjectFile
-	customHeaders    requests.CustomHeaders
-	colorizer        colorizer.NucleiColorizer
-	httpClient       *retryablehttp.Client
-	rawHTTPClient    *rawhttp.Client
-	template         *templates.Template
-	bulkHTTPRequest  *requests.BulkHTTPRequest
-	writer           *bufwriter.Writer
-	CookieJar        *cookiejar.Jar
-	traceLog         tracelog.Log
-	decolorizer      *regexp.Regexp
-	coloredOutput    bool
-	debug            bool
-	Results          bool
-	jsonOutput       bool
-	jsonRequest      bool
-	noMeta           bool
-	stopAtFirstMatch bool
-	ratelimiter      ratelimit.Limiter
-}
-
-// HTTPOptions contains configuration options for the HTTP executer.
-type HTTPOptions struct {
-	CustomHeaders    requests.CustomHeaders
-	ProxyURL         string
-	ProxySocksURL    string
-	Template         *templates.Template
-	BulkHTTPRequest  *requests.BulkHTTPRequest
-	Writer           *bufwriter.Writer
-	Timeout          int
-	Retries          int
-	CookieJar        *cookiejar.Jar
-	Colorizer        *colorizer.NucleiColorizer
-	Decolorizer      *regexp.Regexp
-	TraceLog         tracelog.Log
-	Debug            bool
-	JSON             bool
-	JSONRequests     bool
-	NoMeta           bool
-	CookieReuse      bool
-	ColoredOutput    bool
-	StopAtFirstMatch bool
-	PF               *projetctfile.ProjectFile
-	RateLimiter      ratelimit.Limiter
-	Dialer           *fastdialer.Dialer
-}
-
-// NewHTTPExecuter creates a new HTTP executer from a template
-// and a HTTP request query.
-func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
-	var (
-		proxyURL *url.URL
-		err      error
-	)
-
-	if options.ProxyURL != "" {
-		proxyURL, err = url.Parse(options.ProxyURL)
-	}
-
-	if err != nil {
-		return nil, err
-	}
-
-	// Create the HTTP Client
-	client := makeHTTPClient(proxyURL, options)
-	// nolint:bodyclose // false positive there is no body to close yet
-	client.CheckRetry = retryablehttp.HostSprayRetryPolicy()
-
-	if options.CookieJar != nil {
-		client.HTTPClient.Jar = options.CookieJar
-	} else if options.CookieReuse {
-		jar, err := cookiejar.New(nil)
-		if err != nil {
-			return nil, err
-		}
-		client.HTTPClient.Jar = jar
-	}
-
-	// initiate raw http client
-	rawClient := rawhttp.NewClient(rawhttp.DefaultOptions)
-
-	executer := &HTTPExecuter{
-		debug:            options.Debug,
-		jsonOutput:       options.JSON,
-		jsonRequest:      options.JSONRequests,
-		noMeta:           options.NoMeta,
-		httpClient:       client,
-		rawHTTPClient:    rawClient,
-		traceLog:         options.TraceLog,
-		template:         options.Template,
-		bulkHTTPRequest:  options.BulkHTTPRequest,
-		writer:           options.Writer,
-		customHeaders:    options.CustomHeaders,
-		CookieJar:        options.CookieJar,
-		coloredOutput:    options.ColoredOutput,
-		colorizer:        *options.Colorizer,
-		decolorizer:      options.Decolorizer,
-		stopAtFirstMatch: options.StopAtFirstMatch,
-		pf:               options.PF,
-		ratelimiter:      options.RateLimiter,
-	}
-
-	return executer, nil
-}
-
-func (e *HTTPExecuter) ExecuteRaceRequest(reqURL string) *Result {
-	result := &Result{
-		Matches:     make(map[string]interface{}),
-		Extractions: make(map[string]interface{}),
-	}
-
-	dynamicvalues := make(map[string]interface{})
-
-	// verify if the URL is already being processed
-	if e.bulkHTTPRequest.HasGenerator(reqURL) {
-		return result
-	}
-
-	e.bulkHTTPRequest.CreateGenerator(reqURL)
-
-	// Workers that keeps enqueuing new requests
-	maxWorkers := e.bulkHTTPRequest.RaceNumberRequests
-	swg := sizedwaitgroup.New(maxWorkers)
-	for i := 0; i < e.bulkHTTPRequest.RaceNumberRequests; i++ {
-		swg.Add()
-		// base request
-		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
-			result.Error = err
-			return result
-		}
-		go func(httpRequest *requests.HTTPRequest) {
-			defer swg.Done()
-
-			// If the request was built correctly then execute it
-			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
-			if err != nil {
-				result.Error = errors.Wrap(err, "could not handle http request")
-			}
-		}(request)
-	}
-
-	swg.Wait()
-
-	return result
-}
-
-func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string) *Result {
-	result := &Result{
-		Matches:     make(map[string]interface{}),
-		Extractions: make(map[string]interface{}),
-	}
-
-	dynamicvalues := make(map[string]interface{})
-
-	// verify if the URL is already being processed
-	if e.bulkHTTPRequest.HasGenerator(reqURL) {
-		return result
-	}
-
-	remaining := e.bulkHTTPRequest.GetRequestCount()
-	e.bulkHTTPRequest.CreateGenerator(reqURL)
-
-	// Workers that keeps enqueuing new requests
-	maxWorkers := e.bulkHTTPRequest.Threads
-	swg := sizedwaitgroup.New(maxWorkers)
-	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
-		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
-			result.Error = err
-			p.Drop(remaining)
-		} else {
-			swg.Add()
-			go func(httpRequest *requests.HTTPRequest) {
-				defer swg.Done()
-
-				e.ratelimiter.Take()
-
-				// If the request was built correctly then execute it
-				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
-				if err != nil {
-					e.traceLog.Request(e.template.ID, reqURL, "http", err)
-					result.Error = errors.Wrap(err, "could not handle http request")
-					p.Drop(remaining)
-				} else {
-					e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-				}
-			}(request)
-		}
-		p.Update()
-		e.bulkHTTPRequest.Increment(reqURL)
-	}
-	swg.Wait()
-
-	return result
-}
-
-func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result {
-	result := &Result{
-		Matches:     make(map[string]interface{}),
-		Extractions: make(map[string]interface{}),
-	}
-
-	dynamicvalues := make(map[string]interface{})
-
-	// verify if the URL is already being processed
-	if e.bulkHTTPRequest.HasGenerator(reqURL) {
-		return result
-	}
-
-	e.bulkHTTPRequest.CreateGenerator(reqURL)
-
-	// need to extract the target from the url
-	URL, err := url.Parse(reqURL)
-	if err != nil {
-		return result
-	}
-
-	pipeOptions := rawhttp.DefaultPipelineOptions
-	pipeOptions.Host = URL.Host
-	pipeOptions.MaxConnections = 1
-	if e.bulkHTTPRequest.PipelineConcurrentConnections > 0 {
-		pipeOptions.MaxConnections = e.bulkHTTPRequest.PipelineConcurrentConnections
-	}
-	if e.bulkHTTPRequest.PipelineRequestsPerConnection > 0 {
-		pipeOptions.MaxPendingRequests = e.bulkHTTPRequest.PipelineRequestsPerConnection
-	}
-	pipeclient := rawhttp.NewPipelineClient(pipeOptions)
-
-	// defaultMaxWorkers should be a sufficient value to keep queues always full
-	maxWorkers := defaultMaxWorkers
-	// in case the queue is bigger increase the workers
-	if pipeOptions.MaxPendingRequests > maxWorkers {
-		maxWorkers = pipeOptions.MaxPendingRequests
-	}
-	swg := sizedwaitgroup.New(maxWorkers)
-	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
-		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
-			result.Error = err
-		} else {
-			swg.Add()
-			go func(httpRequest *requests.HTTPRequest) {
-				defer swg.Done()
-
-				// HTTP pipelining ignores rate limit
-				// If the request was built correctly then execute it
-				request.Pipeline = true
-				request.PipelineClient = pipeclient
-				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
-				if err != nil {
-					e.traceLog.Request(e.template.ID, reqURL, "http", err)
-					result.Error = errors.Wrap(err, "could not handle http request")
-				} else {
-					e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-				}
-				request.PipelineClient = nil
-			}(request)
-		}
-
-		e.bulkHTTPRequest.Increment(reqURL)
-	}
-	swg.Wait()
-	return result
-}
-
-// ExecuteHTTP executes the HTTP request on a URL
-func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result {
-	// verify if pipeline was requested
-	if e.bulkHTTPRequest.Pipeline {
-		return e.ExecuteTurboHTTP(reqURL)
-	}
-
-	// verify if a basic race condition was requested
-	if e.bulkHTTPRequest.Race && e.bulkHTTPRequest.RaceNumberRequests > 0 {
-		return e.ExecuteRaceRequest(reqURL)
-	}
-
-	// verify if parallel elaboration was requested
-	if e.bulkHTTPRequest.Threads > 0 {
-		return e.ExecuteParallelHTTP(p, reqURL)
-	}
-
-	var requestNumber int
-
-	result := &Result{
-		Matches:     make(map[string]interface{}),
-		Extractions: make(map[string]interface{}),
-		historyData: make(map[string]interface{}),
-	}
-
-	dynamicvalues := make(map[string]interface{})
-
-	// verify if the URL is already being processed
-	if e.bulkHTTPRequest.HasGenerator(reqURL) {
-		return result
-	}
-
-	remaining := e.bulkHTTPRequest.GetRequestCount()
-	e.bulkHTTPRequest.CreateGenerator(reqURL)
-
-	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
-		requestNumber++
-		httpRequest, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
-			result.Error = err
-			p.Drop(remaining)
-		} else {
-			e.ratelimiter.Take()
-			// If the request was built correctly then execute it
-			format := "%s_" + strconv.Itoa(requestNumber)
-			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, format)
-			if err != nil {
-				result.Error = errors.Wrap(err, "could not handle http request")
-				p.Drop(remaining)
-				e.traceLog.Request(e.template.ID, reqURL, "http", err)
-			} else {
-				e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-			}
-		}
-		p.Update()
-
-		// Check if has to stop processing at first valid result
-		if e.stopAtFirstMatch && result.GotResults {
-			p.Drop(remaining)
-			break
-		}
-
-		// move always forward with requests
-		e.bulkHTTPRequest.Increment(reqURL)
-		remaining--
-	}
-	gologger.Verbosef("Sent for [%s] to %s\n", "http-request", e.template.ID, reqURL)
-	return result
-}
-
-func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, format string) error {
-	e.setCustomHeaders(request)
-
-	var (
-		resp          *http.Response
-		err           error
-		dumpedRequest []byte
-		fromcache     bool
-	)
-
-	if e.debug || e.pf != nil {
-		dumpedRequest, err = requests.Dump(request, reqURL)
-		if err != nil {
-			return err
-		}
-	}
-
-	if e.debug {
-		gologger.Infof("Dumped HTTP request for %s (%s)\n\n", reqURL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
-	}
-
-	timeStart := time.Now()
-
-	if request.Pipeline {
-		resp, err = request.PipelineClient.DoRaw(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)))
-		if err != nil {
-			if resp != nil {
-				resp.Body.Close()
-			}
-			e.traceLog.Request(e.template.ID, reqURL, "http", err)
-			return err
-		}
-		e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-	} else if request.Unsafe {
-		// rawhttp
-		// burp uses "\r\n" as new line character
-		request.RawRequest.Data = strings.ReplaceAll(request.RawRequest.Data, "\n", "\r\n")
-		options := e.rawHTTPClient.Options
-		options.AutomaticContentLength = request.AutomaticContentLengthHeader
-		options.AutomaticHostHeader = request.AutomaticHostHeader
-		options.FollowRedirects = request.FollowRedirects
-		resp, err = e.rawHTTPClient.DoRawWithOptions(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)), options)
-		if err != nil {
-			if resp != nil {
-				resp.Body.Close()
-			}
-			e.traceLog.Request(e.template.ID, reqURL, "http", err)
-			return err
-		}
-		e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-	} else {
-		// if nuclei-project is available check if the request was already sent previously
-		if e.pf != nil {
-			// if unavailable fail silently
-			fromcache = true
-			// nolint:bodyclose // false positive the response is generated at runtime
-			resp, err = e.pf.Get(dumpedRequest)
-			if err != nil {
-				fromcache = false
-			}
-		}
-
-		// retryablehttp
-		if resp == nil {
-			resp, err = e.httpClient.Do(request.Request)
-			if err != nil {
-				if resp != nil {
-					resp.Body.Close()
-				}
-				e.traceLog.Request(e.template.ID, reqURL, "http", err)
-				return err
-			}
-			e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-		}
-	}
-
-	duration := time.Since(timeStart)
-
-	// Dump response - Step 1 - Decompression not yet handled
-	var dumpedResponse []byte
-	if e.debug {
-		var dumpErr error
-		dumpedResponse, dumpErr = httputil.DumpResponse(resp, true)
-		if dumpErr != nil {
-			return errors.Wrap(dumpErr, "could not dump http response")
-		}
-	}
-
-	data, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		_, copyErr := io.Copy(ioutil.Discard, resp.Body)
-		if copyErr != nil {
-			resp.Body.Close()
-			return copyErr
-		}
-
-		resp.Body.Close()
-
-		return errors.Wrap(err, "could not read http body")
-	}
-
-	resp.Body.Close()
-
-	// net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request
-	// so in case we have to manually do it
-	dataOrig := data
-	data, err = requests.HandleDecompression(request, data)
-	if err != nil {
-		return errors.Wrap(err, "could not decompress http body")
-	}
-
-	// Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation)
-	if e.debug {
-		dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data)
-		gologger.Infof("Dumped HTTP response for %s (%s)\n\n", reqURL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
-	}
-
-	// if nuclei-project is enabled store the response if not previously done
-	if e.pf != nil && !fromcache {
-		err := e.pf.Set(dumpedRequest, resp, data)
-		if err != nil {
-			return errors.Wrap(err, "could not store in project file")
-		}
-	}
-
-	// Convert response body from []byte to string with zero copy
-	body := unsafeToString(data)
-
-	headers := headersToString(resp.Header)
-
-	// store for internal purposes the DSL matcher data
-	// hardcode stopping storing data after defaultMaxHistorydata items
-	if len(result.historyData) < defaultMaxHistorydata {
-		result.Lock()
-		result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format))
-		result.Unlock()
-	}
-
-	matcherCondition := e.bulkHTTPRequest.GetMatchersCondition()
-	for _, matcher := range e.bulkHTTPRequest.Matchers {
-		// Check if the matcher matched
-		if !matcher.Match(resp, body, headers, duration, result.historyData) {
-			// If the condition is AND we haven't matched, try next request.
-			if matcherCondition == matchers.ANDCondition {
-				return nil
-			}
-		} else {
-			// If the matcher has matched, and its an OR
-			// write the first output then move to next matcher.
-			if matcherCondition == matchers.ORCondition {
-				result.Lock()
-				result.Matches[matcher.Name] = nil
-				// probably redundant but ensures we snapshot current payload values when matchers are valid
-				result.Meta = request.Meta
-				result.GotResults = true
-				result.Unlock()
-				e.writeOutputHTTP(request, resp, body, matcher, nil, result.Meta, reqURL)
-			}
-		}
-	}
-
-	// All matchers have successfully completed so now start with the
-	// next task which is extraction of input from matchers.
-	var extractorResults, outputExtractorResults []string
-
-	for _, extractor := range e.bulkHTTPRequest.Extractors {
-		for match := range extractor.Extract(resp, body, headers) {
-			if _, ok := dynamicvalues[extractor.Name]; !ok {
-				dynamicvalues[extractor.Name] = match
-			}
-
-			extractorResults = append(extractorResults, match)
-
-			if !extractor.Internal {
-				outputExtractorResults = append(outputExtractorResults, match)
-			}
-		}
-		// probably redundant but ensures we snapshot current payload values when extractors are valid
-		result.Lock()
-		result.Meta = request.Meta
-		result.Extractions[extractor.Name] = extractorResults
-		result.Unlock()
-	}
-
-	// Write a final string of output if matcher type is
-	// AND or if we have extractors for the mechanism too.
-	if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition {
-		e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, result.Meta, reqURL)
-		result.Lock()
-		result.GotResults = true
-		result.Unlock()
-	}
-
-	return nil
-}
-
-// Close closes the http executer for a template.
-func (e *HTTPExecuter) Close() {}
-
-// makeHTTPClient creates a http client
-func makeHTTPClient(proxyURL *url.URL, options *HTTPOptions) *retryablehttp.Client {
-	// Multiple Host
-	retryablehttpOptions := retryablehttp.DefaultOptionsSpraying
-	disableKeepAlives := true
-	maxIdleConns := 0
-	maxConnsPerHost := 0
-	maxIdleConnsPerHost := -1
-
-	if options.BulkHTTPRequest.Threads > 0 {
-		// Single host
-		retryablehttpOptions = retryablehttp.DefaultOptionsSingle
-		disableKeepAlives = false
-		maxIdleConnsPerHost = 500
-		maxConnsPerHost = 500
-	}
-
-	retryablehttpOptions.RetryWaitMax = 10 * time.Second
-	retryablehttpOptions.RetryMax = options.Retries
-	followRedirects := options.BulkHTTPRequest.Redirects
-	maxRedirects := options.BulkHTTPRequest.MaxRedirects
-
-	transport := &http.Transport{
-		DialContext:         options.Dialer.Dial,
-		MaxIdleConns:        maxIdleConns,
-		MaxIdleConnsPerHost: maxIdleConnsPerHost,
-		MaxConnsPerHost:     maxConnsPerHost,
-		TLSClientConfig: &tls.Config{
-			Renegotiation:      tls.RenegotiateOnceAsClient,
-			InsecureSkipVerify: true,
-		},
-		DisableKeepAlives: disableKeepAlives,
-	}
-
-	// Attempts to overwrite the dial function with the socks proxied version
-	if options.ProxySocksURL != "" {
-		var proxyAuth *proxy.Auth
-
-		socksURL, err := url.Parse(options.ProxySocksURL)
-
-		if err == nil {
-			proxyAuth = &proxy.Auth{}
-			proxyAuth.User = socksURL.User.Username()
-			proxyAuth.Password, _ = socksURL.User.Password()
-		}
-
-		dialer, err := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct)
-		dc := dialer.(interface {
-			DialContext(ctx context.Context, network, addr string) (net.Conn, error)
-		})
-
-		if err == nil {
-			transport.DialContext = dc.DialContext
-		}
-	}
-
-	if proxyURL != nil {
-		transport.Proxy = http.ProxyURL(proxyURL)
-	}
-
-	return retryablehttp.NewWithHTTPClient(&http.Client{
-		Transport:     transport,
-		Timeout:       time.Duration(options.Timeout) * time.Second,
-		CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects),
-	}, retryablehttpOptions)
-}
-
-type checkRedirectFunc func(_ *http.Request, requests []*http.Request) error
-
-func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirectFunc {
-	return func(_ *http.Request, requests []*http.Request) error {
-		if !followRedirects {
-			return http.ErrUseLastResponse
-		}
-
-		if maxRedirects == 0 {
-			if len(requests) > ten {
-				return http.ErrUseLastResponse
-			}
-
-			return nil
-		}
-
-		if len(requests) > maxRedirects {
-			return http.ErrUseLastResponse
-		}
-
-		return nil
-	}
-}
-
-func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) {
-	for _, customHeader := range e.customHeaders {
-		// This should be pre-computed somewhere and done only once
-		tokens := strings.SplitN(customHeader, ":", two)
-		// if it's an invalid header skip it
-		if len(tokens) < two {
-			continue
-		}
-
-		headerName, headerValue := tokens[0], strings.Join(tokens[1:], "")
-		if r.RawRequest != nil {
-			// rawhttp
-			r.RawRequest.Headers[headerName] = headerValue
-		} else {
-			// retryablehttp
-			headerName = strings.TrimSpace(headerName)
-			headerValue = strings.TrimSpace(headerValue)
-			r.Request.Header[headerName] = []string{headerValue}
-		}
-	}
-}
-
-type Result struct {
-	sync.Mutex
-	GotResults  bool
-	Done        bool
-	Meta        map[string]interface{}
-	Matches     map[string]interface{}
-	Extractions map[string]interface{}
-	historyData map[string]interface{}
-	Error       error
-}

v2/pkg/executer/output_dns.go

@@ -1,106 +0,0 @@
-package executer
-
-import (
-	"strings"
-
-	"github.com/miekg/dns"
-
-	jsoniter "github.com/json-iterator/go"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-)
-
-// writeOutputDNS writes dns output to streams
-// nolint:interfacer // dns.Msg is out of current scope
-func (e *DNSExecuter) writeOutputDNS(domain string, req, resp *dns.Msg, matcher *matchers.Matcher, extractorResults []string) {
-	if e.jsonOutput {
-		output := make(jsonOutput)
-		output["matched"] = domain
-
-		if !e.noMeta {
-			output["template"] = e.template.ID
-			output["type"] = "dns"
-			output["host"] = domain
-			for k, v := range e.template.Info {
-				output[k] = v
-			}
-			if matcher != nil && len(matcher.Name) > 0 {
-				output["matcher_name"] = matcher.Name
-			}
-			if len(extractorResults) > 0 {
-				output["extracted_results"] = extractorResults
-			}
-			if e.jsonRequest {
-				output["request"] = req.String()
-				output["response"] = resp.String()
-			}
-		}
-
-		data, err := jsoniter.Marshal(output)
-		if err != nil {
-			gologger.Warningf("Could not marshal json output: %s\n", err)
-		}
-		gologger.Silentf("%s", string(data))
-		if e.writer != nil {
-			if err := e.writer.Write(data); err != nil {
-				gologger.Errorf("Could not write output data: %s\n", err)
-				return
-			}
-		}
-		return
-	}
-
-	builder := &strings.Builder{}
-	colorizer := e.colorizer
-
-	if !e.noMeta {
-		builder.WriteRune('[')
-		builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String())
-
-		if matcher != nil && len(matcher.Name) > 0 {
-			builder.WriteString(":")
-			builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String())
-		}
-
-		builder.WriteString("] [")
-		builder.WriteString(colorizer.Colorizer.BrightBlue("dns").String())
-		builder.WriteString("] ")
-
-		if e.template.Info["severity"] != "" {
-			builder.WriteString("[")
-			builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"]))
-			builder.WriteString("] ")
-		}
-	}
-	builder.WriteString(domain)
-
-	// If any extractors, write the results
-	if len(extractorResults) > 0 && !e.noMeta {
-		builder.WriteString(" [")
-
-		for i, result := range extractorResults {
-			builder.WriteString(colorizer.Colorizer.BrightCyan(result).String())
-
-			if i != len(extractorResults)-1 {
-				builder.WriteRune(',')
-			}
-		}
-		builder.WriteString("]")
-	}
-	builder.WriteRune('\n')
-
-	// Write output to screen as well as any output file
-	message := builder.String()
-	gologger.Silentf("%s", message)
-
-	if e.writer != nil {
-		if e.coloredOutput {
-			message = e.decolorizer.ReplaceAllString(message, "")
-		}
-
-		if err := e.writer.WriteString(message); err != nil {
-			gologger.Errorf("Could not write output data: %s\n", err)
-			return
-		}
-	}
-}

v2/pkg/executer/output_http.go

@@ -1,147 +0,0 @@
-package executer
-
-import (
-	"net/http"
-	"net/http/httputil"
-	"strings"
-
-	jsoniter "github.com/json-iterator/go"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-)
-
-// writeOutputHTTP writes http output to streams
-func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Response, body string, matcher *matchers.Matcher, extractorResults []string, meta map[string]interface{}, reqURL string) {
-	var URL string
-	if req.RawRequest != nil {
-		URL = req.RawRequest.FullURL
-	}
-	if req.Request != nil {
-		URL = req.Request.URL.String()
-	}
-
-	if e.jsonOutput {
-		output := make(jsonOutput)
-
-		output["matched"] = URL
-		if !e.noMeta {
-			output["template"] = e.template.ID
-			output["type"] = "http"
-			output["host"] = reqURL
-			if len(meta) > 0 {
-				output["meta"] = meta
-			}
-			for k, v := range e.template.Info {
-				output[k] = v
-			}
-			if matcher != nil && len(matcher.Name) > 0 {
-				output["matcher_name"] = matcher.Name
-			}
-			if len(extractorResults) > 0 {
-				output["extracted_results"] = extractorResults
-			}
-
-			// TODO: URL should be an argument
-			if e.jsonRequest {
-				dumpedRequest, err := requests.Dump(req, URL)
-				if err != nil {
-					gologger.Warningf("could not dump request: %s\n", err)
-				} else {
-					output["request"] = string(dumpedRequest)
-				}
-
-				dumpedResponse, err := httputil.DumpResponse(resp, false)
-				if err != nil {
-					gologger.Warningf("could not dump response: %s\n", err)
-				} else {
-					output["response"] = string(dumpedResponse) + body
-				}
-			}
-		}
-
-		data, err := jsoniter.Marshal(output)
-		if err != nil {
-			gologger.Warningf("Could not marshal json output: %s\n", err)
-		}
-		gologger.Silentf("%s", string(data))
-
-		if e.writer != nil {
-			if err := e.writer.Write(data); err != nil {
-				gologger.Errorf("Could not write output data: %s\n", err)
-				return
-			}
-		}
-		return
-	}
-
-	builder := &strings.Builder{}
-	colorizer := e.colorizer
-
-	if !e.noMeta {
-		builder.WriteRune('[')
-		builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String())
-
-		if matcher != nil && len(matcher.Name) > 0 {
-			builder.WriteString(":")
-			builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String())
-		}
-
-		builder.WriteString("] [")
-		builder.WriteString(colorizer.Colorizer.BrightBlue("http").String())
-		builder.WriteString("] ")
-
-		if e.template.Info["severity"] != "" {
-			builder.WriteString("[")
-			builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"]))
-			builder.WriteString("] ")
-		}
-	}
-	builder.WriteString(URL)
-
-	// If any extractors, write the results
-	if len(extractorResults) > 0 && !e.noMeta {
-		builder.WriteString(" [")
-
-		for i, result := range extractorResults {
-			builder.WriteString(colorizer.Colorizer.BrightCyan(result).String())
-
-			if i != len(extractorResults)-1 {
-				builder.WriteRune(',')
-			}
-		}
-
-		builder.WriteString("]")
-	}
-
-	// write meta if any
-	if len(req.Meta) > 0 && !e.noMeta {
-		builder.WriteString(" [")
-
-		var metas []string
-
-		for name, value := range req.Meta {
-			metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(value.(string)).String())
-		}
-
-		builder.WriteString(strings.Join(metas, ","))
-		builder.WriteString("]")
-	}
-
-	builder.WriteRune('\n')
-
-	// Write output to screen as well as any output file
-	message := builder.String()
-	gologger.Silentf("%s", message)
-
-	if e.writer != nil {
-		if e.coloredOutput {
-			message = e.decolorizer.ReplaceAllString(message, "")
-		}
-
-		if err := e.writer.WriteString(message); err != nil {
-			gologger.Errorf("Could not write output data: %s\n", err)
-			return
-		}
-	}
-}

v2/pkg/executer/utils.go

@@ -1,59 +0,0 @@
-package executer
-
-import (
-	"net/http"
-	"net/url"
-	"strings"
-	"unsafe"
-)
-
-type jsonOutput map[string]interface{}
-
-// unsafeToString converts byte slice to string with zero allocations
-func unsafeToString(bs []byte) string {
-	return *(*string)(unsafe.Pointer(&bs))
-}
-
-// headersToString converts http headers to string
-func headersToString(headers http.Header) string {
-	builder := &strings.Builder{}
-
-	for header, values := range headers {
-		builder.WriteString(header)
-		builder.WriteString(": ")
-
-		for i, value := range values {
-			builder.WriteString(value)
-
-			if i != len(values)-1 {
-				builder.WriteRune('\n')
-				builder.WriteString(header)
-				builder.WriteString(": ")
-			}
-		}
-		builder.WriteRune('\n')
-	}
-	return builder.String()
-}
-
-// isURL tests a string to determine if it is a well-structured url or not.
-func isURL(toTest string) bool {
-	_, err := url.ParseRequestURI(toTest)
-	if err != nil {
-		return false
-	}
-	u, err := url.Parse(toTest)
-	if err != nil || u.Scheme == "" || u.Host == "" {
-		return false
-	}
-	return true
-}
-
-// extractDomain extracts the domain name of a URL
-func extractDomain(theURL string) string {
-	u, err := url.Parse(theURL)
-	if err != nil {
-		return ""
-	}
-	return u.Hostname()
-}

v2/pkg/extractors/extract.go

@@ -1,95 +0,0 @@
-package extractors
-
-import (
-	"net/http"
-
-	"github.com/miekg/dns"
-)
-
-// Extract extracts response from the parts of request using a regex
-func (e *Extractor) Extract(resp *http.Response, body, headers string) map[string]struct{} {
-	switch e.extractorType {
-	case RegexExtractor:
-		if e.part == BodyPart {
-			return e.extractRegex(body)
-		} else if e.part == HeaderPart {
-			return e.extractRegex(headers)
-		} else {
-			matches := e.extractRegex(headers)
-			if len(matches) > 0 {
-				return matches
-			}
-			return e.extractRegex(body)
-		}
-	case KValExtractor:
-		if e.part == HeaderPart {
-			return e.extractKVal(resp)
-		}
-
-		matches := e.extractKVal(resp)
-
-		if len(matches) > 0 {
-			return matches
-		}
-
-		return e.extractCookieKVal(resp)
-	}
-
-	return nil
-}
-
-// ExtractDNS extracts response from dns message using a regex
-// nolint:interfacer // dns.Msg is out of current scope
-func (e *Extractor) ExtractDNS(msg *dns.Msg) map[string]struct{} {
-	switch e.extractorType {
-	case RegexExtractor:
-		return e.extractRegex(msg.String())
-	case KValExtractor:
-	}
-
-	return nil
-}
-
-// extractRegex extracts text from a corpus and returns it
-func (e *Extractor) extractRegex(corpus string) map[string]struct{} {
-	results := make(map[string]struct{})
-
-	groupPlusOne := e.RegexGroup + 1
-	for _, regex := range e.regexCompiled {
-		matches := regex.FindAllStringSubmatch(corpus, -1)
-		for _, match := range matches {
-			if len(match) >= groupPlusOne {
-				results[match[e.RegexGroup]] = struct{}{}
-			}
-		}
-	}
-	return results
-}
-
-// extractKVal extracts text from http response
-func (e *Extractor) extractKVal(r *http.Response) map[string]struct{} {
-	results := make(map[string]struct{})
-
-	for _, k := range e.KVal {
-		for _, v := range r.Header.Values(k) {
-			results[v] = struct{}{}
-		}
-	}
-
-	return results
-}
-
-// extractCookieKVal extracts text from cookies
-func (e *Extractor) extractCookieKVal(r *http.Response) map[string]struct{} {
-	results := make(map[string]struct{})
-
-	for _, k := range e.KVal {
-		for _, cookie := range r.Cookies() {
-			if cookie.Name == k {
-				results[cookie.Value] = struct{}{}
-			}
-		}
-	}
-
-	return results
-}

v2/pkg/generators/attack.go

@@ -1,20 +0,0 @@
-package generators
-
-// Type is type of attack
-type Type int
-
-const (
-	// Sniper attack - each variable replaced with values at a time
-	Sniper Type = iota + 1
-	// PitchFork attack - Each variable replaced with positional value in multiple wordlists
-	PitchFork
-	// ClusterBomb attack - Generate all possible combinations of values
-	ClusterBomb
-)
-
-// AttackTypes is an table for conversion of attack type from string.
-var AttackTypes = map[string]Type{
-	"sniper":      Sniper,
-	"pitchfork":   PitchFork,
-	"clusterbomb": ClusterBomb,
-}

v2/pkg/generators/clusterbomb.go

@@ -1,53 +0,0 @@
-package generators
-
-// ClusterbombGenerator Attack - Generate all possible combinations from an input map with all values listed
-// as slices of the same size
-func ClusterbombGenerator(payloads map[string][]string) (out chan map[string]interface{}) {
-	out = make(chan map[string]interface{})
-
-	// generator
-	go func() {
-		defer close(out)
-
-		var order []string
-
-		var parts [][]string
-
-		for name, wordlist := range payloads {
-			order = append(order, name)
-			parts = append(parts, wordlist)
-		}
-
-		var n = 1
-		for _, ar := range parts {
-			n *= len(ar)
-		}
-
-		var at = make([]int, len(parts))
-	loop:
-		for {
-			// increment position counters
-			for i := len(parts) - 1; i >= 0; i-- {
-				if at[i] > 0 && at[i] >= len(parts[i]) {
-					if i == 0 || (i == 1 && at[i-1] == len(parts[0])-1) {
-						break loop
-					}
-					at[i] = 0
-					at[i-1]++
-				}
-			}
-			// construct permutation
-			item := make(map[string]interface{})
-			for i, ar := range parts {
-				var p = at[i]
-				if p >= 0 && p < len(ar) {
-					item[order[i]] = ar[p]
-				}
-			}
-			out <- item
-			at[len(parts)-1]++
-		}
-	}()
-
-	return out
-}

v2/pkg/generators/pitchfork.go

@@ -1,38 +0,0 @@
-package generators
-
-// PitchforkGenerator Attack - Generate positional combinations from an input map with all values listed
-// as slices of the same size
-func PitchforkGenerator(payloads map[string][]string) (out chan map[string]interface{}) {
-	out = make(chan map[string]interface{})
-
-	size := 0
-
-	// check if all wordlists have the same size
-	for _, wordlist := range payloads {
-		if size == 0 {
-			size = len(wordlist)
-		}
-
-		if len(wordlist) != size {
-			// set size = 0 and exit the cycle
-			size = 0
-			break
-		}
-	}
-
-	// generator
-	go func() {
-		defer close(out)
-
-		for i := 0; i < size; i++ {
-			element := make(map[string]interface{})
-			for name, wordlist := range payloads {
-				element[name] = wordlist[i]
-			}
-
-			out <- element
-		}
-	}()
-
-	return out
-}

v2/pkg/generators/sniper.go

@@ -1,21 +0,0 @@
-package generators
-
-// SniperGenerator Attack - Generate sequential combinations
-func SniperGenerator(payloads map[string][]string) (out chan map[string]interface{}) {
-	out = make(chan map[string]interface{})
-
-	// generator
-	go func() {
-		defer close(out)
-
-		for name, wordlist := range payloads {
-			for _, value := range wordlist {
-				element := CopyMapWithDefaultValue(payloads, "")
-				element[name] = value
-				out <- element
-			}
-		}
-	}()
-
-	return out
-}

v2/pkg/generators/util.go

@@ -1,201 +0,0 @@
-package generators
-
-import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"math/rand"
-	"os"
-	"strings"
-)
-
-const two = 2
-
-// LoadPayloads creating proper data structure
-func LoadPayloads(payloads map[string]interface{}) map[string][]string {
-	loadedPayloads := make(map[string][]string)
-	// load all wordlists
-	for name, payload := range payloads {
-		switch pt := payload.(type) {
-		case string:
-			elements := strings.Split(pt, "\n")
-			if len(elements) >= two {
-				loadedPayloads[name] = elements
-			} else {
-				loadedPayloads[name] = LoadFile(pt)
-			}
-		case []interface{}, interface{}:
-			vv := payload.([]interface{})
-
-			var v []string
-
-			for _, vvv := range vv {
-				v = append(v, fmt.Sprintf("%v", vvv))
-			}
-
-			loadedPayloads[name] = v
-		}
-	}
-
-	return loadedPayloads
-}
-
-// LoadFile into slice of strings
-func LoadFile(filepath string) (lines []string) {
-	for line := range StreamFile(filepath) {
-		lines = append(lines, line)
-	}
-
-	return
-}
-
-// StreamFile content to a chan
-func StreamFile(filepath string) (content chan string) {
-	content = make(chan string)
-
-	go func() {
-		defer close(content)
-
-		file, err := os.Open(filepath)
-
-		if err != nil {
-			return
-		}
-		defer file.Close()
-
-		// yql filter applied
-		scanner := bufio.NewScanner(file)
-		for scanner.Scan() {
-			content <- scanner.Text()
-		}
-
-		if err := scanner.Err(); err != nil {
-			return
-		}
-	}()
-
-	return
-}
-
-// MergeMaps into a new one
-func MergeMaps(m1, m2 map[string]interface{}) (m map[string]interface{}) {
-	m = make(map[string]interface{})
-
-	for k, v := range m1 {
-		m[k] = v
-	}
-
-	for k, v := range m2 {
-		m[k] = v
-	}
-
-	return
-}
-
-// MergeMapsWithStrings into a new string one
-func MergeMapsWithStrings(m1, m2 map[string]string) (m map[string]string) {
-	m = make(map[string]string)
-	for k, v := range m1 {
-		m[k] = v
-	}
-
-	for k, v := range m2 {
-		m[k] = v
-	}
-
-	return
-}
-
-func reverseString(s string) string {
-	runes := []rune(s)
-	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
-		runes[i], runes[j] = runes[j], runes[i]
-	}
-
-	return string(runes)
-}
-
-// CopyMap creates a new copy of an existing map
-func CopyMap(originalMap map[string]interface{}) map[string]interface{} {
-	newMap := make(map[string]interface{})
-	for key, value := range originalMap {
-		newMap[key] = value
-	}
-
-	return newMap
-}
-
-// CopyMapWithDefaultValue creates a new copy of an existing map and set a default value
-func CopyMapWithDefaultValue(originalMap map[string][]string, defaultValue interface{}) map[string]interface{} {
-	newMap := make(map[string]interface{})
-	for key := range originalMap {
-		newMap[key] = defaultValue
-	}
-
-	return newMap
-}
-
-// StringContainsAnyMapItem verifies is a string contains any value of a map
-func StringContainsAnyMapItem(m map[string]interface{}, s string) bool {
-	for key := range m {
-		if strings.Contains(s, key) {
-			return true
-		}
-	}
-
-	return false
-}
-
-// TrimDelimiters removes trailing brackets
-func TrimDelimiters(s string) string {
-	return strings.TrimSuffix(strings.TrimPrefix(s, "{{"), "}}")
-}
-
-// FileExists checks if a file exists and is not a directory
-func FileExists(filename string) bool {
-	info, err := os.Stat(filename)
-	if os.IsNotExist(err) {
-		return false
-	}
-
-	return !info.IsDir()
-}
-
-// TrimDelimiters removes trailing brackets
-func SliceContins(s []string, k string) bool {
-	for _, a := range s {
-		if a == k {
-			return true
-		}
-	}
-	return false
-}
-
-func TrimAll(s, cutset string) string {
-	for _, c := range cutset {
-		s = strings.ReplaceAll(s, string(c), "")
-	}
-	return s
-}
-
-func RandSeq(base string, n int) string {
-	b := make([]rune, n)
-	for i := range b {
-		b[i] = rune(base[rand.Intn(len(base))])
-	}
-	return string(b)
-}
-
-func insertInto(s string, interval int, sep rune) string {
-	var buffer bytes.Buffer
-	before := interval - 1
-	last := len(s) - 1
-	for i, char := range s {
-		buffer.WriteRune(char)
-		if i%interval == before && i != last {
-			buffer.WriteRune(sep)
-		}
-	}
-	buffer.WriteRune(sep)
-	return buffer.String()
-}

v2/pkg/matchers/util.go

@@ -1,85 +0,0 @@
-package matchers
-
-import (
-	"fmt"
-	"net/http"
-	"net/http/httputil"
-	"strings"
-	"time"
-
-	"github.com/miekg/dns"
-)
-
-const defaultFormat = "%s"
-
-// HTTPToMap Converts HTTP to Matcher Map
-func HTTPToMap(resp *http.Response, body, headers string, duration time.Duration, format string) (m map[string]interface{}) {
-	m = make(map[string]interface{})
-
-	if format == "" {
-		format = defaultFormat
-	}
-
-	m[fmt.Sprintf(format, "content_length")] = resp.ContentLength
-	m[fmt.Sprintf(format, "status_code")] = resp.StatusCode
-
-	for k, v := range resp.Header {
-		k = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(k, "-", "_")))
-		m[fmt.Sprintf(format, k)] = strings.Join(v, " ")
-	}
-
-	m[fmt.Sprintf(format, "all_headers")] = headers
-	m[fmt.Sprintf(format, "body")] = body
-
-	if r, err := httputil.DumpResponse(resp, true); err == nil {
-		m[fmt.Sprintf(format, "raw")] = string(r)
-	}
-
-	// Converts duration to seconds (floating point) for DSL syntax
-	m[fmt.Sprintf(format, "duration")] = duration.Seconds()
-
-	return m
-}
-
-// DNSToMap Converts DNS to Matcher Map
-func DNSToMap(msg *dns.Msg, format string) (m map[string]interface{}) {
-	m = make(map[string]interface{})
-
-	if format == "" {
-		format = defaultFormat
-	}
-
-	m[fmt.Sprintf(format, "rcode")] = msg.Rcode
-
-	var qs string
-
-	for _, question := range msg.Question {
-		qs += fmt.Sprintln(question.String())
-	}
-
-	m[fmt.Sprintf(format, "question")] = qs
-
-	var exs string
-	for _, extra := range msg.Extra {
-		exs += fmt.Sprintln(extra.String())
-	}
-
-	m[fmt.Sprintf(format, "extra")] = exs
-
-	var ans string
-	for _, answer := range msg.Answer {
-		ans += fmt.Sprintln(answer.String())
-	}
-
-	m[fmt.Sprintf(format, "answer")] = ans
-
-	var nss string
-	for _, ns := range msg.Ns {
-		nss += fmt.Sprintln(ns.String())
-	}
-
-	m[fmt.Sprintf(format, "ns")] = nss
-	m[fmt.Sprintf(format, "raw")] = msg.String()
-
-	return m
-}

v2/pkg/operators/common/dsl/dsl.go

@@ -1,6 +1,7 @@
-package generators
+package dsl
 
 import (
+	"bytes"
 	"crypto/md5"
 	"crypto/sha1"
 	"crypto/sha256"
@@ -16,166 +17,160 @@ import (
 	"time"
 
 	"github.com/Knetic/govaluate"
-	"github.com/projectdiscovery/nuclei/v2/pkg/collaborator"
+	"github.com/projectdiscovery/nuclei/v2/internal/collaborator"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/spaolacci/murmur3"
 )
 
 const (
+	numbers              = "1234567890"
+	letters              = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 	withCutSetArgsSize   = 2
-	withMaxRandArgsSize  = withCutSetArgsSize
 	withBaseRandArgsSize = 3
+	withMaxRandArgsSize  = withCutSetArgsSize
 )
 
-var letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
-var numbers = "1234567890"
+// HelperFunctions contains the dsl helper functions
+func HelperFunctions() map[string]govaluate.ExpressionFunction {
+	functions := make(map[string]govaluate.ExpressionFunction)
 
-// HelperFunctions contains the dsl functions
-func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
-	functions = make(map[string]govaluate.ExpressionFunction)
-
-	// strings
 	functions["len"] = func(args ...interface{}) (interface{}, error) {
-		length := len(args[0].(string))
-
+		length := len(types.ToString(args[0]))
 		return float64(length), nil
 	}
 
 	functions["toupper"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ToUpper(args[0].(string)), nil
+		return strings.ToUpper(types.ToString(args[0])), nil
 	}
 
 	functions["tolower"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ToLower(args[0].(string)), nil
+		return strings.ToLower(types.ToString(args[0])), nil
 	}
 
 	functions["replace"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ReplaceAll(args[0].(string), args[1].(string), args[2].(string)), nil
+		return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil
 	}
 
 	functions["replace_regex"] = func(args ...interface{}) (interface{}, error) {
-		compiled, err := regexp.Compile(args[1].(string))
+		compiled, err := regexp.Compile(types.ToString(args[1]))
 		if err != nil {
 			return nil, err
 		}
-		return compiled.ReplaceAllString(args[0].(string), args[2].(string)), nil
+		return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil
 	}
 
 	functions["trim"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Trim(args[0].(string), args[2].(string)), nil
+		return strings.Trim(types.ToString(args[0]), types.ToString(args[2])), nil
 	}
 
 	functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimLeft(args[0].(string), args[1].(string)), nil
+		return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil
 	}
 
 	functions["trimright"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimRight(args[0].(string), args[1].(string)), nil
+		return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil
 	}
 
 	functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimSpace(args[0].(string)), nil
+		return strings.TrimSpace(types.ToString(args[0])), nil
 	}
 
 	functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimPrefix(args[0].(string), args[1].(string)), nil
+		return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil
 	}
 
 	functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimSuffix(args[0].(string), args[1].(string)), nil
+		return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil
 	}
 
 	functions["reverse"] = func(args ...interface{}) (interface{}, error) {
-		return reverseString(args[0].(string)), nil
+		return reverseString(types.ToString(args[0])), nil
 	}
 
 	// encoding
 	functions["base64"] = func(args ...interface{}) (interface{}, error) {
-		sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
+		sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
 
 		return sEnc, nil
 	}
 
 	// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
 	functions["base64_py"] = func(args ...interface{}) (interface{}, error) {
-		sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
-
+		sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
 		return insertInto(sEnc, 76, '\n'), nil
 	}
 
 	functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
-		return base64.StdEncoding.DecodeString(args[0].(string))
+		return base64.StdEncoding.DecodeString(types.ToString(args[0]))
 	}
 
 	functions["url_encode"] = func(args ...interface{}) (interface{}, error) {
-		return url.PathEscape(args[0].(string)), nil
+		return url.PathEscape(types.ToString(args[0])), nil
 	}
 
 	functions["url_decode"] = func(args ...interface{}) (interface{}, error) {
-		return url.PathUnescape(args[0].(string))
+		return url.PathUnescape(types.ToString(args[0]))
 	}
 
 	functions["hex_encode"] = func(args ...interface{}) (interface{}, error) {
-		return hex.EncodeToString([]byte(args[0].(string))), nil
+		return hex.EncodeToString([]byte(types.ToString(args[0]))), nil
 	}
 
 	functions["hex_decode"] = func(args ...interface{}) (interface{}, error) {
-		hx, _ := hex.DecodeString(args[0].(string))
+		hx, _ := hex.DecodeString(types.ToString(args[0]))
 		return string(hx), nil
 	}
 
 	functions["html_escape"] = func(args ...interface{}) (interface{}, error) {
-		return html.EscapeString(args[0].(string)), nil
+		return html.EscapeString(types.ToString(args[0])), nil
 	}
 
 	functions["html_unescape"] = func(args ...interface{}) (interface{}, error) {
-		return html.UnescapeString(args[0].(string)), nil
+		return html.UnescapeString(types.ToString(args[0])), nil
 	}
 
 	// hashing
 	functions["md5"] = func(args ...interface{}) (interface{}, error) {
-		hash := md5.Sum([]byte(args[0].(string)))
+		hash := md5.Sum([]byte(types.ToString(args[0])))
 
 		return hex.EncodeToString(hash[:]), nil
 	}
 
 	functions["sha256"] = func(args ...interface{}) (interface{}, error) {
 		h := sha256.New()
-		_, err := h.Write([]byte(args[0].(string)))
+		_, err := h.Write([]byte(types.ToString(args[0])))
 
 		if err != nil {
 			return nil, err
 		}
-
 		return hex.EncodeToString(h.Sum(nil)), nil
 	}
 
 	functions["sha1"] = func(args ...interface{}) (interface{}, error) {
 		h := sha1.New()
-		_, err := h.Write([]byte(args[0].(string)))
+		_, err := h.Write([]byte(types.ToString(args[0])))
 
 		if err != nil {
 			return nil, err
 		}
-
 		return hex.EncodeToString(h.Sum(nil)), nil
 	}
 
 	functions["mmh3"] = func(args ...interface{}) (interface{}, error) {
-		return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(args[0].(string)), 0))), nil
+		return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil
 	}
 
 	// search
 	functions["contains"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Contains(args[0].(string), args[1].(string)), nil
+		return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil
 	}
 
 	functions["regex"] = func(args ...interface{}) (interface{}, error) {
-		compiled, err := regexp.Compile(args[0].(string))
+		compiled, err := regexp.Compile(types.ToString(args[0]))
 		if err != nil {
 			return nil, err
 		}
-
-		return compiled.MatchString(args[1].(string)), nil
+		return compiled.MatchString(types.ToString(args[1])), nil
 	}
 
 	// random generators
@@ -183,14 +178,12 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 		chars := letters + numbers
 		bad := ""
 		if len(args) >= 1 {
-			chars = args[0].(string)
+			chars = types.ToString(args[0])
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = args[1].(string)
+			bad = types.ToString(args[1])
 		}
-
-		chars = TrimAll(chars, bad)
-
+		chars = trimAll(chars, bad)
 		return chars[rand.Intn(len(chars))], nil
 	}
 
@@ -203,15 +196,13 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = args[1].(string)
+			bad = types.ToString(args[1])
 		}
 		if len(args) >= withBaseRandArgsSize {
-			base = args[2].(string)
+			base = types.ToString(args[2])
 		}
-
-		base = TrimAll(base, bad)
-
-		return RandSeq(base, l), nil
+		base = trimAll(base, bad)
+		return randSeq(base, l), nil
 	}
 
 	functions["rand_text_alphanumeric"] = func(args ...interface{}) (interface{}, error) {
@@ -223,12 +214,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = args[1].(string)
+			bad = types.ToString(args[1])
 		}
-
-		chars = TrimAll(chars, bad)
-
-		return RandSeq(chars, l), nil
+		chars = trimAll(chars, bad)
+		return randSeq(chars, l), nil
 	}
 
 	functions["rand_text_alpha"] = func(args ...interface{}) (interface{}, error) {
@@ -240,12 +229,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = args[1].(string)
+			bad = types.ToString(args[1])
 		}
-
-		chars = TrimAll(chars, bad)
-
-		return RandSeq(chars, l), nil
+		chars = trimAll(chars, bad)
+		return randSeq(chars, l), nil
 	}
 
 	functions["rand_text_numeric"] = func(args ...interface{}) (interface{}, error) {
@@ -257,12 +244,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = args[1].(string)
+			bad = types.ToString(args[1])
 		}
-
-		chars = TrimAll(chars, bad)
-
-		return RandSeq(chars, l), nil
+		chars = trimAll(chars, bad)
+		return randSeq(chars, l), nil
 	}
 
 	functions["rand_int"] = func(args ...interface{}) (interface{}, error) {
@@ -275,7 +260,6 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 		if len(args) >= withMaxRandArgsSize {
 			max = args[1].(int)
 		}
-
 		return rand.Intn(max-min) + min, nil
 	}
 
@@ -289,8 +273,44 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 	// Collaborator
 	functions["collab"] = func(args ...interface{}) (interface{}, error) {
 		// check if collaborator contains a specific pattern
-		return collaborator.DefaultCollaborator.Has(args[0].(string)), nil
+		return collaborator.DefaultCollaborator.Has(types.ToString(args[0])), nil
 	}
-
 	return functions
 }
+
+func reverseString(s string) string {
+	runes := []rune(s)
+	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
+		runes[i], runes[j] = runes[j], runes[i]
+	}
+	return string(runes)
+}
+
+func trimAll(s, cutset string) string {
+	for _, c := range cutset {
+		s = strings.ReplaceAll(s, string(c), "")
+	}
+	return s
+}
+
+func randSeq(base string, n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = rune(base[rand.Intn(len(base))])
+	}
+	return string(b)
+}
+
+func insertInto(s string, interval int, sep rune) string {
+	var buffer bytes.Buffer
+	before := interval - 1
+	last := len(s) - 1
+	for i, char := range s {
+		buffer.WriteRune(char)
+		if i%interval == before && i != last {
+			buffer.WriteRune(sep)
+		}
+	}
+	buffer.WriteRune(sep)
+	return buffer.String()
+}

v2/pkg/operators/extractors/compile.go

@@ -3,6 +3,7 @@ package extractors
 import (
 	"fmt"
 	"regexp"
+	"strings"
 )
 
 // CompileExtractors performs the initial setup operation on a extractor
@@ -20,19 +21,16 @@ func (e *Extractor) CompileExtractors() error {
 		if err != nil {
 			return fmt.Errorf("could not compile regex: %s", regex)
 		}
-
 		e.regexCompiled = append(e.regexCompiled, compiled)
 	}
 
-	// Setup the part of the request to match, if any.
-	if e.Part != "" {
-		e.part, ok = PartTypes[e.Part]
-		if !ok {
-			return fmt.Errorf("unknown matcher part specified: %s", e.Part)
-		}
-	} else {
-		e.part = BodyPart
+	for i, kval := range e.KVal {
+		e.KVal[i] = strings.ToLower(kval)
 	}
 
+	// Setup the part of the request to match, if any.
+	if e.Part == "" {
+		e.Part = "body"
+	}
 	return nil
 }

v2/pkg/operators/extractors/extract.go

@@ -0,0 +1,44 @@
+package extractors
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+// ExtractRegex extracts text from a corpus and returns it
+func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} {
+	results := make(map[string]struct{})
+
+	groupPlusOne := e.RegexGroup + 1
+	for _, regex := range e.regexCompiled {
+		matches := regex.FindAllStringSubmatch(corpus, -1)
+
+		for _, match := range matches {
+			if len(match) < groupPlusOne {
+				continue
+			}
+			matchString := match[e.RegexGroup]
+
+			if _, ok := results[matchString]; !ok {
+				results[matchString] = struct{}{}
+			}
+		}
+	}
+	return results
+}
+
+// ExtractKval extracts key value pairs from a data map
+func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{} {
+	results := make(map[string]struct{})
+
+	for _, k := range e.KVal {
+		item, ok := data[k]
+		if !ok {
+			continue
+		}
+		itemString := types.ToString(item)
+		if _, ok := results[itemString]; !ok {
+			results[itemString] = struct{}{}
+		}
+	}
+	return results
+}

v2/pkg/operators/extractors/extractors.go

@@ -25,8 +25,6 @@ type Extractor struct {
 	//
 	// By default, matching is performed in request body.
 	Part string `yaml:"part,omitempty"`
-	// part is the part of the request to match
-	part Part
 	// Internal defines if this is used internally
 	Internal bool `yaml:"internal,omitempty"`
 }
@@ -47,26 +45,7 @@ var ExtractorTypes = map[string]ExtractorType{
 	"kval":  KValExtractor,
 }
 
-// Part is the part of the request to match
-type Part int
-
-const (
-	// BodyPart matches body of the response.
-	BodyPart Part = iota + 1
-	// HeaderPart matches headers of the response.
-	HeaderPart
-	// AllPart matches both response body and headers of the response.
-	AllPart
-)
-
-// PartTypes is an table for conversion of part type from string.
-var PartTypes = map[string]Part{
-	"body":   BodyPart,
-	"header": HeaderPart,
-	"all":    AllPart,
-}
-
-// GetPart returns the part of the matcher
-func (e *Extractor) GetPart() Part {
-	return e.part
+// GetType returns the type of the matcher
+func (e *Extractor) GetType() ExtractorType {
+	return e.extractorType
 }

v2/pkg/operators/matchers/compile.go

@@ -1,22 +1,36 @@
 package matchers
 
 import (
+	"encoding/hex"
 	"fmt"
 	"regexp"
 
 	"github.com/Knetic/govaluate"
-	"github.com/projectdiscovery/nuclei/v2/pkg/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
 )
 
 // CompileMatchers performs the initial setup operation on a matcher
 func (m *Matcher) CompileMatchers() error {
 	var ok bool
 
+	// Support hexadecimal encoding for matchers too.
+	if m.Encoding == "hex" {
+		for i, word := range m.Words {
+			if decoded, err := hex.DecodeString(word); err == nil && len(decoded) > 0 {
+				m.Words[i] = string(decoded)
+			}
+		}
+	}
+
 	// Setup the matcher type
 	m.matcherType, ok = MatcherTypes[m.Type]
 	if !ok {
 		return fmt.Errorf("unknown matcher type specified: %s", m.Type)
 	}
+	// By default, match on body if user hasn't provided any specific items
+	if m.Part == "" {
+		m.Part = "body"
+	}
 
 	// Compile the regexes
 	for _, regex := range m.Regex {
@@ -24,17 +38,15 @@ func (m *Matcher) CompileMatchers() error {
 		if err != nil {
 			return fmt.Errorf("could not compile regex: %s", regex)
 		}
-
 		m.regexCompiled = append(m.regexCompiled, compiled)
 	}
 
 	// Compile the dsl expressions
-	for _, dsl := range m.DSL {
-		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(dsl, generators.HelperFunctions())
+	for _, expr := range m.DSL {
+		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
 		if err != nil {
-			return fmt.Errorf("could not compile dsl: %s", dsl)
+			return fmt.Errorf("could not compile dsl: %s", expr)
 		}
-
 		m.dslCompiled = append(m.dslCompiled, compiled)
 	}
 
@@ -47,16 +59,5 @@ func (m *Matcher) CompileMatchers() error {
 	} else {
 		m.condition = ORCondition
 	}
-
-	// Setup the part of the request to match, if any.
-	if m.Part != "" {
-		m.part, ok = PartTypes[m.Part]
-		if !ok {
-			return fmt.Errorf("unknown matcher part specified: %s", m.Part)
-		}
-	} else {
-		m.part = BodyPart
-	}
-
 	return nil
 }