Bug fix

v2.3.1 bug fixes

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,14 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: "[feature]"
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.

.github/ISSUE_TEMPLATE/issue-report.md

@@ -0,0 +1,18 @@
+---
+name: Issue report
+about: Create a report to help us improve
+title: "[issue]"
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Nuclei version**
+Please share the version of the nuclei you are running with `nuclei -version` 
+
+
+**Screenshot of the error or bug**
+please add the screenshot showing bug or issue you are facing.

.github/dependabot.yml

@@ -0,0 +1,34 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "chore"
+      include: "scope"
+
+  # Maintain dependencies for go modules
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "chore"
+      include: "scope"
+
+  # Maintain dependencies for docker
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "chore"
+      include: "scope"

.github/workflows/build.yaml

@@ -4,24 +4,42 @@ on:
     branches:
       - master
   pull_request:
-  
-jobs:          
+
+jobs:
+  lint:
+    name: golangci-lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
+          version: v1.33
+          args: --timeout 5m
+          working-directory: v2/
+        
   build:
     name: Build
-    runs-on: ubuntu-latest 
+    runs-on: ubuntu-latest
     steps:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.14
+          go-version: 1.15
 
       - name: Check out code
         uses: actions/checkout@v2
 
       - name: Test
-        run: go test .
-        working-directory: v2/cmd/nuclei/
+        run: go test ./...
+        working-directory: v2/
+
+      - name: Integration Tests
+        run: bash run.sh
+        working-directory: integration_tests/
 
       - name: Build
         run: go build .
-        working-directory: v2/cmd/nuclei/
+        working-directory: v2/cmd/nuclei/

.github/workflows/dockerhub-push.yml

@@ -5,13 +5,15 @@ on:
   release:
     types: [published]
 jobs:
-  build:
+  update:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@master
     - name: Publish to Dockerhub Registry
+      #pre: echo ::save-state name=RELEASE_VERSION::$(echo ${GITHUB_REF:10})
       uses: elgohr/Publish-Docker-Github-Action@master
       with:
         name: projectdiscovery/nuclei
         username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        tags: "latest" #"latest,${{ env.STATE_RELEASE_VERSION }}"

.github/workflows/release.yml

@@ -4,22 +4,27 @@ on:
     tags:
       - v*
 
-jobs:
-  release:
-    name: Release on GitHub
+jobs: 
+  release: 
     runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v1
-
-      - name: Validates GO releaser config
-        uses: docker://goreleaser/goreleaser:latest
-        with:
-          args: check
-
-      - name: Create release on GitHub
-        uses: docker://goreleaser/goreleaser:latest
-        with:
-          args: release
-        env:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+    steps: 
+      - 
+        name: "Check out code"
+        uses: actions/checkout@v2
+        with: 
+          fetch-depth: 0
+      - 
+        name: "Set up Go"
+        uses: actions/setup-go@v2
+        with: 
+          go-version: 1.15
+      - 
+        env: 
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        name: "Create release on GitHub"
+        uses: goreleaser/goreleaser-action@v2
+        with: 
+          args: "release --rm-dist"
+          version: latest
+          workdir: v2/
+        

.gitignore

@@ -1,2 +1,6 @@
 cmd/nuclei/nuclei*
 v2/cmd/nuclei/nuclei
+.idea
+integration_tests/integration-test
+integration_tests/nuclei
+v2/cmd/integration-test/integration-test

.golangci.yml

@@ -0,0 +1,115 @@
+linters-settings:
+  dupl:
+    threshold: 100
+  exhaustive:
+    default-signifies-exhaustive: false
+  # funlen:
+  #   lines: 100
+  #   statements: 50
+  #goconst:
+  #  min-len: 2
+  #  min-occurrences: 2
+  gocritic:
+    enabled-tags:
+      - diagnostic
+      - experimental
+      - opinionated
+      - performance
+      - style
+    disabled-checks:
+      - dupImport # https://github.com/go-critic/go-critic/issues/845
+      - ifElseChain
+  # gocyclo:
+  #   min-complexity: 15
+  goimports:
+    local-prefixes: github.com/golangci/golangci-lint
+  golint:
+    min-confidence: 0
+  gomnd:
+    settings:
+      mnd:
+        # don't include the "operation" and "assign"
+        checks: argument,case,condition,return
+  govet:
+    check-shadowing: true
+    settings:
+      printf:
+        funcs:
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
+  # lll:
+  #   line-length: 140
+  maligned:
+    suggest-new: true
+  misspell:
+    locale: US
+  nolintlint:
+    allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
+    allow-unused: false # report any unused nolint directives
+    require-explanation: false # don't require an explanation for nolint directives
+    require-specific: false # don't require nolint directives to be specific about which linter is being skipped
+
+linters:
+  # please, do not use `enable-all`: it's deprecated and will be removed soon.
+  # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
+  disable-all: true
+  enable:
+    #- bodyclose
+    - deadcode
+    - dogsled
+    - errcheck
+    - exhaustive
+    - gochecknoinits
+    #- goconst
+    - gocritic
+    - gofmt
+    - goimports
+    - golint
+    #- gomnd
+    - goprintffuncname
+    - gosimple
+    - govet
+    - ineffassign
+    - interfacer
+    - maligned
+    - misspell
+    - nakedret
+    - noctx
+    - nolintlint
+    - rowserrcheck
+    - scopelint
+    - staticcheck
+    - structcheck
+    - stylecheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+    - varcheck
+    - whitespace
+
+  # don't enable:
+  # - depguard
+  # - asciicheck
+  # - funlen
+  # - gochecknoglobals
+  # - gocognit
+  # - gocyclo
+  # - godot
+  # - godox
+  # - goerr113
+  # - gosec
+  # - lll
+  # - nestif
+  # - prealloc
+  # - testpackage
+  # - wsl
+
+# golangci.com configuration
+# https://github.com/golangci/golangci/wiki/Configuration
+service:
+  golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly
+  prepare:
+    - echo "here I can run custom commands, but no preparation needed for this repo"

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at contact@projectdiscovery.io. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

Dockerfile

@@ -1,10 +1,7 @@
-FROM golang:1.13.11-alpine3.10 AS build-env
+FROM golang:1.15-alpine as build-env
+RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
 
-RUN apk add --no-cache --upgrade git openssh-client ca-certificates
-RUN go get -u github.com/golang/dep/cmd/dep
-WORKDIR /go/src/app
-
-# Install
-RUN GO111MODULE=on go get -u github.com/projectdiscovery/nuclei/v2/cmd/nuclei
-
-ENTRYPOINT ["nuclei"]
+FROM alpine:latest
+RUN apk add --no-cache bind-tools ca-certificates
+COPY --from=build-env /go/bin/nuclei /usr/local/bin/nuclei
+ENTRYPOINT ["nuclei"]

LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Exposed Atoms
+Copyright (c) 2021 ProjectDiscovery, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,177 +1,165 @@
-<h1 align="left">
-  <img src="static/nuclei-logo.png" alt="nuclei" width="200px"></a>
+<h1 align="center">
   <br>
+  <a href="https://nuclei.projectdiscovery.io"><img src="static/nuclei-logo.png" width="200px" alt="Nuclei"></a>
 </h1>
 
-[![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT)
-[![Go Report Card](https://goreportcard.com/badge/github.com/projectdiscovery/nuclei)](https://goreportcard.com/report/github.com/projectdiscovery/nuclei)
-[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectdiscovery/nuclei/issues)
+<h4 align="center">Fast and customisable vulnerability scanner based on simple YAML based DSL.</h4>
 
-Nuclei is a fast tool for configurable targeted scanning based on templates offering massive extensibility and ease of use. 
 
-Nuclei is used to send requests across targets based on a template leading to zero false positives and providing effective scanning for known paths. Main use cases for nuclei are during initial reconnaissance phase to quickly check for low hanging fruits or CVEs across targets that are known and easily detectable. It uses [retryablehttp-go library](https://github.com/projectdiscovery/retryablehttp-go) designed to handle various errors and retries in case of blocking by WAFs, this is also one of our core modules from custom-queries.
+<p align="center">
+<a href="https://goreportcard.com/report/github.com/projectdiscovery/nuclei"><img src="https://goreportcard.com/badge/github.com/projectdiscovery/nuclei"></a>
+<a href="https://github.com/projectdiscovery/nuclei/issues"><img src="https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat"></a>
+<a href="https://github.com/projectdiscovery/nuclei/releases"><img src="https://img.shields.io/github/release/projectdiscovery/nuclei"></a>
+<a href="https://twitter.com/pdnuclei"><img src="https://img.shields.io/twitter/follow/pdnuclei.svg?logo=twitter"></a>
+<a href="https://discord.gg/KECAGdH"><img src="https://img.shields.io/discord/695645237418131507.svg?logo=discord"></a>
+<a href="https://github.com/projectdiscovery/nuclei/actions/workflows/build.yaml"><img src="https://github.com/projectdiscovery/nuclei/actions/workflows/build.yaml/badge.svg?branch=master"></a>
+</p>
+      
+<p align="center">
+  <a href="#how-it-works">How</a> •
+  <a href="#install-nuclei">Install</a> •
+  <a href="#for-security-engineers">For Security Engineers</a> •
+  <a href="#for-developers-and-organisations">For Developers</a> •
+  <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">Documentation</a> •
+  <a href="#credits">Credits</a> •
+  <a href="#license">License</a> •
+  <a href="https://discord.gg/KECAGdH">Join Discord</a>
+</p>
 
-We have also [open-sourced a dedicated repository](https://github.com/projectdiscovery/nuclei-templates) to maintain various type of templates, we hope that you will contribute there too. Templates are provided in hopes that these will be useful and will allow everyone to build their own templates for the scanner. Checkout the guide at [**GUIDE.md**](https://github.com/projectdiscovery/nuclei-templates/blob/master/GUIDE.md) for a primer on nuclei templates.
+---
 
-# Resources
-- [Resources](#resources)
-- [Features](#features)
-- [Usage](#usage)
-- [Installation Instructions](#installation-instructions)
-    - [From Binary](#from-binary)
-    - [From Source](#from-source)
-    - [Running in a Docker Container](#running-in-a-docker-container)
-- [Nuclei templates](#nuclei-templates)
-- [Running nuclei](#running-nuclei)
-    - [1. Running nuclei with a single template.](#1-running-nuclei-with-a-single-template)
-    - [2. Running nuclei with multiple templates.](#2-running-nuclei-with-multiple-templates)
-    - [3. Automating nuclei with subfinder and any other similar tool.](#3-automating-nuclei-with-subfinder-and-any-other-similar-tool)
-- [Thanks](#thanks)
+Nuclei is used to send requests across targets based on a template leading to zero false positives and providing fast scanning on large number of hosts. Nuclei offers scanning for a variety of protocols including TCP, DNS, HTTP, File, etc. With powerful and flexible templating, all kinds of security checks can be modelled with Nuclei.
 
- # Features
+We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-templates) that houses various type of vulnerability templates contributed by **more than 100** security researchers and engineers. It is preloaded with ready to use templates using `-update-templates` flag.
+
+
+
+## How it works
+
+
+<h3 align="center">
+  <img src="static/nuclei-flow.jpg" alt="nuclei-flow" width="700px"></a>
+</h3>
+
+
+# Install Nuclei
+
+```sh
+▶ GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+```
+
+**More installation [methods can be found here](https://nuclei.projectdiscovery.io/nuclei/get-started/#nuclei-installation).**
+
+<table>
+<tr>
+<td>  
+
+### Download Templates
+
+You can download and update the nuclei templates using <ins>*update-templates*</ins> flag of nuclei that downloads all the available **nuclei-templates** from [Github project](https://github.com/projectdiscovery/nuclei-templates), a community curated list of templates that are ready to use.
+
+`▶ nuclei -update-templates`
+
+Nuclei is designed to used with custom templates according to the target and workflow, you can write your own checks for your specific workflow and needs, please refer to nuclei [templating guide](https://nuclei.projectdiscovery.io/templating-guide/) to write your own custom templates.
+
+</td>
+</tr>
+</table>
+
+### Running Nuclei
+
+Scanning for CVEs on given list of URLs.
+
+```sh
+▶ nuclei -l target_urls.txt -t cves/
+```
+
+**More detailed examples of running nuclei can be found [here](https://nuclei.projectdiscovery.io/nuclei/get-started/#running-nuclei).**
+
+# For Security Engineers
+
+Nuclei offers great number of features that are helpful for security engineers to customise workflow in their organisation. With the varieties of scan capabilities (like DNS, HTTP, TCP), security engineers can easily create their suite of custom checks with Nuclei.
+
+- Varieties of protocols supported: TCP, DNS, HTTP, File, etc
+- Achieve complex vulnerability steps with workflows and [dynamic requests.](https://blog.projectdiscovery.io/nuclei-unleashed-quickly-write-complex-exploits/)
+- Easy to integrate into CI/CD, designed to be easily integrated into regression cycle to actively check the fix and re-appearance of vulnerability. 
 
 <h1 align="left">
-  <img src="static/nuclei-run.png" alt="nuclei" width="700px"></a>
-  <br>
+  <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/"><img src="static/learn-more-button.png" width="170px" alt="Learn More"></a>
 </h1>
 
- - Simple and modular code base making it easy to contribute.
- - Fast And fully configurable using a template based engine.
- - Handles edge cases doing retries, backoffs etc for handling WAFs.
- - Smart matching functionality for zero false positive scanning.
+<table>
+<tr>
+<td>  
 
-# Usage
+**For bugbounty hunters:**
 
-```bash
-nuclei -h
-```
+Nuclei allows you to customise your testing approach with your own suite of checks and easily run across your bug bounty programs. Moroever, Nuclei can be easily integrated into any continuous scanning workflow.
 
-This will display help for the tool. Here are all the switches it supports.
+- Designed to be easily integrated into other tool workflow.
+- Can process thousands of hosts in few minutes.
+- Easily automate your custom testing approach with our simple YAML DSL.
 
-| Flag              | Description                                           | Example                                            |
-|-------------------|-------------------------------------------------------|----------------------------------------------------|
-| -c                | Number of concurrent requests (default 10)            | nuclei -c 100                                      |
-| -l                | List of urls to run templates                         | nuclei -l urls.txt                                 |
-| -target           | Target to scan using templates                        | nuclei -target hxxps://example.com                 |
-| -t                | Templates input file/files to check across hosts      | nuclei -t git-core.yaml                            |
-| -t                | Templates input file/files to check across hosts      | nuclei -t nuclei-templates/cves/                   |
-| -nC               | Don't Use colors in output                            | nuclei -nC                                         |
-| -json             | Prints and write output in json format                | nuclei -json                                       |
-| -o                | File to save output result (optional)                 | nuclei -o output.txt                               |
-| -silent           | Show only found results in output                     | nuclei -silent                                     |
-| -retries          | Number of times to retry a failed request (default 1) | nuclei -retries 1                                  |
-| -timeout          | Seconds to wait before timeout (default 5)            | nuclei -timeout 5                                  |
-| -debug            | Allow debugging of request/responses.                 | nuclei -debug                                      |
-| -update-templates | Download and updates nuclei templates                 | nuclei -update-templates                           |
-| -update-directory | Directory for storing nuclei-templates(optional)      | nuclei -update-directory templates                 |
-| -v                | Shows verbose output of all sent requests             | nuclei -v                                          |
-| -version          | Show version of nuclei                                | nuclei -version                                    |
-| -proxy-url        | Proxy URL                                             | nuclei -proxy-url hxxp://127.0.0.1:8080            |
-| -proxy-socks-url  | Socks proxy  URL                                      | nuclei -proxy-socks-url socks5://127.0.0.1:8080    |
-| -H                | Custom Header                                         | nuclei -H "x-bug-bounty: hacker"                   |
+Please check our other open-source projects that might fit into your bug bounty workflow: [github.com/projectdiscovery](http://github.com/projectdiscovery), we also host daily [refresh of DNS data at Chaos](http://chaos.projectdiscovery.io).
+
+</td>
+</tr>
+</table>
+
+<table>
+<tr>
+<td>
+  
+**For pentesters:**
+
+Nuclei immensely improve how you approach security assessment by augmenting the manual repetitve processes. Consultancies are already converting their manual assessment steps with Nuclei, it allows them to run set of their custom assessment approach across thousands of hosts in an automated manner. 
+
+Pen-testers get the full power of our public templates and customization capabilities to speed-up their assessment process, and specifically with the regression cycle where you can easily verify the fix.
+
+- Easily create your compliance, standards suite (e.g. OWASP Top 10) checklist.
+- With capabilities like [fuzz](https://nuclei.projectdiscovery.io/templating-guide/#advance-fuzzing) and [workflows](https://nuclei.projectdiscovery.io/templating-guide/#workflows), complex manual steps and repetitive assessment can be easily automated with Nuclei.
+- Easy to re-test vulnerability-fix by just re-running the template.
+
+</td>
+</tr>
+</table>
 
 
-# Installation Instructions
+# For Developers and Organisations
 
+Nuclei is built with simplicity in mind, with the community backed templates by hundreds of security researchers, it allows you to stay updated with latest security threats using continuous Nuclei scanning on the hosts. It is designed to be easily integrated into regression tests cycle, to verify the fixes and  eliminate vulnerabilities from occuring in future.
 
-### From Binary
+- **CI/CD:** Engineers are already utilising Nuclei within their CI/CD pipeline, it allows them to constantly monitor their staging and production environments with customised templates.
+- **Continuous Regression Cycle:** With Nuclei, you can create your custom template on every new identified vulnerability and put into Nuclei engine to eliminate in the continuous regression cycle.
 
-The installation is easy. You can download the pre-built binaries for your platform from the [Releases](https://github.com/projectdiscovery/nuclei/releases/) page. Extract them using tar, move it to your `$PATH`and you're ready to go.
+We have [a discussion thread around this](https://github.com/projectdiscovery/nuclei-templates/discussions/693), there are already some bug bounty programs giving incentives to hackers on writing nuclei templates with every submission, that helps them to eliminate the vulnerability across all their assets, as well as to eliminate future risk in reappearing on productions. If you're interested in implementing it in your organisation, feel free to [reach out to us](mailto:[email protected]). We will be more than happy to help you in the getting started process, or you can also post into the [discussion thread for any help](https://github.com/projectdiscovery/nuclei-templates/discussions/693).
 
-```bash
-> tar -xzvf nuclei-linux-amd64.tar.gz
-> mv nuclei /usr/bin/
-> nuclei -h
-```
+<h3 align="center">
+  <img src="static/regression-with-nuclei.jpg" alt="regression-cycle-with-nuclei" width="1100px"></a>
+</h3>
 
-### From Source
+<h1 align="left">
+  <a href="https://github.com/projectdiscovery/nuclei-action"><img src="static/learn-more-button.png" width="170px" alt="Learn More"></a>
+</h1>
 
-nuclei requires go1.13+ to install successfully. Run the following command to get the repo - 
+### Resources
 
-```bash
-> GO111MODULE=on go get -u -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
-```
+- [Community Powered Scanning with Nuclei](https://blog.projectdiscovery.io/community-powered-scanning-with-nuclei/)
+- [Nuclei Unleashed - Quickly write complex exploits](https://blog.projectdiscovery.io/nuclei-unleashed-quickly-write-complex-exploits/)
+- [Nuclei - Fuzz all the things](https://blog.projectdiscovery.io/nuclei-fuzz-all-the-things/)
+- [Weaponizes nuclei Workflows to Pwn All the Things](https://medium.com/@dwisiswant0/weaponizes-nuclei-workflows-to-pwn-all-the-things-cd01223feb77) by [@dwisiswant0](https://github.com/dwisiswant0)
+- [How to Scan Continuously with Nuclei?](https://medium.com/@dwisiswant0/how-to-scan-continuously-with-nuclei-fcb7e9d8b8b9) by [@dwisiswant0](https://github.com/dwisiswant0)
 
-In order to update the tool, you can use -u flag with `go get` command.
+### Credits
 
-### Running in a Docker Container
+Thanks to all the amazing community [contributors for sending PRs](https://github.com/projectdiscovery/nuclei/graphs/contributors). Do also check out the below similar open-source projects that may fit in your workflow:
 
-You can use the [nuclei dockerhub image](https://hub.docker.com/r/projectdiscovery/nuclei). Simply run -
+[FFuF](https://github.com/ffuf/ffuf), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [Jaeles](https://github.com/jaeles-project/jaeles), [ChopChop](https://github.com/michelin/ChopChop)
 
-```bash
-> docker pull projectdiscovery/nuclei
-```
+### License
 
-- After downloading or building the container, run the following:
-```bash
-> docker run -it projectdiscovery/nuclei
-```
+Nuclei is distributed under [MIT License](https://github.com/projectdiscovery/nuclei/blob/master/LICENSE.md)
 
-For example, this will run the tool against all the hosts in `urls.txt` and output the results to your host file system:
-```bash
-> cat urls.txt | docker run -v /path-to-nuclei-templates:/go/src/app/ -i projectdiscovery/nuclei -t ./files/git-config.yaml > results.txt
-```
-Remember to change `/path-to-nuclei-templates` to the real path on your host file system.
-
-# Nuclei templates
-
-You can download or update the nuclei templates using `update-templates` flag. 
-
-```bash
-> nuclei -update-templates
-```
-
-or download it from [nuclei templates](https://github.com/projectdiscovery/nuclei-templates) Github project.
-
-```bash
-> git clone https://github.com/projectdiscovery/nuclei-templates.git
-```
-
-# Running nuclei
-
-### 1. Running nuclei with a single template. 
-
-This will run the tool against all the hosts in `urls.txt` and returns the matched results. 
-
-```bash
-> nuclei -l urls.txt -t files/git-core.yaml -o results.txt
-```
-
-You can also pass the list of hosts at standard input (STDIN). This allows for easy integration in automation pipelines.
-
-This will run the tool against all the hosts in `urls.txt` and returns the matched results. 
-
-```bash
-> cat urls.txt | nuclei -t files/git-core.yaml -o results.txt
-```
-
-### 2. Running nuclei with multiple templates. 
-
-This will run the tool against all the hosts in `urls.txt` with all the templates in the `path-to-templates` directory and returns the matched results. 
-
-```bash
-> nuclei -l urls.txt -t cves/ -o results.txt 
-```
-
-### 3. Automating nuclei with subfinder and any other similar tool.
-
-
-```bash
-> subfinder -d hackerone.com -silent | httpx -silent | nuclei -t cves/ -o results.txt
-```
-
--------
-
-**Please refer to [this guide](https://github.com/projectdiscovery/nuclei-templates/blob/master/GUIDE.md) to build your own custom templates.**
-
-
-# Thanks
-
-nuclei is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)** file for more details. Do also check out these similar awesome projects that may fit in your workflow:
-
-[https://portswigger.net/burp](https://portswigger.net/burp)</br>
-[https://github.com/ffuf/ffuf](https://github.com/ffuf/ffuf)</br>
-[https://github.com/jaeles-project/jaeles](https://github.com/jaeles-project/jaeles)</br>
-[https://github.com/ameenmaali/qsfuzz](https://github.com/ameenmaali/qsfuzz)</br>
-[https://github.com/proabiral/inception](https://github.com/proabiral/inception)</br>
-[https://github.com/hannob/snallygaster](https://github.com/hannob/snallygaster)</br>
+<h1 align="left">
+  <a href="https://discord.gg/KECAGdH"><img src="static/Join-Discord.png" width="380" alt="Join Discord"></a> <a href="https://nuclei.projectdiscovery.io"><img src="static/check-nuclei-documentation.png" width="380" alt="Check Nuclei Documentation"></a>
+</h1>

README_CN.md

@@ -0,0 +1,253 @@
+<h1 align="center">
+  <img src="static/nuclei-logo.png" alt="nuclei" width="200px"></a>
+  <br>
+</h1>
+
+[![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT)
+[![Go Report Card](https://goreportcard.com/badge/github.com/projectdiscovery/nuclei)](https://goreportcard.com/report/github.com/projectdiscovery/nuclei)
+[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectdiscovery/nuclei/issues)
+[![GitHub Release](https://img.shields.io/github/release/projectdiscovery/nuclei)](https://github.com/projectdiscovery/nuclei/releases)
+[![Follow on Twitter](https://img.shields.io/twitter/follow/pdnuclei.svg?logo=twitter)](https://twitter.com/pdnuclei)
+[![Docker Images](https://img.shields.io/docker/pulls/projectdiscovery/nuclei.svg)](https://hub.docker.com/r/projectdiscovery/nuclei)
+[![Chat on Discord](https://img.shields.io/discord/695645237418131507.svg?logo=discord)](https://discord.gg/KECAGdH)
+
+<p align="center">
+<a href="https://nuclei.projectdiscovery.io/templating-guide/" target="_blank"><img src="static/read-the-docs-button.png" height="42px"/></center></a>  <a href="https://github.com/projectdiscovery/nuclei-templates" target="_blank"><img src="static/download-templates-button.png" height="42px"/></a>
+</p>
+
+
+Nuclei是一个基于模板的、可配置攻击目标的扫描快速工具，同时还提供了强大的可扩展性和易用性。
+
+基于模板的nuclei被用来发送请求给目标，有着实现零误报的优点，并且可以对已知的路径进行有效的扫描。nuclei的主要用于在初期的探测阶段快速地对已知的且易于检测的漏洞或者CVE进行扫描。如果存在WAF的话，nuclei使用[retryablehttp-go库](https://github.com/projectdiscovery/retryablehttp-go)来处理各种错误，并且重新尝试攻击，这也是我们自定义功能的核心模块之一。
+
+我们也维护一个具有各个类型的模板的[开源库](https://github.com/projectdiscovery/nuclei-templates)，我们希望你也能贡献一些模板，贡献的这些模板最好是有效的，并且能允许每个人基于你的模板重新构建。查看[**nuclei.projectdiscovery.io**](https://nuclei.projectdiscovery.io/templating-guide/)这个网站去学习制作模板的入门知识。
+
+## 目录
+
+- [目录](#目录)
+- [功能](#功能)
+- [安装](#安装)
+- [Nuclei模板](#nuclei模板)
+- [用法](#用法)
+- [运行Nuclei](#运行nuclei)
+- [排除模板](#排除模板)
+- [致谢](#致谢)
+
+## 功能
+
+<h1 align="left">
+  <img src="static/nuclei-run.png" alt="nuclei" width="700px"></a>
+  <br>
+</h1>
+
+- 有着易于开发的、简单的、模块化的代码库
+- 使用了基于模板的引擎，运行速度极快，可以修改所以配置
+- 可以对特殊情况处理、重试、绕过等，可以绕过WAF
+- 智能匹配，零误报
+
+## 安装
+
+### 二进制文件安装
+
+二进制文件安装很简单，你可以从[Releases](https://github.com/projectdiscovery/nuclei/releases/)页面下载已经构建好的二进制文件压缩包，使用解压工具提取下载的压缩包，并将解压的文件夹移动到$PATH目录，就可以直接使用了。
+
+```sh
+Download latest binary from https://github.com/projectdiscovery/nuclei/releases
+
+▶ tar -xzvf nuclei-linux-amd64.tar.gz
+▶ mv nuclei /usr/local/bin/
+▶ nuclei -version
+```
+
+### 源码安装
+
+nuclei需要**go1.14+**才能成功安装，运行以下命令获取repo
+
+```sh
+▶ GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+```
+
+### GitHub安装
+
+```sh
+▶ git clone https://github.com/projectdiscovery/nuclei.git; cd nuclei/v2/cmd/nuclei/; go build; mv nuclei /usr/local/bin/; nuclei -version
+```
+
+## Nuclei模板
+
+你可以使用`update-templates`来下载和更新nuclei模板，该命令会从unclei的[模板库]()中下载最新版本，这个由社区来维护的库是可以随时使用的。
+
+```sh
+▶ nuclei -update-templates
+```
+
+此外您可以根据自己的工作情况或者需求编写模板，请参阅**nuclei[模板向导](https://nuclei.projectdiscovery.io/templating-guide/)去编写自定义模板**
+
+## 用法
+
+```sh
+nuclei -h
+```
+
+这条命令会显示帮助，以下是nuclei支持的所有命令
+
+|命令|描述|例子|
+|-----|-----|-----|
+|bulk-size|每个模板最大并行的主机数(默认25)|nuclei -bulk-size 25|
+|burp-collaborator-biid|使用burp-collaborator插件|nuclei -burp-collaborator-biid XXXX|
+|c|并行的最大模板数量(默认10)|nuclei -c 10|
+|l|对URL列表进行测试|nuclei -l urls.txt|
+|target|对目标进行测试|nuclei -target hxxps://example.com|
+|t|要检测的模板种类|nuclei -t git-core.yaml -t cves/|
+|no-color|输出不显示颜色|nuclei -no-color|
+|no-meta|不显示匹配的元数据|nuclei -no-meta|
+|json|输出为json格式|nuclei -json|
+|include-rr|json输出格式中包含请求和响应数据|nuclei -json -include-rr|
+|o|输出为文件|nuclei -o output.txt|
+|project|避免发送相同的请求|nuclei -project|
+|stats|使用进度条|nuclei -stats|
+|silent|只输出测试成功的结果|nuclei -silent|
+|retries|失败后的重试次数|nuclei -retries 1|
+|timeout|超时时间(默认为5秒)|nuclei -timeout 5|
+|trace-log|输出日志到log文件|nuclei -trace-log logs|
+|rate-limit|每秒最大请求数(默认150)|nuclei -rate-limit 150|
+|severity|根据严重性选择模板|nuclei  -severity critical,high|
+|stop-at-first-match|第一次匹配不要处理HTTP请求|nuclei -stop-at-frst-match|
+|exclude|排除的模板或文件夹|nuclei -exclude panels -exclude tokens|
+|debug|调试请求或者响应|nuclei -debug|
+|update-templates|下载或者升级模板|nuclei -update-templates|
+|update-directory|选择储存模板的目录(可选)|nuclei -update-directory templates|
+|tl|列出可用的模板|nuclei -tl|
+|templates-version|显示已安装的模板版本|nuclei -templates-version|
+|v|显示发送请求的详细信息|nuclei -v|
+|version|显示nuclei的版本号|nuclei -version|
+|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080|
+|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080|
+|random-agent|使用随机的UA|nuclei -random-agent|
+|H|自定义请求头|nuclei -H “x-bug-bounty:hacker”|
+
+## 运行Nuclei
+
+### 运行单个模板
+
+这将对`urls.txt`中所有的主机运行`git-core.yaml`并返回结果到`results.txt`
+
+```sh
+▶ nuclei -l urls.txt -t files/git-core.yaml -o results.txt
+```
+
+你可以轻松的通过管道使用标准的输入(STDIN)传递URL列表。
+
+```sh
+▶ cat urls.txt | nuclei -t files/git-core.yaml -o results.txt
+```
+
+💡 Nuclei可以接受如下列表的URL作为输入，例如以下URL：
+
+```
+https://test.some-site.com
+http://vuls-testing.com
+https://test.com
+```
+### 运行多个模板
+
+这将会对`urls.txt`中所有的URL运行`cves`和`files`模板检查，并返回输出到`results.txt`
+
+```sh
+▶ nuclei -l urls.txt -t cves/ -t files/ -o results.txt
+```
+
+### 使用subfinder运行
+
+```sh
+▶ subfinder -d hackerone.com -silent | httpx -silent | nuclei -t cves/ -o results.txt
+```
+
+### 在docker中运行
+
+你需要使用[nuclei的docker镜像](https://hub.docker.com/r/projectdiscovery/nuclei)来运行
+
+```sh
+▶ docker pull projectdiscovery/nuclei
+```
+
+下载并构建完成后，运行以下命令：
+
+```sh
+▶ docker run -it projectdiscovery/nuclei
+```
+
+这将会对`urls.txt`中的URL通过docker中的nuclei进行检测，并将结果输出到本机的`results.txt`文件的：
+
+```sh
+▶ cat urls.txt | docker run -v /path/to/nuclei-templates:/app/nuclei-templates -v /path/to/nuclei/config:/app/.nuclei-config.json -i projectdiscovery/nuclei -t /app/nuclei-templates/files/git-config.yaml > results.txt
+```
+记住更改的模板路径到本机
+
+### 速率限制
+
+Nuclei有多种控制速率的方法，包括并行执行多个模板、并行检查多个主机，以及使nuclei限制全局的请求速率，下面就是示例。
+
+- `-c`参数 => 限制并行的模板数
+- `-bulk-size`参数 => 限制并行的主机数
+- `-rate-limit`参数 => 全局速率限制
+
+如果你想快速扫描或者控制扫描，请使用这些标志并输入限制数，`速率限制`只保证控制传出的请求，与其他参数无关。
+
+### 排除模板
+
+[Nuclei模板](https://github.com/projectdiscovery/nuclei-templates)包含多种检查，其中有许多对攻击有用的检查，但并不是都有用的。如果您只希望扫描少数特定的模板或目录，则可以使用如下的参数筛选模板，或将某些模板排除。
+
+#### 排除模板运行
+
+我们不建议同时运行所有的nuclei模板，如果要排除模板，可以使用`exclude`参数来排除特定的目录或模板。
+
+```sh
+nuclei -l urls.txt -t nuclei-templates -exclude panels/ -exclude technologies -exclude files/wp-xmlrpc.yaml
+```
+
+注意：如上述示例中显示的那样，目录和特定模板都将不会扫描
+
+#### 基于严重性运行模板
+
+您可以根据模板的严重性运行模板，扫描时可以选择单个严重性或多个严重性。
+
+```sh
+nuclei -l urls.txt -t cves/ -severity critical,medium
+```
+
+上面的例子将运行`cves`目录下所有`严重`和`中等`的模板。
+
+```sh
+nuclei -l urls.txt -t panels/ -t technologies -severity info
+```
+
+上面的例子将运行`panels`和`technologies`目录下严重性标记为`info`的模板
+
+#### 使用`.nuclei-ignore`文件排除模板
+
+自从nuclei的[v2.1.1版本](https://github.com/projectdiscovery/nuclei/releases/tag/v2.1.1)以来，我们添加了对`.nuclei-ignore`文件的支持，该文件与`update-templates`参数一起使用，在 **.nuclei-ignore** 文件中，您可以定义要从nuclei扫描中排除的所有模板目录或者模板路径，要开始使用此功能，请确保使用`nuclei-update-templates`参数安装nuclei模板，现在可以根据`.nuclei-ignore`的文件来添加、更新、删除模板文件。
+
+```
+nano ~/nuclei-templates/.nuclei-ignore
+```
+
+默认的**nuclei忽略**列表可以访问[这里]((https://github.com/projectdiscovery/nuclei-templates/blob/master/.nuclei-ignore)，如果不想排除任何内容，只需要删除`.nuclei-ignore`文件。
+
+* * *
+
+### 📋 笔记
+
+- 进度条是实验性功能，在某些情况下可能无法使用。
+- 进度条不适用于工作流，因为是条件执行，所以不准确。
+
+
+## 致谢
+
+也要看看这些类似的好项目，或许它们也适合你：
+
+[Burp Suite](https://portswigger.net/burp), [FFuF](https://github.com/ffuf/ffuf), [Jaeles](https://github.com/jaeles-project/jaeles), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [ChopChop](https://github.com/michelin/ChopChop)
+
+--------
+
+Nuclei是由[projectdiscovery](https://projectdiscovery.io)团队用🖤制作的，当然社区也贡献了很多，通过 **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)**文件以获取更多详细信息。

THANKS.md

@@ -1 +1,17 @@
 
+### Thanks
+
+Many people have contributed to **nuclei** making it a wonderful tool either by making a pull request fixing some stuff. Here, we recognize these persons and thank them. 
+
+- All the contributors at [CONTRIBUTORS](https://github.com/projectdiscovery/nuclei/graphs/contributors) who made Nuclei what it is.
+
+We'd like to thank some additional amazing people, who contributed a lot in nuclei's journey - 
+
+- [@manuelbua](https://www.github.com/manuelbua) - Multiple feature additions including progress bar. 
+- [@dwisiswant0](https://www.github.com/dwisiswant0) - For fixing multiple bugs with HTTP Raw requests. 
+- [@toufik-airane](https://www.github.com/toufik-airane) - For adding binary matchers.
+- [@lc](https://www.github.com/lc) - For adding directory support for templates.
+- [@wdahlenburg](https://www.github.com/wdahlenburg) - For adding wildcard globing support to run templates.
+- [@Marmelatze](https://www.github.com/Marmelatze) - For adding support to save matched request/response in JSON format. 
+- [@ankh2054](https://www.github.com/ankh2054) - For adding description field for template information.
+- [@rotemreiss](https://www.github.com/rotemreiss) - For adding docker support.

go.mod

@@ -1,22 +0,0 @@
-module github.com/projectdiscovery/nuclei
-
-go 1.14
-
-require (
-	github.com/Knetic/govaluate v3.0.0+incompatible
-	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
-	github.com/blang/semver v3.5.1+incompatible
-	github.com/d5/tengo v1.24.8
-	github.com/d5/tengo/v2 v2.6.0
-	github.com/google/go-github v17.0.0+incompatible
-	github.com/google/go-github/v32 v32.0.0
-	github.com/json-iterator/go v1.1.10
-	github.com/karrick/godirwalk v1.15.6
-	github.com/miekg/dns v1.1.29
-	github.com/pkg/errors v0.9.1
-	github.com/projectdiscovery/gologger v1.0.0
-	github.com/projectdiscovery/retryabledns v1.0.4
-	github.com/projectdiscovery/retryablehttp-go v1.0.1
-	golang.org/x/net v0.0.0-20200528225125-3c3fba18258b
-	gopkg.in/yaml.v2 v2.3.0
-)

go.sum

@@ -1,71 +0,0 @@
-github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4=
-github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
-github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
-github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg=
-github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
-github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
-github.com/d5/tengo v1.24.8/go.mod h1:VhLq8Q2QFhCIJO3NhvM934qOThykMqJi9y9Siqd1ocQ=
-github.com/d5/tengo/v2 v2.6.0 h1:D0cJtpiBzaLJ/Smv6nnUc/LIfO46oKwDx85NZtIRNRI=
-github.com/d5/tengo/v2 v2.6.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
-github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
-github.com/google/go-github/v32 v32.0.0 h1:q74KVb22spUq0U5HqZ9VCYqQz8YRuOtL/39ZnfwO+NM=
-github.com/google/go-github/v32 v32.0.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
-github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
-github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
-github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA=
-github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
-github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
-github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s=
-github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
-github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60=
-github.com/projectdiscovery/retryabledns v1.0.4/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
-github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
-github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200528225125-3c3fba18258b h1:IYiJPiJfzktmDAO1HQiwjMjwjlYKHAL7KzeD544RJPs=
-golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

integration_tests/dns/basic.yaml

@@ -0,0 +1,17 @@
+id: basic-dns-example
+
+info:
+  name: Test DNS Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: A
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        words:
+          - "1.1.1.1"

integration_tests/http/get-headers.yaml

@@ -0,0 +1,17 @@
+id: basic-get-headers
+
+info:
+  name: Basic GET Headers Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    headers:
+      test: nuclei
+    matchers:
+      - type: word
+        words:
+          - "This is test headers matcher text"

integration_tests/http/get-query-string.yaml

@@ -0,0 +1,15 @@
+id: basic-get-querystring
+
+info:
+  name: Basic GET QueryString Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}?test=nuclei"
+    matchers:
+      - type: word
+        words:
+          - "This is test querystring matcher text"

integration_tests/http/get-redirects.yaml

@@ -0,0 +1,17 @@
+id: basic-get-redirects
+
+info:
+  name: Basic GET Redirects Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    redirects: true
+    max-redirects: 3
+    matchers:
+      - type: word
+        words:
+          - "This is test redirects matcher text"

integration_tests/http/get.yaml

@@ -0,0 +1,15 @@
+id: basic-get
+
+info:
+  name: Basic GET Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "This is test matcher text"

integration_tests/http/post-body.yaml

@@ -0,0 +1,19 @@
+id: basic-post-body
+
+info:
+  name: Basic POST Body Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: POST
+    path:
+      - "{{BaseURL}}"
+    headers: 
+      Content-Type: application/x-www-form-urlencoded
+      Content-Length: 1 # as long as there is a value, nuclei will auto-recalculate it.
+    body: username=test&password=nuclei
+    matchers:
+      - type: word
+        words:
+          - "This is test post-body matcher text"

integration_tests/http/post-json-body.yaml

@@ -0,0 +1,19 @@
+id: basic-post-json-body
+
+info:
+  name: Basic POST JSON Body Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: POST
+    path:
+      - "{{BaseURL}}"
+    headers: 
+      Content-Type: application/json
+      Content-Length: 1
+    body: '{"username":"test","password":"nuclei"}'
+    matchers:
+      - type: word
+        words:
+          - "This is test post-json-body matcher text"

integration_tests/http/post-multipart-body.yaml

@@ -0,0 +1,29 @@
+id: basic-post-multipart-body
+
+info:
+  name: Basic POST Multipart Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: POST
+    path:
+      - "{{BaseURL}}"
+    headers: 
+      Content-Type: multipart/form-data; boundary=d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8
+      Content-Length: 1
+    body: |
+      --d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8
+      Content-Disposition: form-data; name="username"; filename="username"
+      Content-Type: application/octet-stream
+
+      test
+      --d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8
+      Content-Disposition: form-data; name="password"
+
+      nuclei
+      --d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8--
+    matchers:
+      - type: word
+        words:
+          - "This is test post-multipart matcher text"

integration_tests/http/raw-cookie-reuse.yaml

@@ -0,0 +1,34 @@
+id: cookiereuse-raw-example
+info:
+  name: Test CookieReuse RAW Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        POST / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+
+        testing=parameter
+      - |
+        GET / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+        
+    cookie-reuse: true
+    matchers:
+      - type: word
+        words:
+          - "Test is test-cookie-reuse matcher text"

integration_tests/http/raw-dynamic-extractor.yaml

@@ -0,0 +1,43 @@
+id: dynamic-extractor-raw-example
+
+info:
+  name: Test Dynamic Extractor RAW Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        POST / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+
+        testing=parameter
+      - |
+        GET /?username={{randkey}} HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+    extractors:
+      - type: regex
+        name: randkey
+        part: body
+        group: 1
+        internal: true
+        regex:
+          - "Token: '([A-Za-z0-9]+)'"
+
+    cookie-reuse: true
+    matchers:
+      - type: word
+        words:
+          - "Test is test-dynamic-extractor-raw matcher text"

integration_tests/http/raw-get-query.yaml

@@ -0,0 +1,18 @@
+id: basic-raw-query-example
+
+info:
+  name: Test RAW GET Query Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        GET ?test=nuclei HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+
+    matchers:
+      - type: word
+        words:
+          - "Test is test raw-get-query-matcher text"

integration_tests/http/raw-get.yaml

@@ -0,0 +1,18 @@
+id: basic-raw-http-example
+
+info:
+  name: Test RAW GET Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        GET / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+
+    matchers:
+      - type: word
+        words:
+          - "Test is test raw-get-matcher text"

integration_tests/http/raw-payload.yaml

@@ -0,0 +1,29 @@
+id: payload-raw-example
+info:
+  name: Test RAW With Payload Template
+  author: pdteam
+  severity: info
+
+requests:
+  - payloads:
+      username:
+        - test
+      password:
+        - nuclei
+        - guest
+    attack: clusterbomb
+    raw:
+      - |
+        POST / HTTP/1.1
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5)
+        Host: {{Hostname}}
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+        another_header: {{base64('§password§')}}
+        Accept: */*
+
+        username=§username§&password={{password}}
+    matchers:
+      - type: word
+        words:
+          - "Test is raw-payload matcher text"

integration_tests/http/raw-post-body.yaml

@@ -0,0 +1,21 @@
+id: basic-raw-http-body-example
+
+info:
+  name: Test RAW POST Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        POST / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+
+        username=test&password=nuclei
+    matchers:
+      - type: word
+        words:
+          - "Test is test raw-post-body-matcher text"

integration_tests/http/raw-unsafe-request.yaml

@@ -0,0 +1,20 @@
+id: basic-raw-unsafe-request-example
+
+info:
+  name: Test RAW Unsafe Request Template
+  author: pd-team
+  severity: info
+
+requests:
+  - raw:
+      - |
+        GET / HTTP/1.1
+        Host: 
+        Content-Length: 4
+
+    unsafe: true
+    matchers-condition: and
+    matchers:
+      - type: word
+        words:
+          - "This is test-raw-unsafe request matcher."

integration_tests/http/request-condition-new.yaml

@@ -0,0 +1,19 @@
+id: request-condition-new
+
+info:
+  name: request-condition-new
+  author: pd-team
+  severity: info
+
+requests:
+  - method: GET
+    id: first
+    path:
+      - "{{BaseURL}}/200"
+  - method: GET
+    path:
+      - "{{BaseURL}}/400"
+    matchers:
+      - type: dsl
+        dsl:
+          - "first_status_code==200 && status_code==400"

integration_tests/http/request-condition.yaml

@@ -0,0 +1,18 @@
+id: request-condition
+
+info:
+  name: request-condition
+  author: pd-team
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}/200"
+      - "{{BaseURL}}/400"
+ 
+    req-condition: true
+    matchers:
+      - type: dsl
+        dsl:
+          - "status_code_1==200 && status_code_2==400"

integration_tests/network/basic.yaml

@@ -0,0 +1,18 @@
+id: basic-network-request
+
+info:
+  name: Basic Network Request
+  author: pdteam
+  severity: info
+
+network:
+  - host: 
+      - "{{Hostname}}"
+    inputs:
+      - data: "PING\r\n"
+    read-size: 4
+    matchers:
+      - type: word
+        part: data
+        words:
+          - "PONG"

integration_tests/network/hex.yaml

@@ -0,0 +1,22 @@
+id: hex-network-request
+
+info:
+  name: Hex Input Network Request
+  author: pdteam
+  severity: info
+
+network:
+  - host: 
+      - "{{Hostname}}"
+    inputs:
+      - data: "50494e47"
+        type: hex
+      - data: "\r\n"
+
+    read-size: 4
+    matchers:
+      - type: word
+        part: data
+        encoding: hex
+        words:
+          - "504f4e47"

integration_tests/network/multi-step.yaml

@@ -0,0 +1,32 @@
+id: multi-step
+
+info:
+  name: Multi-Step Network Template
+  author: pd-team
+  severity: info
+
+network:
+  - inputs:
+      - data: "FIRST"
+        read: 4
+        name: first
+      - data: "SECOND"
+        read: 4
+        name: second
+    host:
+      - "{{Hostname}}"
+    read-size: 6
+    matchers:
+      - type: word
+        part: first
+        words:
+          - "PING" 
+      - type: word
+        part: second
+        words:
+          - "PONG" 
+      - type: word
+        part: data
+        words:
+          - "NUCLEI" 
+    matchers-condition: and

integration_tests/run.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+cd ../v2/cmd/nuclei
+go build
+cp nuclei ../../../integration_tests/nuclei 
+cd ../integration-test
+go build
+cp integration-test ../../../integration_tests/integration-test 
+cd ../../../integration_tests
+./integration-test
+if [ $? -eq 0 ]
+then
+  exit 0
+else
+  exit 1
+fi

integration_tests/workflow/basic.yaml

@@ -0,0 +1,10 @@
+id: workflow-example
+
+info:
+  name: Test Workflow Template
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-1.yaml
+  - template: workflow/match-2.yaml

integration_tests/workflow/condition-matched.yaml

@@ -0,0 +1,11 @@
+id: condition-matched-workflow
+
+info:
+  name: Condition Matched Workflow
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-1.yaml
+    subtemplates:
+      - template: workflow/match-2.yaml

integration_tests/workflow/condition-unmatched.yaml

@@ -0,0 +1,11 @@
+id: condition-unmatched-workflow
+
+info:
+  name: Condition UnMatched Workflow
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/nomatch-1.yaml
+    subtemplates:
+      - template: workflow/match-2.yaml

integration_tests/workflow/match-1.yaml

@@ -0,0 +1,15 @@
+id: basic-get
+
+info:
+  name: Basic GET Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "This is test matcher text"

integration_tests/workflow/match-2.yaml

@@ -0,0 +1,16 @@
+id: basic-get-another
+
+info:
+  name: Basic Another GET Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        name: test-matcher
+        words:
+          - "This is test matcher text"

integration_tests/workflow/matcher-name.yaml

@@ -0,0 +1,13 @@
+id: matcher-name-workflow
+
+info:
+  name: Matcher Name Workflow
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-2.yaml
+    matchers: 
+      - name: test-matcher
+        subtemplates:
+          - template: workflow/match-1.yaml

integration_tests/workflow/nomatch-1.yaml

@@ -0,0 +1,15 @@
+id: basic-get-nomatch
+
+info:
+  name: Basic GET Request NoMatch
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "Random"

static/regression-cycle.md

@@ -0,0 +1 @@
+

v2/.goreleaser.yml

@@ -1,6 +1,10 @@
+before:
+  hooks:
+    - go mod tidy
+
 builds:
     - binary: nuclei
-      main: v2/cmd/nuclei/main.go
+      main: cmd/nuclei/main.go
       goos:
         - linux
         - windows
@@ -18,4 +22,4 @@ archives:
           darwin: macOS
       format_overrides:
           - goos: windows
-            format: zip
+            format: zip

v2/cmd/integration-test/dns.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var dnsTestCases = map[string]testutils.TestCase{
+	"dns/basic.yaml": &dnsBasic{},
+}
+
+type dnsBasic struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *dnsBasic) Execute(filePath string) error {
+	var routerErr error
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, "one.one.one.one", debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/integration-test/http.go

@@ -0,0 +1,524 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"strings"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var httpTestcases = map[string]testutils.TestCase{
+	"http/get-headers.yaml":           &httpGetHeaders{},
+	"http/get-query-string.yaml":      &httpGetQueryString{},
+	"http/get-redirects.yaml":         &httpGetRedirects{},
+	"http/get.yaml":                   &httpGet{},
+	"http/post-body.yaml":             &httpPostBody{},
+	"http/post-json-body.yaml":        &httpPostJSONBody{},
+	"http/post-multipart-body.yaml":   &httpPostMultipartBody{},
+	"http/raw-cookie-reuse.yaml":      &httpRawCookieReuse{},
+	"http/raw-dynamic-extractor.yaml": &httpRawDynamicExtractor{},
+	"http/raw-get-query.yaml":         &httpRawGetQuery{},
+	"http/raw-get.yaml":               &httpRawGet{},
+	"http/raw-payload.yaml":           &httpRawPayload{},
+	"http/raw-post-body.yaml":         &httpRawPostBody{},
+	"http/raw-unsafe-request.yaml":    &httpRawUnsafeRequest{},
+	"http/request-condition.yaml":     &httpRequestCondition{},
+	"http/request-condition-new.yaml": &httpRequestCondition{},
+}
+
+func httpDebugRequestDump(r *http.Request) {
+	if debug {
+		if dump, err := httputil.DumpRequest(r, true); err == nil {
+			fmt.Printf("\nRequest dump: \n%s\n\n", string(dump))
+		}
+	}
+}
+
+type httpGetHeaders struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGetHeaders) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.Header.Get("test"), "nuclei") {
+			fmt.Fprintf(w, "This is test headers matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpGetQueryString struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGetQueryString) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") {
+			fmt.Fprintf(w, "This is test querystring matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpGetRedirects struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGetRedirects) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		http.Redirect(w, r, "/redirected", http.StatusFound)
+	}))
+	router.GET("/redirected", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test redirects matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpGet struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGet) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpPostBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpPostBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("username"), "test") && strings.EqualFold(r.Form.Get("password"), "nuclei") {
+			fmt.Fprintf(w, "This is test post-body matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpPostJSONBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpPostJSONBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+
+		type doc struct {
+			Username string `json:"username"`
+			Password string `json:"password"`
+		}
+		obj := &doc{}
+		if err := json.NewDecoder(r.Body).Decode(obj); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(obj.Username, "test") && strings.EqualFold(obj.Password, "nuclei") {
+			fmt.Fprintf(w, "This is test post-json-body matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpPostMultipartBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpPostMultipartBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseMultipartForm(1 * 1024); err != nil {
+			routerErr = err
+			return
+		}
+		password, ok := r.MultipartForm.Value["password"]
+		if !ok || len(password) != 1 {
+			routerErr = errors.New("no password in request")
+			return
+		}
+		file := r.MultipartForm.File["username"]
+		if len(file) != 1 {
+			routerErr = errors.New("no file in request")
+			return
+		}
+		if strings.EqualFold(password[0], "nuclei") && strings.EqualFold(file[0].Filename, "username") {
+			fmt.Fprintf(w, "This is test post-multipart matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawDynamicExtractor struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawDynamicExtractor) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("testing"), "parameter") {
+			fmt.Fprintf(w, "Token: 'nuclei'")
+		}
+	}))
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.URL.Query().Get("username"), "nuclei") {
+			fmt.Fprintf(w, "Test is test-dynamic-extractor-raw matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawGetQuery struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawGetQuery) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") {
+			fmt.Fprintf(w, "Test is test raw-get-query-matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawGet struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawGet) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+
+		fmt.Fprintf(w, "Test is test raw-get-matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawPayload struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawPayload) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if !(strings.EqualFold(r.Header.Get("another_header"), "bnVjbGVp") || strings.EqualFold(r.Header.Get("another_header"), "Z3Vlc3Q=")) {
+			return
+		}
+		if strings.EqualFold(r.Form.Get("username"), "test") && (strings.EqualFold(r.Form.Get("password"), "nuclei") || strings.EqualFold(r.Form.Get("password"), "guest")) {
+			fmt.Fprintf(w, "Test is raw-payload matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 2 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawPostBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawPostBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("username"), "test") && strings.EqualFold(r.Form.Get("password"), "nuclei") {
+			fmt.Fprintf(w, "Test is test raw-post-body-matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawCookieReuse struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawCookieReuse) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("testing"), "parameter") {
+			http.SetCookie(w, &http.Cookie{Name: "nuclei", Value: "test"})
+		}
+	}))
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		cookie, err := r.Cookie("nuclei")
+		if err != nil {
+			routerErr = err
+			return
+		}
+
+		if strings.EqualFold(cookie.Value, "test") {
+			fmt.Fprintf(w, "Test is test-cookie-reuse matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawUnsafeRequest struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawUnsafeRequest) Execute(filePath string) error {
+	var routerErr error
+
+	ts := testutils.NewTCPServer(func(conn net.Conn) {
+		defer conn.Close()
+
+		_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 40\r\nContent-Type: text/plain; charset=utf-8\r\nDate: Thu, 25 Feb 2021 17:17:28 GMT\r\n\r\nThis is test-raw-unsafe request matcher.\r\n"))
+	})
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, "http://"+ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRequestCondition struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRequestCondition) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.GET("/200", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		w.WriteHeader(200)
+	}))
+	router.GET("/400", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		w.WriteHeader(400)
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/integration-test/integration-test.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var (
+	debug      = os.Getenv("DEBUG") == "true"
+	customTest = os.Getenv("TEST")
+	protocol   = os.Getenv("PROTO")
+)
+
+func main() {
+	success := aurora.Green("[✓]").String()
+	failed := aurora.Red("[✘]").String()
+
+	protocolTests := map[string]map[string]testutils.TestCase{
+		"http":     httpTestcases,
+		"network":  networkTestcases,
+		"dns":      dnsTestCases,
+		"workflow": workflowTestcases,
+	}
+	for proto, tests := range protocolTests {
+		if protocol == "" || protocol == proto {
+			fmt.Printf("Running test cases for \"%s\" protocol\n", aurora.Blue(proto))
+
+			for file, test := range tests {
+				if customTest != "" && !strings.Contains(file, customTest) {
+					continue // only run tests user asked
+				}
+				err := test.Execute(file)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, file, err)
+					os.Exit(1)
+				} else {
+					fmt.Printf("%s Test \"%s\" passed!\n", success, file)
+				}
+			}
+		}
+	}
+}
+
+func errIncorrectResultsCount(results []string) error {
+	return fmt.Errorf("incorrect number of results %s", strings.Join(results, "\n\t"))
+}

v2/cmd/integration-test/network.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"net"
+
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var networkTestcases = map[string]testutils.TestCase{
+	"network/basic.yaml":      &networkBasic{},
+	"network/hex.yaml":        &networkBasic{},
+	"network/multi-step.yaml": &networkMultiStep{},
+}
+
+type networkBasic struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *networkBasic) Execute(filePath string) error {
+	var routerErr error
+
+	ts := testutils.NewTCPServer(func(conn net.Conn) {
+		defer conn.Close()
+
+		data := make([]byte, 4)
+		if _, err := conn.Read(data); err != nil {
+			routerErr = err
+			return
+		}
+		if string(data) == "PING" {
+			_, _ = conn.Write([]byte("PONG"))
+		}
+	})
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type networkMultiStep struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *networkMultiStep) Execute(filePath string) error {
+	var routerErr error
+
+	ts := testutils.NewTCPServer(func(conn net.Conn) {
+		defer conn.Close()
+
+		data := make([]byte, 5)
+		if _, err := conn.Read(data); err != nil {
+			routerErr = err
+			return
+		}
+		if string(data) == "FIRST" {
+			_, _ = conn.Write([]byte("PING"))
+		}
+
+		data = make([]byte, 6)
+		if _, err := conn.Read(data); err != nil {
+			routerErr = err
+			return
+		}
+		if string(data) == "SECOND" {
+			_, _ = conn.Write([]byte("PONG"))
+		}
+		_, _ = conn.Write([]byte("NUCLEI"))
+	})
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/integration-test/workflow.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var workflowTestcases = map[string]testutils.TestCase{
+	"workflow/basic.yaml":               &workflowBasic{},
+	"workflow/condition-matched.yaml":   &workflowConditionMatched{},
+	"workflow/condition-unmatched.yaml": &workflowConditionUnmatch{},
+	"workflow/matcher-name.yaml":        &workflowMatcherName{},
+}
+
+type workflowBasic struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowBasic) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 2 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type workflowConditionMatched struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowConditionMatched) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type workflowConditionUnmatch struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowConditionUnmatch) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 0 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type workflowMatcherName struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowMatcherName) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/nuclei/issue-tracker-config.yaml

@@ -0,0 +1,47 @@
+#allow-list:
+#  severity: high,critical
+#deny-list:
+#  severity: low
+
+# github contains configuration options for github issue tracker
+#github: 
+#  # base-url is the optional self-hosted github application url
+#  base-url: ""
+#  # username is the username of the github user
+#  username: ""
+#  # owner is the owner name of the repository for issues.
+#  owner: ""
+#  # token is the token for github account.
+#  token: ""
+#  # project-name is the name of the repository.
+#  project-name: ""
+#  # issue-label is the label of the created issue type
+#  issue-label: ""
+
+# gitlab contains configuration options for gitlab issue tracker
+#gitlab: 
+#  # base-url is the optional self-hosted gitlab application url
+#  base-url: ""
+#  # username is the username of the gitlab user
+#  username: ""
+#  # token is the token for gitlab account.
+#  token: ""
+#  # project-id is the ID of the repository.
+#  project-id: ""
+#  # issue-label is the label of the created issue type
+#  issue-label: ""
+
+# jira contains configuration options for jira issue tracker
+#jira: 
+#  # URL is the jira application url
+#  url: ""
+#  # account-id is the account-id of the jira user
+#  account-id: ""
+#  # email is the email of the user for jira instance
+#  email: ""
+#  # token is the token for jira instance.
+#  token: ""
+#  # project-name is the name of the project.
+#  project-name: ""
+#  # issue-type is the name of the created issue type
+#  issue-type: ""

v2/cmd/nuclei/main.go

@@ -1,19 +1,96 @@
 package main
 
 import (
+	"os"
+	"path"
+
+	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/internal/runner"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+var (
+	cfgFile string
+	options = &types.Options{}
 )
 
 func main() {
-	// Parse the command line flags and read config files
-	options := runner.ParseOptions()
+	readConfig()
 
-	runner, err := runner.New(options)
+	runner.ParseOptions(options)
+
+	nucleiRunner, err := runner.New(options)
 	if err != nil {
-		gologger.Fatalf("Could not create runner: %s\n", err)
+		gologger.Fatal().Msgf("Could not create runner: %s\n", err)
+	}
+	nucleiRunner.RunEnumeration()
+}
+
+func readConfig() {
+	home, _ := os.UserHomeDir()
+	templatesDirectory := path.Join(home, "nuclei-templates")
+
+	set := goflags.New()
+	set.SetDescription(`Nuclei is a fast tool for configurable targeted scanning 
+based on templates offering massive extensibility and ease of use.`)
+	set.StringVar(&cfgFile, "config", "", "Nuclei configuration file")
+	set.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port")
+	set.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on")
+	set.StringVarP(&options.Target, "target", "u", "", "URL to scan with nuclei")
+	set.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "Templates to run, supports single and multiple templates using directory.")
+	set.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "Workflows to run for nuclei")
+	set.StringSliceVarP(&options.ExcludedTemplates, "exclude", "et", []string{}, "Templates to exclude, supports single and multiple templates using directory.")
+	set.StringSliceVarP(&options.Severity, "severity", "impact", []string{}, "Templates to run based on severity, supports single and multiple severity.")
+	set.StringVarP(&options.Targets, "list", "l", "", "List of URLs to run templates on")
+	set.StringVarP(&options.Output, "output", "o", "", "File to write output to (optional)")
+	set.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server")
+	set.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the proxy socks server")
+	set.BoolVar(&options.Silent, "silent", false, "Show only results in output")
+	set.BoolVar(&options.Version, "version", false, "Show version of nuclei")
+	set.BoolVarP(&options.Verbose, "verbose", "v", false, "Show verbose output")
+	set.BoolVarP(&options.NoColor, "no-color", "nc", false, "Disable colors in output")
+	set.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout")
+	set.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
+	set.BoolVarP(&options.RandomAgent, "random-agent", "ra", false, "Use randomly selected HTTP User-Agent header value")
+	set.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "Custom Header.")
+	set.BoolVar(&options.Debug, "debug", false, "Debugging request and responses")
+	set.BoolVar(&options.DebugRequests, "debug-req", false, "Debugging request")
+	set.BoolVar(&options.DebugResponse, "debug-resp", false, "Debugging response")
+	set.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "Download / updates nuclei community templates")
+	set.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log")
+	set.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "Directory storing nuclei-templates")
+	set.BoolVar(&options.JSON, "json", false, "Write json output to files")
+	set.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "Write requests/responses for matches in JSON output")
+	set.BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan")
+	set.BoolVar(&options.TemplateList, "tl", false, "List available templates")
+	set.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "Maximum requests to send per second")
+	set.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "Stop processing http requests at first match (this may break template/workflow logic)")
+	set.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "Maximum Number of hosts analyzed in parallel per template")
+	set.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "Maximum Number of templates executed in parallel")
+	set.BoolVar(&options.Project, "project", false, "Use a project folder to avoid sending same request multiple times")
+	set.StringVar(&options.ProjectPath, "project-path", "", "Use a user defined project folder, temporary folder is used if not specified but enabled")
+	set.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "Don't display metadata for the matches")
+	set.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "Shows the installed nuclei-templates version")
+	set.BoolVar(&options.OfflineHTTP, "passive", false, "Enable Passive HTTP response processing mode")
+	set.StringVarP(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "biid", "", "Burp Collaborator BIID")
+	set.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "Nuclei Reporting Module configuration file")
+	set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database (Always use this to persistent report data)")
+	set.StringSliceVar(&options.Tags, "tags", []string{}, "Tags to execute templates for")
+	set.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "Exclude templates with the provided tags")
+	set.StringVarP(&options.ResolversFile, "resolvers", "r", "", "File containing resolver list for nuclei")
+	set.BoolVar(&options.Headless, "headless", false, "Enable headless browser based templates support")
+	set.BoolVar(&options.ShowBrowser, "show-browser", false, "Show the browser on the screen")
+	set.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "Number of seconds between each stats line")
+	set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback")
+	set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless")
+	set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates")
+	set.StringVarP(&options.DiskExportDirectory, "disk-export", "de", "", "Directory on disk to export reports in markdown to")
+	_ = set.Parse()
+
+	if cfgFile != "" {
+		if err := set.MergeConfigFile(cfgFile); err != nil {
+			gologger.Fatal().Msgf("Could not read config: %s\n", err)
+		}
 	}
-
-	runner.RunEnumeration()
-	runner.Close()
 }

v2/go.mod

@@ -1,22 +1,58 @@
 module github.com/projectdiscovery/nuclei/v2
 
-go 1.14
+go 1.15
 
 require (
 	github.com/Knetic/govaluate v3.0.0+incompatible
-	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
+	github.com/andygrunwald/go-jira v1.13.0
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/d5/tengo v1.24.8
-	github.com/d5/tengo/v2 v2.6.0
+	github.com/corpix/uarand v0.1.1
+	github.com/fatih/structs v1.1.0 // indirect
+	github.com/go-rod/rod v0.91.1
+	github.com/golang/protobuf v1.4.3 // indirect
 	github.com/google/go-github v17.0.0+incompatible
-	github.com/google/go-github/v32 v32.0.0
+	github.com/google/go-github/v32 v32.1.0
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
 	github.com/json-iterator/go v1.1.10
-	github.com/karrick/godirwalk v1.15.6
-	github.com/miekg/dns v1.1.29
+	github.com/julienschmidt/httprouter v1.3.0
+	github.com/karrick/godirwalk v1.16.1
+	github.com/logrusorgru/aurora v2.0.3+incompatible
+	github.com/mattn/go-runewidth v0.0.10 // indirect
+	github.com/miekg/dns v1.1.38
+	github.com/mitchellh/go-ps v1.0.0
+	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/projectdiscovery/gologger v1.0.0
-	github.com/projectdiscovery/retryabledns v1.0.4
+	github.com/projectdiscovery/clistats v0.0.8
+	github.com/projectdiscovery/collaborator v0.0.2
+	github.com/projectdiscovery/fastdialer v0.0.8
+	github.com/projectdiscovery/goflags v0.0.4
+	github.com/projectdiscovery/gologger v1.1.4
+	github.com/projectdiscovery/hmap v0.0.1
+	github.com/projectdiscovery/rawhttp v0.0.6
+	github.com/projectdiscovery/retryabledns v1.0.10
 	github.com/projectdiscovery/retryablehttp-go v1.0.1
-	golang.org/x/net v0.0.0-20200528225125-3c3fba18258b
-	gopkg.in/yaml.v2 v2.3.0
+	github.com/remeh/sizedwaitgroup v1.0.0
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/rs/xid v1.2.1
+	github.com/segmentio/ksuid v1.0.3
+	github.com/spaolacci/murmur3 v1.1.0
+	github.com/spf13/cast v1.3.1
+	github.com/stretchr/testify v1.7.0
+	github.com/syndtr/goleveldb v1.0.0
+	github.com/trivago/tgo v1.0.7 // indirect
+	github.com/valyala/fasttemplate v1.2.1
+	github.com/xanzy/go-gitlab v0.44.0
+	go.uber.org/atomic v1.7.0
+	go.uber.org/multierr v1.6.0
+	go.uber.org/ratelimit v0.1.0
+	golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect
+	golang.org/x/net v0.0.0-20210119194325-5f4716e94777
+	golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
+	golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b // indirect
+	golang.org/x/text v0.3.4 // indirect
+	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	gopkg.in/yaml.v2 v2.4.0
 )

v2/go.sum

@@ -1,69 +1,557 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
+github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
+github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
+github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE=
+github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
-github.com/d5/tengo v1.24.8/go.mod h1:VhLq8Q2QFhCIJO3NhvM934qOThykMqJi9y9Siqd1ocQ=
-github.com/d5/tengo/v2 v2.6.0 h1:D0cJtpiBzaLJ/Smv6nnUc/LIfO46oKwDx85NZtIRNRI=
-github.com/d5/tengo/v2 v2.6.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
+github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4=
+github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
+github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
+github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
+github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-rod/rod v0.91.1 h1:7xIlC/bXCXosZqZUl2x6GVB8tv4yMQ4W/ZVdGVa1qYI=
+github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
+github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
-github.com/google/go-github/v32 v32.0.0 h1:q74KVb22spUq0U5HqZ9VCYqQz8YRuOtL/39ZnfwO+NM=
-github.com/google/go-github/v32 v32.0.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
+github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
+github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
+github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
+github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA=
-github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
-github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
+github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
+github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
+github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw=
+github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
+github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s=
-github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
-github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60=
-github.com/projectdiscovery/retryabledns v1.0.4/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
+github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20=
+github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
+github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5zmiJ6tI7+6mB3s=
+github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
+github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0=
+github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4=
+github.com/projectdiscovery/goflags v0.0.4 h1:fWKLMAr3KmPlZxE1b54pfei+vGIUJn9q6aM7woZIbCY=
+github.com/projectdiscovery/goflags v0.0.4/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
+github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
+github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
+github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
+github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
+github.com/projectdiscovery/rawhttp v0.0.6 h1:HbgPB1eKXQVV5F9sq0Uxflm95spWFyZYD8dgFpeOC9M=
+github.com/projectdiscovery/rawhttp v0.0.6/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
+github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
+github.com/projectdiscovery/retryabledns v1.0.10 h1:xJZ2aKoqrNg/OZEw1+4+QIOH40V/WkZDYY1ZZc+uphE=
+github.com/projectdiscovery/retryabledns v1.0.10/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
 github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
+github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY=
+github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
+github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/trivago/tgo v1.0.1/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
+github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM=
+github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
+github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
+github.com/xanzy/go-gitlab v0.44.0 h1:cEiGhqu7EpFGuei2a2etAwB+x6403E5CvpLn35y+GPs=
+github.com/xanzy/go-gitlab v0.44.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
+github.com/ysmood/goob v0.3.0 h1:XZ51cZJ4W3WCoCiUktixzMIQF86W7G5VFL4QQ/Q2uS0=
+github.com/ysmood/goob v0.3.0/go.mod h1:S3lq113Y91y1UBf1wj1pFOxeahvfKkCk6mTWTWbDdWs=
+github.com/ysmood/got v0.9.3 h1:qx51X49jL/WAiqZzPTkPZ0zp5pTmrWJa4zYFTYo0gHI=
+github.com/ysmood/got v0.9.3/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
+github.com/ysmood/gotrace v0.2.0 h1:IkTC6rJREwXSaG8yWK+NFwIJGIsxA1DjC6/gxYyQttE=
+github.com/ysmood/gotrace v0.2.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
+github.com/ysmood/gson v0.6.3 h1:4cU+5oOdsyundXHy00t99H0rLXLthuseD3x6W+xmCiU=
+github.com/ysmood/gson v0.6.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
+github.com/ysmood/leakless v0.6.12 h1:XxtRYl97bJklfv4BZVdyGnd/y42p6w8lu1hUzfCkT4M=
+github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
+go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df h1:y7QZzfUiTwWam+xBn29Ulb8CBwVN5UdzmMDavl9Whlw=
+golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200528225125-3c3fba18258b h1:IYiJPiJfzktmDAO1HQiwjMjwjlYKHAL7KzeD544RJPs=
-golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 h1:5vD4XjIc0X5+kHZjx4UecYdjA6mJo+XXNoaW0EjU5Os=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b h1:lAZ0/chPUDWwjqosYR0X4M490zQhMsiJ4K3DbA7o+3g=
+golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE=
+golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

v2/internal/collaborator/collaborator.go

@@ -0,0 +1,72 @@
+package collaborator
+
+import (
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/projectdiscovery/collaborator"
+)
+
+var (
+	// PollSeconds is the seconds to poll at.
+	PollSeconds = 5
+	// DefaultMaxBufferLimit is the default request buffer limit
+	DefaultMaxBufferLimit = 150
+	// DefaultPollInterval is the default poll interval for burp collabortor polling.
+	DefaultPollInterval time.Duration = time.Second * time.Duration(PollSeconds)
+	// DefaultCollaborator is the default burp collaborator instance
+	DefaultCollaborator = &Collaborator{Collab: collaborator.NewBurpCollaborator()}
+)
+
+// Collaborator is a client for recording burp collaborator interactions
+type Collaborator struct {
+	sync.RWMutex
+	options *Options // unused
+	Collab  *collaborator.BurpCollaborator
+}
+
+// Options contains configuration options for collaborator client
+type Options struct {
+	BIID           string
+	PollInterval   time.Duration
+	MaxBufferLimit int
+}
+
+// New creates a new collaborator client
+func New(options *Options) *Collaborator {
+	collab := collaborator.NewBurpCollaborator()
+	collab.AddBIID(options.BIID)
+	collab.MaxBufferLimit = options.MaxBufferLimit
+	return &Collaborator{Collab: collab, options: options}
+}
+
+// Poll initiates collaborator polling if any BIIDs were provided
+func (b *Collaborator) Poll() {
+	// if no valid biids were provided just return
+	if len(b.Collab.BIIDs) > 0 {
+		go b.Collab.PollEach(DefaultPollInterval)
+	}
+}
+
+// Has checks if a collabrator hit was found for a URL
+func (b *Collaborator) Has(s string) bool {
+	for _, r := range b.Collab.RespBuffer {
+		for i := 0; i < len(r.Responses); i++ {
+			// search in dns - http - smtp
+			b.RLock()
+			found := strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) ||
+				strings.Contains(r.Responses[i].Data.RequestDecoded, s) ||
+				strings.Contains(r.Responses[i].Data.MessageDecoded, s)
+			b.RUnlock()
+
+			if found {
+				b.Lock()
+				r.Responses = append(r.Responses[:i], r.Responses[i+1:]...)
+				b.Unlock()
+				return true
+			}
+		}
+	}
+	return false
+}

v2/internal/colorizer/colorizer.go

@@ -0,0 +1,24 @@
+package colorizer
+
+import "github.com/logrusorgru/aurora"
+
+// Colorizer returns a colorized severity printer
+type Colorizer struct {
+	Data map[string]string
+}
+
+const (
+	fgOrange uint8 = 208
+)
+
+// New returns a new severity based colorizer
+func New(colorizer aurora.Aurora) *Colorizer {
+	severityMap := map[string]string{
+		"info":     colorizer.Blue("info").String(),
+		"low":      colorizer.Green("low").String(),
+		"medium":   colorizer.Yellow("medium").String(),
+		"high":     colorizer.Index(fgOrange, "high").String(),
+		"critical": colorizer.Red("critical").String(),
+	}
+	return &Colorizer{Data: severityMap}
+}

v2/internal/runner/banner.go

@@ -3,21 +3,21 @@ package runner
 import "github.com/projectdiscovery/gologger"
 
 const banner = `
-                       __     _ 
+                       __     _
      ____  __  _______/ /__  (_)
-    / __ \/ / / / ___/ / _ \/ / 
-   / / / / /_/ / /__/ /  __/ /  
-  /_/ /_/\__,_/\___/_/\___/_/   v2									  
+    / __ \/ / / / ___/ / _ \/ /
+   / / / / /_/ / /__/ /  __/ /
+  /_/ /_/\__,_/\___/_/\___/_/   v2.3.2
 `
 
 // Version is the current version of nuclei
-const Version = `2.0.2`
+const Version = `2.3.2`
 
 // showBanner is used to show the banner to the user
 func showBanner() {
-	gologger.Printf("%s\n", banner)
-	gologger.Printf("\t\tprojectdiscovery.io\n\n")
+	gologger.Print().Msgf("%s\n", banner)
+	gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n")
 
-	gologger.Labelf("Use with caution. You are responsible for your actions\n")
-	gologger.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n")
+	gologger.Warning().Msgf("Use with caution. You are responsible for your actions\n")
+	gologger.Warning().Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
 }

v2/internal/runner/config.go

@@ -0,0 +1,130 @@
+package runner
+
+import (
+	"bufio"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+)
+
+// nucleiConfig contains some configuration options for nuclei
+type nucleiConfig struct {
+	TemplatesDirectory string    `json:"templates-directory,omitempty"`
+	CurrentVersion     string    `json:"current-version,omitempty"`
+	LastChecked        time.Time `json:"last-checked,omitempty"`
+	IgnoreURL          string    `json:"ignore-url,omitempty"`
+	NucleiVersion      string    `json:"nuclei-version,omitempty"`
+	LastCheckedIgnore  time.Time `json:"last-checked-ignore,omitempty"`
+	// IgnorePaths ignores all the paths listed unless specified manually
+	IgnorePaths []string `json:"ignore-paths,omitempty"`
+}
+
+// nucleiConfigFilename is the filename of nuclei configuration file.
+const nucleiConfigFilename = ".templates-config.json"
+
+var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
+
+// readConfiguration reads the nuclei configuration file from disk.
+func readConfiguration() (*nucleiConfig, error) {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return nil, err
+	}
+	configDir := path.Join(home, "/.config", "/nuclei")
+	_ = os.MkdirAll(configDir, os.ModePerm)
+
+	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
+	file, err := os.Open(templatesConfigFile)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	config := &nucleiConfig{}
+	err = jsoniter.NewDecoder(file).Decode(config)
+	if err != nil {
+		return nil, err
+	}
+	return config, nil
+}
+
+// readConfiguration reads the nuclei configuration file from disk.
+func (r *Runner) writeConfiguration(config *nucleiConfig) error {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return err
+	}
+	configDir := path.Join(home, "/.config", "/nuclei")
+	_ = os.MkdirAll(configDir, os.ModePerm)
+
+	if config.IgnoreURL == "" {
+		config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
+	}
+	config.LastChecked = time.Now()
+	config.LastCheckedIgnore = time.Now()
+	config.NucleiVersion = Version
+	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
+	file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	err = jsoniter.NewEncoder(file).Encode(config)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+const nucleiIgnoreFile = ".nuclei-ignore"
+
+// readNucleiIgnoreFile reads the nuclei ignore file marking it in map
+func (r *Runner) readNucleiIgnoreFile() {
+	file, err := os.Open(r.getIgnoreFilePath())
+	if err != nil {
+		return
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text == "" {
+			continue
+		}
+		if strings.HasPrefix(text, "#") {
+			continue
+		}
+		r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, text)
+	}
+}
+
+// getIgnoreFilePath returns the ignore file path for the runner
+func (r *Runner) getIgnoreFilePath() string {
+	var defIgnoreFilePath string
+
+	home, err := os.UserHomeDir()
+	if err == nil {
+		configDir := path.Join(home, "/.config", "/nuclei")
+		_ = os.MkdirAll(configDir, os.ModePerm)
+
+		defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile)
+	}
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return defIgnoreFilePath
+	}
+	cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile)
+
+	cwdIfpInfo, err := os.Stat(cwdIgnoreFilePath)
+	if os.IsNotExist(err) || cwdIfpInfo.IsDir() {
+		return defIgnoreFilePath
+	}
+	return cwdIgnoreFilePath
+}

v2/internal/runner/options.go

@@ -1,93 +1,158 @@
 package runner
 
 import (
-	"flag"
+	"bufio"
+	"errors"
+	"net/url"
 	"os"
+	"strings"
 
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
+	"github.com/projectdiscovery/gologger/formatter"
+	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
-// Options contains the configuration options for tuning
-// the template requesting process.
-type Options struct {
-	Debug              bool                   // Debug mode allows debugging request/responses for the engine
-	Templates          string                 // Signature specifies the template/templates to use
-	Target             string                 // Target is a single URL/Domain to scan usng a template
-	Targets            string                 // Targets specifies the targets to scan using templates.
-	Threads            int                    // Thread controls the number of concurrent requests to make.
-	Timeout            int                    // Timeout is the seconds to wait for a response from the server.
-	Retries            int                    // Retries is the number of times to retry the request
-	Output             string                 // Output is the file to write found subdomains to.
-	ProxyURL           string                 // ProxyURL is the URL for the proxy server
-	ProxySocksURL      string                 // ProxySocksURL is the URL for the proxy socks server
-	Silent             bool                   // Silent suppresses any extra text and only writes found URLs on screen.
-	Version            bool                   // Version specifies if we should just show version and exit
-	Verbose            bool                   // Verbose flag indicates whether to show verbose output or not
-	NoColor            bool                   // No-Color disables the colored output.
-	CustomHeaders      requests.CustomHeaders // Custom global headers
-	UpdateTemplates    bool                   // UpdateTemplates updates the templates installed at startup
-	TemplatesDirectory string                 // TemplatesDirectory is the directory to use for storing templates
-	JSON               bool                   // JSON writes json output to files
-
-	Stdin bool // Stdin specifies whether stdin input was given to the process
-}
-
 // ParseOptions parses the command line flags provided by a user
-func ParseOptions() *Options {
-	options := &Options{}
-
-	flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template")
-	flag.StringVar(&options.Templates, "t", "", "Template input file/files to run on host")
-	flag.StringVar(&options.Targets, "l", "", "List of URLs to run templates on")
-	flag.StringVar(&options.Output, "o", "", "File to write output to (optional)")
-	flag.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server")
-	flag.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the proxy socks server")
-	flag.BoolVar(&options.Silent, "silent", false, "Show only results in output")
-	flag.BoolVar(&options.Version, "version", false, "Show version of nuclei")
-	flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output")
-	flag.BoolVar(&options.NoColor, "nC", false, "Don't Use colors in output")
-	flag.IntVar(&options.Threads, "c", 10, "Number of concurrent requests to make")
-	flag.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout")
-	flag.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
-	flag.Var(&options.CustomHeaders, "H", "Custom Header.")
-	flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses")
-	flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)")
-	flag.StringVar(&options.TemplatesDirectory, "update-directory", "", "Directory to use for storing nuclei-templates")
-	flag.BoolVar(&options.JSON, "json", false, "Write json output to files")
-
-	flag.Parse()
-
+func ParseOptions(options *types.Options) {
 	// Check if stdin pipe was given
 	options.Stdin = hasStdin()
 
 	// Read the inputs and configure the logging
-	options.configureOutput()
+	configureOutput(options)
 
 	// Show the user the banner
 	showBanner()
 
+	options.ExcludeTags = append(options.ExcludeTags, "dos")
 	if options.Version {
-		gologger.Infof("Current Version: %s\n", Version)
+		gologger.Info().Msgf("Current Version: %s\n", Version)
+		os.Exit(0)
+	}
+	if options.TemplatesVersion {
+		config, err := readConfiguration()
+		if err != nil {
+			gologger.Fatal().Msgf("Could not read template configuration: %s\n", err)
+		}
+		gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", config.CurrentVersion, config.TemplatesDirectory)
 		os.Exit(0)
 	}
 
 	// Validate the options passed by the user and if any
 	// invalid options have been used, exit.
-	err := options.validateOptions()
+	if err := validateOptions(options); err != nil {
+		gologger.Fatal().Msgf("Program exiting: %s\n", err)
+	}
+
+	// Auto adjust rate limits when using headless mode if the user
+	// hasn't specified any custom limits.
+	if options.Headless && options.BulkSize == 25 && options.TemplateThreads == 10 {
+		options.BulkSize = 2
+		options.TemplateThreads = 2
+	}
+
+	// Load the resolvers if user asked for them
+	loadResolvers(options)
+
+	err := protocolinit.Init(options)
 	if err != nil {
-		gologger.Fatalf("Program exiting: %s\n", err)
+		gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err)
 	}
-	return options
 }
 
+// hasStdin returns true if we have stdin input
 func hasStdin() bool {
-	fi, err := os.Stdin.Stat()
+	stat, err := os.Stdin.Stat()
 	if err != nil {
 		return false
 	}
-	if fi.Mode()&os.ModeNamedPipe == 0 {
-		return false
+
+	isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0
+	isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0
+
+	return isPipedFromChrDev || isPipedFromFIFO
+}
+
+// validateOptions validates the configuration options passed
+func validateOptions(options *types.Options) error {
+	// Both verbose and silent flags were used
+	if options.Verbose && options.Silent {
+		return errors.New("both verbose and silent mode specified")
 	}
-	return true
+
+	if !options.TemplateList {
+		// Check if a list of templates was provided and it exists
+		if len(options.Templates) == 0 && !options.NewTemplates && len(options.Workflows) == 0 && len(options.Tags) == 0 && !options.UpdateTemplates {
+			return errors.New("no template/templates provided")
+		}
+	}
+
+	// Validate proxy options if provided
+	err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)")
+	if err != nil {
+		return err
+	}
+
+	err = validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)")
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func validateProxyURL(proxyURL, message string) error {
+	if proxyURL != "" && !isValidURL(proxyURL) {
+		return errors.New(message)
+	}
+
+	return nil
+}
+
+func isValidURL(urlString string) bool {
+	_, err := url.Parse(urlString)
+	return err == nil
+}
+
+// configureOutput configures the output on the screen
+func configureOutput(options *types.Options) {
+	// If the user desires verbose output, show verbose output
+	if options.Verbose {
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
+	}
+	if options.Debug {
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
+	}
+	if options.NoColor {
+		gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true))
+	}
+	if options.Silent {
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
+	}
+}
+
+// loadResolvers loads resolvers from both user provided flag and file
+func loadResolvers(options *types.Options) {
+	if options.ResolversFile == "" {
+		return
+	}
+
+	file, err := os.Open(options.ResolversFile)
+	if err != nil {
+		gologger.Fatal().Msgf("Could not open resolvers file: %s\n", err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		part := scanner.Text()
+		if part == "" {
+			continue
+		}
+		if strings.Contains(part, ":") {
+			options.InternalResolversList = append(options.InternalResolversList, part)
+		} else {
+			options.InternalResolversList = append(options.InternalResolversList, part+":53")
+		}
+	}
 }

v2/internal/runner/processor.go

@@ -0,0 +1,50 @@
+package runner
+
+import (
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/remeh/sizedwaitgroup"
+	"go.uber.org/atomic"
+)
+
+// processTemplateWithList process a template on the URL list
+func (r *Runner) processTemplateWithList(template *templates.Template) bool {
+	results := &atomic.Bool{}
+	wg := sizedwaitgroup.New(r.options.BulkSize)
+	r.hostMap.Scan(func(k, _ []byte) error {
+		URL := string(k)
+
+		wg.Add()
+		go func(URL string) {
+			defer wg.Done()
+
+			match, err := template.Executer.Execute(URL)
+			if err != nil {
+				gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err)
+			}
+			results.CAS(false, match)
+		}(URL)
+		return nil
+	})
+	wg.Wait()
+	return results.Load()
+}
+
+// processTemplateWithList process a template on the URL list
+func (r *Runner) processWorkflowWithList(template *templates.Template) bool {
+	results := &atomic.Bool{}
+	wg := sizedwaitgroup.New(r.options.BulkSize)
+
+	r.hostMap.Scan(func(k, _ []byte) error {
+		URL := string(k)
+		wg.Add()
+		go func(URL string) {
+			defer wg.Done()
+			match := template.CompiledWorkflow.RunWorkflow(URL)
+			results.CAS(false, match)
+		}(URL)
+		return nil
+	})
+	wg.Wait()
+	return results.Load()
+}

v2/internal/runner/runner.go

@@ -2,472 +2,388 @@ package runner
 
 import (
 	"bufio"
-	"context"
-	"errors"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"os"
+	"path"
 	"strings"
-	"sync"
 
-	"github.com/d5/tengo/v2"
-	"github.com/karrick/godirwalk"
+	"github.com/logrusorgru/aurora"
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/executor"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
+	"github.com/projectdiscovery/hmap/store/hybrid"
+	"github.com/projectdiscovery/nuclei/v2/internal/collaborator"
+	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"github.com/remeh/sizedwaitgroup"
+	"github.com/rs/xid"
+	"go.uber.org/atomic"
+	"go.uber.org/ratelimit"
+	"gopkg.in/yaml.v2"
 )
 
 // Runner is a client for running the enumeration process.
 type Runner struct {
-	// output is the output file to write if any
-	output      *os.File
-	outputMutex *sync.Mutex
-
-	tempFile        string
+	hostMap         *hybrid.HybridMap
+	output          output.Writer
+	inputCount      int64
 	templatesConfig *nucleiConfig
-	// options contains configuration options for runner
-	options *Options
+	options         *types.Options
+	projectFile     *projectfile.ProjectFile
+	catalog         *catalog.Catalog
+	progress        progress.Progress
+	colorizer       aurora.Aurora
+	issuesClient    *reporting.Client
+	severityColors  *colorizer.Colorizer
+	browser         *engine.Browser
+	ratelimiter     ratelimit.Limiter
 }
 
 // New creates a new client for running enumeration process.
-func New(options *Options) (*Runner, error) {
+func New(options *types.Options) (*Runner, error) {
 	runner := &Runner{
-		outputMutex: &sync.Mutex{},
-		options:     options,
+		options: options,
+	}
+	if options.Headless {
+		browser, err := engine.New(options)
+		if err != nil {
+			return nil, err
+		}
+		runner.browser = browser
+	}
+	if err := runner.updateTemplates(); err != nil {
+		gologger.Warning().Msgf("Could not update templates: %s\n", err)
+	}
+	// Read nucleiignore file if given a templateconfig
+	if runner.templatesConfig != nil {
+		runner.readNucleiIgnoreFile()
+	}
+	runner.catalog = catalog.New(runner.options.TemplatesDirectory)
+
+	var reportingOptions *reporting.Options
+	if options.ReportingConfig != "" {
+		file, err := os.Open(options.ReportingConfig)
+		if err != nil {
+			gologger.Fatal().Msgf("Could not open reporting config file: %s\n", err)
+		}
+
+		reportingOptions = &reporting.Options{}
+		if parseErr := yaml.NewDecoder(file).Decode(reportingOptions); parseErr != nil {
+			file.Close()
+			gologger.Fatal().Msgf("Could not parse reporting config file: %s\n", parseErr)
+		}
+		file.Close()
+	}
+	if options.DiskExportDirectory != "" {
+		if reportingOptions != nil {
+			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
+		} else {
+			reportingOptions = &reporting.Options{}
+			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
+		}
+	}
+	if reportingOptions != nil {
+		if client, err := reporting.New(reportingOptions, options.ReportingDB); err != nil {
+			gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err)
+		} else {
+			runner.issuesClient = client
+		}
 	}
 
-	if err := runner.updateTemplates(); err != nil {
-		gologger.Warningf("Could not update templates: %s\n", err)
-	}
-	if (options.Templates == "" || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
+	// output coloring
+	useColor := !options.NoColor
+	runner.colorizer = aurora.NewAurora(useColor)
+	runner.severityColors = colorizer.New(runner.colorizer)
+
+	if options.TemplateList {
+		runner.listAvailableTemplates()
 		os.Exit(0)
 	}
 
-	// If we have stdin, write it to a new file
-	if options.Stdin {
-		tempInput, err := ioutil.TempFile("", "stdin-input-*")
-		if err != nil {
-			return nil, err
-		}
-		if _, err := io.Copy(tempInput, os.Stdin); err != nil {
-			return nil, err
-		}
-		runner.tempFile = tempInput.Name()
-		tempInput.Close()
+	if (len(options.Templates) == 0 || !options.NewTemplates || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
+		os.Exit(0)
 	}
-	// If we have single target, write it to a new file
+	if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil {
+		gologger.Fatal().Msgf("Could not create temporary input file: %s\n", err)
+	} else {
+		runner.hostMap = hm
+	}
+
+	runner.inputCount = 0
+	dupeCount := 0
+
+	// Handle single target
 	if options.Target != "" {
-		tempInput, err := ioutil.TempFile("", "stdin-input-*")
-		if err != nil {
-			return nil, err
+		runner.inputCount++
+		// nolint:errcheck // ignoring error
+		runner.hostMap.Set(options.Target, nil)
+	}
+
+	// Handle stdin
+	if options.Stdin {
+		scanner := bufio.NewScanner(os.Stdin)
+		for scanner.Scan() {
+			url := strings.TrimSpace(scanner.Text())
+			if url == "" {
+				continue
+			}
+			if _, ok := runner.hostMap.Get(url); ok {
+				dupeCount++
+				continue
+			}
+			runner.inputCount++
+			// nolint:errcheck // ignoring error
+			runner.hostMap.Set(url, nil)
 		}
-		tempInput.WriteString(options.Target)
-		runner.tempFile = tempInput.Name()
-		tempInput.Close()
+	}
+
+	// Handle taget file
+	if options.Targets != "" {
+		input, err := os.Open(options.Targets)
+		if err != nil {
+			gologger.Fatal().Msgf("Could not open targets file '%s': %s\n", options.Targets, err)
+		}
+		scanner := bufio.NewScanner(input)
+		for scanner.Scan() {
+			url := strings.TrimSpace(scanner.Text())
+			if url == "" {
+				continue
+			}
+			if _, ok := runner.hostMap.Get(url); ok {
+				dupeCount++
+				continue
+			}
+			runner.inputCount++
+			// nolint:errcheck // ignoring error
+			runner.hostMap.Set(url, nil)
+		}
+		input.Close()
+	}
+
+	if dupeCount > 0 {
+		gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", dupeCount)
 	}
 
 	// Create the output file if asked
-	if options.Output != "" {
-		output, err := os.Create(options.Output)
-		if err != nil {
-			gologger.Fatalf("Could not create output file '%s': %s\n", options.Output, err)
+	outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.JSON, options.Output, options.TraceLogFile)
+	if err != nil {
+		gologger.Fatal().Msgf("Could not create output file '%s': %s\n", options.Output, err)
+	}
+	runner.output = outputWriter
+
+	// Creates the progress tracking object
+	var progressErr error
+	runner.progress, progressErr = progress.NewStatsTicker(options.StatsInterval, options.EnableProgressBar, options.Metrics, options.MetricsPort)
+	if progressErr != nil {
+		return nil, progressErr
+	}
+
+	// create project file if requested or load existing one
+	if options.Project {
+		var projectFileErr error
+		runner.projectFile, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""})
+		if projectFileErr != nil {
+			return nil, projectFileErr
 		}
-		runner.output = output
+	}
+
+	// Enable Polling
+	if options.BurpCollaboratorBiid != "" {
+		collaborator.DefaultCollaborator.Collab.AddBIID(options.BurpCollaboratorBiid)
+	}
+
+	if options.RateLimit > 0 {
+		runner.ratelimiter = ratelimit.New(options.RateLimit)
+	} else {
+		runner.ratelimiter = ratelimit.NewUnlimited()
 	}
 	return runner, nil
 }
 
 // Close releases all the resources and cleans up
 func (r *Runner) Close() {
-	r.output.Close()
-	os.Remove(r.tempFile)
+	if r.output != nil {
+		r.output.Close()
+	}
+	r.hostMap.Close()
+	if r.projectFile != nil {
+		r.projectFile.Close()
+	}
 }
 
 // RunEnumeration sets up the input layer for giving input nuclei.
 // binary and runs the actual enumeration
 func (r *Runner) RunEnumeration() {
-	var err error
+	defer r.Close()
 
-	// Check if the template is an absolute path or relative path.
-	// If the path is absolute, use it. Otherwise,
-	if r.isRelative(r.options.Templates) {
-		newPath, err := r.resolvePath(r.options.Templates)
+	// If we have no templates, run on whole template directory with provided tags
+	if len(r.options.Templates) == 0 && len(r.options.Workflows) == 0 && !r.options.NewTemplates && (len(r.options.Tags) > 0 || len(r.options.ExcludeTags) > 0) {
+		r.options.Templates = append(r.options.Templates, r.options.TemplatesDirectory)
+	}
+	if r.options.NewTemplates {
+		templatesLoaded, err := r.readNewTemplatesFile()
 		if err != nil {
-			gologger.Errorf("Could not find template file '%s': %s\n", r.options.Templates, err)
-			return
+			gologger.Warning().Msgf("Could not get newly added templates: %s\n", err)
 		}
-		r.options.Templates = newPath
+		r.options.Templates = append(r.options.Templates, templatesLoaded...)
 	}
+	includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates)
+	excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates)
+	// defaults to all templates
+	allTemplates := includedTemplates
 
-	// Single yaml provided
-	if strings.HasSuffix(r.options.Templates, ".yaml") {
-		t, err := r.parse(r.options.Templates)
-		switch t.(type) {
-		case *templates.Template:
-			var results bool
-			template := t.(*templates.Template)
-			// process http requests
-			for _, request := range template.RequestsHTTP {
-				results = r.processTemplateRequest(template, request)
-			}
-			// process dns requests
-			for _, request := range template.RequestsDNS {
-				dnsResults := r.processTemplateRequest(template, request)
-				if !results {
-					results = dnsResults
-				}
-			}
-
-			if !results {
-				if r.output != nil {
-					outputFile := r.output.Name()
-					r.output.Close()
-					os.Remove(outputFile)
-				}
-				gologger.Infof("No results found for the template. Happy hacking!")
-			}
-		case *workflows.Workflow:
-			workflow := t.(*workflows.Workflow)
-			r.ProcessWorkflowWithList(workflow)
-		default:
-			gologger.Errorf("Could not parse file '%s': %s\n", r.options.Templates, err)
+	if len(excludedTemplates) > 0 {
+		excludedMap := make(map[string]struct{}, len(excludedTemplates))
+		for _, excl := range excludedTemplates {
+			excludedMap[excl] = struct{}{}
 		}
-		return
-	}
+		// rebuild list with only non-excluded templates
+		allTemplates = []string{}
 
-	// If the template passed is a directory
-	matches := []string{}
-
-	// Recursively walk down the Templates directory and run all the template file checks
-	err = godirwalk.Walk(r.options.Templates, &godirwalk.Options{
-		Callback: func(path string, d *godirwalk.Dirent) error {
-			if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
-				matches = append(matches, path)
+		for _, incl := range includedTemplates {
+			if _, found := excludedMap[incl]; !found {
+				allTemplates = append(allTemplates, incl)
+			} else {
+				gologger.Warning().Msgf("Excluding '%s'", incl)
 			}
-			return nil
-		},
-		ErrorCallback: func(path string, err error) godirwalk.ErrorAction {
-			return godirwalk.SkipNode
-		},
-		Unsorted: true,
-	})
-	if err != nil {
-		gologger.Fatalf("Could not find templates in directory '%s': %s\n", r.options.Templates, err)
-	}
-	// 0 matches means no templates were found in directory
-	if len(matches) == 0 {
-		gologger.Fatalf("Error, no templates found in directory: '%s'\n", r.options.Templates)
-	}
-
-	var results bool
-	for _, match := range matches {
-		t, err := r.parse(match)
-		switch t.(type) {
-		case *templates.Template:
-			template := t.(*templates.Template)
-			for _, request := range template.RequestsDNS {
-				dnsResults := r.processTemplateRequest(template, request)
-				if dnsResults {
-					results = dnsResults
-				}
-			}
-			for _, request := range template.RequestsHTTP {
-				httpResults := r.processTemplateRequest(template, request)
-				if httpResults {
-					results = httpResults
-				}
-			}
-		case *workflows.Workflow:
-			workflow := t.(*workflows.Workflow)
-			r.ProcessWorkflowWithList(workflow)
-		default:
-			gologger.Errorf("Could not parse file '%s': %s\n", r.options.Templates, err)
 		}
 	}
-	if !results {
-		if r.output != nil {
-			outputFile := r.output.Name()
-			r.output.Close()
-			os.Remove(outputFile)
-		}
-		gologger.Infof("No results found for the template. Happy hacking!")
-	}
-	return
-}
 
-// processTemplate processes a template and runs the enumeration on all the targets
-func (r *Runner) processTemplateRequest(template *templates.Template, request interface{}) bool {
-	var file *os.File
-	var err error
+	// pre-parse all the templates, apply filters
+	finalTemplates := []*templates.Template{}
 
-	// Handle a list of hosts as argument
-	if r.options.Targets != "" {
-		file, err = os.Open(r.options.Targets)
-	} else if r.options.Stdin || r.options.Target != "" {
-		file, err = os.Open(r.tempFile)
-	}
-	if err != nil {
-		gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err)
-	}
-	results := r.processTemplateWithList(template, request, file)
-	file.Close()
-	return results
-}
+	workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows)
+	availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false)
+	availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true)
 
-// processDomain processes the list with a template
-func (r *Runner) processTemplateWithList(template *templates.Template, request interface{}, reader io.Reader) bool {
-	// Display the message for the template
-	message := fmt.Sprintf("[%s] Loaded template %s (@%s)", template.ID, template.Info.Name, template.Info.Author)
-	if template.Info.Severity != "" {
-		message += " [" + template.Info.Severity + "]"
-	}
-	gologger.Infof("%s\n", message)
-
-	var writer *bufio.Writer
-	if r.output != nil {
-		writer = bufio.NewWriter(r.output)
-		defer writer.Flush()
-	}
-
-	var httpExecutor *executor.HTTPExecutor
-	var dnsExecutor *executor.DNSExecutor
-	var err error
-
-	// Create an executor based on the request type.
-	switch value := request.(type) {
-	case *requests.DNSRequest:
-		dnsExecutor = executor.NewDNSExecutor(&executor.DNSOptions{
-			Debug:      r.options.Debug,
-			Template:   template,
-			DNSRequest: value,
-			Writer:     writer,
-			JSON:       r.options.JSON,
-		})
-	case *requests.HTTPRequest:
-		httpExecutor, err = executor.NewHTTPExecutor(&executor.HTTPOptions{
-			Debug:         r.options.Debug,
-			Template:      template,
-			HTTPRequest:   value,
-			Writer:        writer,
-			Timeout:       r.options.Timeout,
-			Retries:       r.options.Retries,
-			ProxyURL:      r.options.ProxyURL,
-			ProxySocksURL: r.options.ProxySocksURL,
-			CustomHeaders: r.options.CustomHeaders,
-			JSON:          r.options.JSON,
-		})
-	}
-	if err != nil {
-		gologger.Warningf("Could not create http client: %s\n", err)
-		return false
-	}
-
-	limiter := make(chan struct{}, r.options.Threads)
-	wg := &sync.WaitGroup{}
-
-	scanner := bufio.NewScanner(reader)
-	for scanner.Scan() {
-		text := scanner.Text()
-		if text == "" {
+	var unclusteredRequests int64 = 0
+	for _, template := range availableTemplates {
+		// workflows will dynamically adjust the totals while running, as
+		// it can't be know in advance which requests will be called
+		if len(template.Workflows) > 0 {
 			continue
 		}
-		limiter <- struct{}{}
-		wg.Add(1)
-
-		go func(URL string) {
-			var err error
-
-			if httpExecutor != nil {
-				err = httpExecutor.ExecuteHTTP(URL)
-			}
-			if dnsExecutor != nil {
-				err = dnsExecutor.ExecuteDNS(URL)
-			}
-			if err != nil {
-				gologger.Warningf("Could not execute step: %s\n", err)
-			}
-			<-limiter
-			wg.Done()
-		}(text)
+		unclusteredRequests += int64(template.TotalRequests) * r.inputCount
 	}
-	close(limiter)
-	wg.Wait()
 
-	// See if we got any results from the executors
-	var results bool
-	if httpExecutor != nil {
-		results = httpExecutor.GotResults()
-	}
-	if dnsExecutor != nil {
-		if !results {
-			results = dnsExecutor.GotResults()
+	originalTemplatesCount := len(availableTemplates)
+	clusterCount := 0
+	clusters := clusterer.Cluster(availableTemplates)
+	for _, cluster := range clusters {
+		if len(cluster) > 1 && !r.options.OfflineHTTP {
+			executerOpts := protocols.ExecuterOptions{
+				Output:       r.output,
+				Options:      r.options,
+				Progress:     r.progress,
+				Catalog:      r.catalog,
+				RateLimiter:  r.ratelimiter,
+				IssuesClient: r.issuesClient,
+				Browser:      r.browser,
+				ProjectFile:  r.projectFile,
+			}
+			clusterID := fmt.Sprintf("cluster-%s", xid.New().String())
+
+			finalTemplates = append(finalTemplates, &templates.Template{
+				ID:            clusterID,
+				RequestsHTTP:  cluster[0].RequestsHTTP,
+				Executer:      clusterer.NewExecuter(cluster, &executerOpts),
+				TotalRequests: len(cluster[0].RequestsHTTP),
+			})
+			clusterCount += len(cluster)
+		} else {
+			finalTemplates = append(finalTemplates, cluster...)
 		}
 	}
-	return results
+	for _, workflows := range availableWorkflows {
+		finalTemplates = append(finalTemplates, workflows)
+	}
+
+	var totalRequests int64 = 0
+	for _, t := range finalTemplates {
+		if len(t.Workflows) > 0 {
+			continue
+		}
+		totalRequests += int64(t.TotalRequests) * r.inputCount
+	}
+	if totalRequests < unclusteredRequests {
+		gologger.Info().Msgf("Reduced %d requests to %d (%d templates clustered)", unclusteredRequests, totalRequests, clusterCount)
+	}
+	templateCount := originalTemplatesCount + len(availableWorkflows)
+
+	// 0 matches means no templates were found in directory
+	if templateCount == 0 {
+		gologger.Fatal().Msgf("Error, no templates were found.\n")
+	}
+
+	gologger.Info().Msgf("Using %s rules (%s templates, %s workflows)",
+		r.colorizer.Bold(templateCount).String(),
+		r.colorizer.Bold(templateCount-workflowCount).String(),
+		r.colorizer.Bold(workflowCount).String())
+
+	results := &atomic.Bool{}
+	wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
+	// Starts polling or ignore
+	collaborator.DefaultCollaborator.Poll()
+
+	// tracks global progress and captures stdout/stderr until p.Wait finishes
+	r.progress.Init(r.inputCount, templateCount, totalRequests)
+
+	for _, t := range finalTemplates {
+		wgtemplates.Add()
+		go func(template *templates.Template) {
+			defer wgtemplates.Done()
+
+			if len(template.Workflows) > 0 {
+				results.CAS(false, r.processWorkflowWithList(template))
+			} else {
+				results.CAS(false, r.processTemplateWithList(template))
+			}
+		}(t)
+	}
+	wgtemplates.Wait()
+	r.progress.Stop()
+
+	if r.issuesClient != nil {
+		r.issuesClient.Close()
+	}
+	if !results.Load() {
+		if r.output != nil {
+			r.output.Close()
+			os.Remove(r.options.Output)
+		}
+		gologger.Info().Msgf("No results found. Better luck next time!")
+	}
+
+	if r.browser != nil {
+		r.browser.Close()
+	}
 }
 
-// ProcessWorkflowWithList coming from stdin or list of targets
-func (r *Runner) ProcessWorkflowWithList(workflow *workflows.Workflow) {
-	var file *os.File
-	var err error
-	// Handle a list of hosts as argument
-	if r.options.Targets != "" {
-		file, err = os.Open(r.options.Targets)
-	} else if r.options.Stdin {
-		file, err = os.Open(r.tempFile)
-	}
+// readNewTemplatesFile reads newly added templates from directory if it exists
+func (r *Runner) readNewTemplatesFile() ([]string, error) {
+	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	file, err := os.Open(additionsFile)
 	if err != nil {
-		gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err)
+		return nil, err
 	}
 	defer file.Close()
 
+	templatesList := []string{}
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
 		text := scanner.Text()
 		if text == "" {
 			continue
 		}
-		if err := r.ProcessWorkflow(workflow, text); err != nil {
-			gologger.Warningf("Could not run workflow for %s: %s\n", text, err)
-		}
+		templatesList = append(templatesList, text)
 	}
-}
-
-// ProcessWorkflow towards an URL
-func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error {
-	script := tengo.NewScript([]byte(workflow.Logic))
-
-	for name, value := range workflow.Variables {
-		var writer *bufio.Writer
-		if r.output != nil {
-			writer = bufio.NewWriter(r.output)
-			defer writer.Flush()
-		}
-
-		// Check if the template is an absolute path or relative path.
-		// If the path is absolute, use it. Otherwise,
-		if r.isRelative(value) {
-			newPath, err := r.resolvePath(value)
-			if err != nil {
-				return err
-			}
-			value = newPath
-		}
-
-		// Single yaml provided
-		var templatesList []*workflows.Template
-		if strings.HasSuffix(value, ".yaml") {
-			t, err := templates.Parse(value)
-			if err != nil {
-				return err
-			}
-			template := &workflows.Template{}
-			if len(t.RequestsHTTP) > 0 {
-				template.HTTPOptions = &executor.HTTPOptions{
-					Debug:         r.options.Debug,
-					Writer:        writer,
-					Template:      t,
-					Timeout:       r.options.Timeout,
-					Retries:       r.options.Retries,
-					ProxyURL:      r.options.ProxyURL,
-					ProxySocksURL: r.options.ProxySocksURL,
-					CustomHeaders: r.options.CustomHeaders,
-				}
-			} else if len(t.RequestsDNS) > 0 {
-				template.DNSOptions = &executor.DNSOptions{
-					Debug:    r.options.Debug,
-					Template: t,
-					Writer:   writer,
-				}
-			}
-			if template.DNSOptions != nil || template.HTTPOptions != nil {
-				templatesList = append(templatesList, template)
-			}
-		} else {
-			matches := []string{}
-
-			err := godirwalk.Walk(value, &godirwalk.Options{
-				Callback: func(path string, d *godirwalk.Dirent) error {
-					if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
-						matches = append(matches, path)
-					}
-					return nil
-				},
-				ErrorCallback: func(path string, err error) godirwalk.ErrorAction {
-					return godirwalk.SkipNode
-				},
-				Unsorted: true,
-			})
-			if err != nil {
-				return err
-			}
-			// 0 matches means no templates were found in directory
-			if len(matches) == 0 {
-				return errors.New("no match found in the directory")
-			}
-
-			for _, match := range matches {
-				t, err := templates.Parse(match)
-				if err != nil {
-					return err
-				}
-				template := &workflows.Template{}
-				if len(t.RequestsHTTP) > 0 {
-					template.HTTPOptions = &executor.HTTPOptions{
-						Debug:         r.options.Debug,
-						Writer:        writer,
-						Template:      t,
-						Timeout:       r.options.Timeout,
-						Retries:       r.options.Retries,
-						ProxyURL:      r.options.ProxyURL,
-						ProxySocksURL: r.options.ProxySocksURL,
-						CustomHeaders: r.options.CustomHeaders,
-					}
-				} else if len(t.RequestsDNS) > 0 {
-					template.DNSOptions = &executor.DNSOptions{
-						Debug:    r.options.Debug,
-						Template: t,
-						Writer:   writer,
-					}
-				}
-				if template.DNSOptions != nil || template.HTTPOptions != nil {
-					templatesList = append(templatesList, template)
-				}
-			}
-		}
-
-		script.Add(name, &workflows.NucleiVar{Templates: templatesList, URL: URL})
-	}
-
-	_, err := script.RunContext(context.Background())
-	if err != nil {
-		gologger.Errorf("Could not execute workflow '%s': %s\n", workflow.ID, err)
-		return err
-	}
-	return nil
-}
-
-func (r *Runner) parse(file string) (interface{}, error) {
-	// check if it's a template
-	template, errTemplate := templates.Parse(file)
-	if errTemplate == nil {
-		return template, nil
-	}
-
-	// check if it's a workflow
-	workflow, errWorkflow := workflows.Parse(file)
-	if errWorkflow == nil {
-		return workflow, nil
-	}
-
-	if errTemplate != nil {
-		return nil, errTemplate
-	}
-	if errWorkflow != nil {
-		return nil, errWorkflow
-	}
-	return nil, errors.New("unknown error occured")
+	return templatesList, nil
 }

v2/internal/runner/templates.go

@@ -0,0 +1,166 @@
+package runner
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/karrick/godirwalk"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+// getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered
+// by severity, along with a flag indicating if workflows are present.
+func (r *Runner) getParsedTemplatesFor(templatePaths, severities []string, workflows bool) (parsedTemplates map[string]*templates.Template, workflowCount int) {
+	filterBySeverity := len(severities) > 0
+
+	if !workflows {
+		gologger.Info().Msgf("Loading templates...")
+	} else {
+		gologger.Info().Msgf("Loading workflows...")
+	}
+
+	parsedTemplates = make(map[string]*templates.Template)
+	for _, match := range templatePaths {
+		t, err := r.parseTemplateFile(match)
+		if err != nil {
+			gologger.Warning().Msgf("Could not parse file '%s': %s\n", match, err)
+			continue
+		}
+		if t == nil {
+			continue
+		}
+		if len(t.Workflows) == 0 && workflows {
+			continue // don't print if user only wants to run workflows
+		}
+		if len(t.Workflows) > 0 && !workflows {
+			continue // don't print workflow if user only wants to run templates
+		}
+		if len(t.Workflows) > 0 {
+			workflowCount++
+		}
+		sev := strings.ToLower(types.ToString(t.Info["severity"]))
+		if !filterBySeverity || hasMatchingSeverity(sev, severities) {
+			parsedTemplates[t.ID] = t
+			gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), sev))
+		} else {
+			gologger.Warning().Msgf("Excluding template %s due to severity filter (%s not in [%s])", t.ID, sev, severities)
+		}
+	}
+	return parsedTemplates, workflowCount
+}
+
+// parseTemplateFile returns the parsed template file
+func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
+	executerOpts := protocols.ExecuterOptions{
+		Output:       r.output,
+		Options:      r.options,
+		Progress:     r.progress,
+		Catalog:      r.catalog,
+		IssuesClient: r.issuesClient,
+		RateLimiter:  r.ratelimiter,
+		ProjectFile:  r.projectFile,
+		Browser:      r.browser,
+	}
+	template, err := templates.Parse(file, executerOpts)
+	if err != nil {
+		return nil, err
+	}
+	if template == nil {
+		return nil, nil
+	}
+	return template, nil
+}
+
+func (r *Runner) templateLogMsg(id, name, author, severity string) string {
+	// Display the message for the template
+	message := fmt.Sprintf("[%s] %s (%s)",
+		r.colorizer.BrightBlue(id).String(),
+		r.colorizer.Bold(name).String(),
+		r.colorizer.BrightYellow("@"+author).String())
+	if severity != "" {
+		message += " [" + r.severityColors.Data[severity] + "]"
+	}
+	return message
+}
+
+func (r *Runner) logAvailableTemplate(tplPath string) {
+	t, err := r.parseTemplateFile(tplPath)
+	if err != nil {
+		gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
+	} else {
+		gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
+	}
+}
+
+// ListAvailableTemplates prints available templates to stdout
+func (r *Runner) listAvailableTemplates() {
+	if r.templatesConfig == nil {
+		return
+	}
+
+	if _, err := os.Stat(r.templatesConfig.TemplatesDirectory); os.IsNotExist(err) {
+		gologger.Error().Msgf("%s does not exists", r.templatesConfig.TemplatesDirectory)
+		return
+	}
+
+	gologger.Print().Msgf(
+		"\nListing available v.%s nuclei templates for %s",
+		r.templatesConfig.CurrentVersion,
+		r.templatesConfig.TemplatesDirectory,
+	)
+	err := directoryWalker(
+		r.templatesConfig.TemplatesDirectory,
+		func(path string, d *godirwalk.Dirent) error {
+			if d.IsDir() && path != r.templatesConfig.TemplatesDirectory {
+				gologger.Print().Msgf("\n%s:\n\n", r.colorizer.Bold(r.colorizer.BgBrightBlue(d.Name())).String())
+			} else if strings.HasSuffix(path, ".yaml") {
+				r.logAvailableTemplate(path)
+			}
+			return nil
+		},
+	)
+	// directory couldn't be walked
+	if err != nil {
+		gologger.Error().Msgf("Could not find templates in directory '%s': %s\n", r.templatesConfig.TemplatesDirectory, err)
+	}
+}
+
+func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool {
+	for _, s := range allowedSeverities {
+		finalSeverities := []string{}
+		if strings.Contains(s, ",") {
+			finalSeverities = strings.Split(s, ",")
+		} else {
+			finalSeverities = append(finalSeverities, s)
+		}
+
+		for _, sev := range finalSeverities {
+			sev = strings.ToLower(sev)
+			if sev != "" && strings.HasPrefix(templateSeverity, sev) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error {
+	err := godirwalk.Walk(fsPath, &godirwalk.Options{
+		Callback: callback,
+		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
+			return godirwalk.SkipNode
+		},
+		Unsorted: true,
+	})
+
+	// directory couldn't be walked
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

v2/internal/runner/update.go

@@ -2,9 +2,11 @@ package runner
 
 import (
 	"archive/zip"
+	"bufio"
 	"bytes"
 	"context"
-	"errors"
+	"crypto/md5"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -12,66 +14,21 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"regexp"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/blang/semver"
 	"github.com/google/go-github/v32/github"
-	jsoniter "github.com/json-iterator/go"
+	"github.com/olekukonko/tablewriter"
+	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
 )
 
-// nucleiConfig contains some configuration options for nuclei
-type nucleiConfig struct {
-	TemplatesDirectory string `json:"templates-directory,omitempty"`
-	CurrentVersion     string `json:"current-version,omitempty"`
-}
-
-// nucleiConfigFilename is the filename of nuclei configuration file.
-const nucleiConfigFilename = ".nuclei-config.json"
-
-var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
-
-// readConfiguration reads the nuclei configuration file from disk.
-func (r *Runner) readConfiguration() (*nucleiConfig, error) {
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return nil, err
-	}
-
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
-	file, err := os.Open(templatesConfigFile)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-
-	config := &nucleiConfig{}
-	if err = jsoniter.NewDecoder(file).Decode(config); err != nil {
-		return nil, err
-	}
-	return config, nil
-}
-
-// readConfiguration reads the nuclei configuration file from disk.
-func (r *Runner) writeConfiguration(config *nucleiConfig) error {
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return err
-	}
-
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
-	file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	if err = jsoniter.NewEncoder(file).Encode(config); err != nil {
-		return err
-	}
-	return nil
-}
+const (
+	userName = "projectdiscovery"
+	repoName = "nuclei-templates"
+)
 
 // updateTemplates checks if the default list of nuclei-templates
 // exist in the users home directory, if not the latest revision
@@ -84,46 +41,101 @@ func (r *Runner) updateTemplates() error {
 	if err != nil {
 		return err
 	}
+	configDir := path.Join(home, "/.config", "/nuclei")
+	_ = os.MkdirAll(configDir, os.ModePerm)
 
 	templatesConfigFile := path.Join(home, nucleiConfigFilename)
-	if _, err := os.Stat(templatesConfigFile); !os.IsNotExist(err) {
-		config, err := r.readConfiguration()
+	if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
+		config, readErr := readConfiguration()
 		if err != nil {
-			return err
+			return readErr
 		}
 		r.templatesConfig = config
 	}
 
-	if r.templatesConfig == nil || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
+	ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
+	if r.templatesConfig == nil {
+		currentConfig := &nucleiConfig{
+			TemplatesDirectory: path.Join(home, "nuclei-templates"),
+			IgnoreURL:          ignoreURL,
+			NucleiVersion:      Version,
+		}
+		if writeErr := r.writeConfiguration(currentConfig); writeErr != nil {
+			return errors.Wrap(writeErr, "could not write template configuration")
+		}
+		r.templatesConfig = currentConfig
+	}
+	// Check if last checked for nuclei-ignore is more than 1 hours.
+	// and if true, run the check.
+	if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
+		if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" {
+			ignoreURL = r.templatesConfig.IgnoreURL
+		}
+		gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
+
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
+		if reqErr == nil {
+			resp, httpGet := http.DefaultClient.Do(req)
+			if httpGet != nil {
+				if resp != nil && resp.Body != nil {
+					resp.Body.Close()
+				}
+				gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
+			} else {
+				data, _ := ioutil.ReadAll(resp.Body)
+				resp.Body.Close()
+
+				if len(data) > 0 {
+					_ = ioutil.WriteFile(path.Join(configDir, nucleiIgnoreFile), data, 0644)
+				}
+				if r.templatesConfig != nil {
+					r.templatesConfig.LastCheckedIgnore = time.Now()
+				}
+			}
+		}
+		cancel()
+	}
+
+	ctx := context.Background()
+	if r.templatesConfig.CurrentVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
 		if !r.options.UpdateTemplates {
-			gologger.Labelf("nuclei-templates are not installed, use update-templates flag.\n")
+			gologger.Warning().Msgf("nuclei-templates are not installed (or indexed), use update-templates flag.\n")
 			return nil
 		}
 
 		// Use custom location if user has given a template directory
-		if r.options.TemplatesDirectory != "" {
-			home = r.options.TemplatesDirectory
+		r.templatesConfig = &nucleiConfig{
+			TemplatesDirectory: path.Join(home, "nuclei-templates"),
+		}
+		if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") {
+			r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
 		}
-		r.templatesConfig = &nucleiConfig{TemplatesDirectory: path.Join(home, "nuclei-templates")}
-		os.RemoveAll(r.templatesConfig.TemplatesDirectory)
 
 		// Download the repository and also write the revision to a HEAD file.
-		version, asset, err := r.getLatestReleaseFromGithub()
+		version, asset, getErr := r.getLatestReleaseFromGithub()
+		if getErr != nil {
+			return getErr
+		}
+		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
+
+		_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
 		if err != nil {
 			return err
 		}
-
-		gologger.Verbosef("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory)
-
-		if err = r.downloadReleaseAndUnzip(asset.GetZipballURL()); err != nil {
-			return err
-		}
 		r.templatesConfig.CurrentVersion = version.String()
-		if err = r.writeConfiguration(r.templatesConfig); err != nil {
+
+		err = r.writeConfiguration(r.templatesConfig)
+		if err != nil {
 			return err
 		}
+		gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). Enjoy!\n", version.String())
+		return nil
+	}
 
-		gologger.Infof("Successfully downloaded nuclei-templates (v%s). Enjoy!\n", version.String())
+	// Check if last checked is more than 24 hours.
+	// If not, return since we don't want to do anything now.
+	if time.Since(r.templatesConfig.LastChecked) < 24*time.Hour && !r.options.UpdateTemplates {
 		return nil
 	}
 
@@ -136,6 +148,7 @@ func (r *Runner) updateTemplates() error {
 	if indices[0] > 0 {
 		verText = verText[indices[0]:]
 	}
+
 	oldVersion, err := semver.Make(verText)
 	if err != nil {
 		return err
@@ -147,39 +160,35 @@ func (r *Runner) updateTemplates() error {
 	}
 
 	if version.EQ(oldVersion) {
-		gologger.Labelf("Latest version of nuclei-templates installed: v%s\n", oldVersion.String())
-		return nil
+		gologger.Info().Msgf("Your nuclei-templates are up to date: v%s\n", oldVersion.String())
+		return r.writeConfiguration(r.templatesConfig)
 	}
+
 	if version.GT(oldVersion) {
 		if !r.options.UpdateTemplates {
-			gologger.Labelf("You're using outdated nuclei-templates. Latest v%s\n", version.String())
-			return nil
+			gologger.Warning().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
+			return r.writeConfiguration(r.templatesConfig)
 		}
+
 		if r.options.TemplatesDirectory != "" {
-			home = r.options.TemplatesDirectory
-			r.templatesConfig.TemplatesDirectory = path.Join(home, "nuclei-templates")
+			r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
 		}
 		r.templatesConfig.CurrentVersion = version.String()
-		os.RemoveAll(r.templatesConfig.TemplatesDirectory)
 
-		gologger.Verbosef("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory)
-
-		if err = r.downloadReleaseAndUnzip(asset.GetZipballURL()); err != nil {
+		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
+		_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
+		if err != nil {
 			return err
 		}
-		if err = r.writeConfiguration(r.templatesConfig); err != nil {
+		err = r.writeConfiguration(r.templatesConfig)
+		if err != nil {
 			return err
 		}
-		gologger.Infof("Successfully updated nuclei-templates (v%s). Enjoy!\n", version.String())
+		gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). Enjoy!\n", version.String())
 	}
 	return nil
 }
 
-const (
-	userName = "projectdiscovery"
-	repoName = "nuclei-templates"
-)
-
 // getLatestReleaseFromGithub returns the latest release from github
 func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.RepositoryRelease, error) {
 	client := github.NewClient(nil)
@@ -201,6 +210,7 @@ func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.Repositor
 		if indices[0] > 0 {
 			verText = verText[indices[0]:]
 		}
+
 		ver, err := semver.Make(verText)
 		if err != nil {
 			return semver.Version{}, nil, err
@@ -218,36 +228,86 @@ func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.Repositor
 }
 
 // downloadReleaseAndUnzip downloads and unzips the release in a directory
-func (r *Runner) downloadReleaseAndUnzip(downloadURL string) error {
-	req, err := http.NewRequest("GET", downloadURL, nil)
+func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadURL string) (*templateUpdateResults, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
-		return fmt.Errorf("Failed to create HTTP request to %s: %s", downloadURL, err)
+		return nil, fmt.Errorf("failed to create HTTP request to %s: %s", downloadURL, err)
 	}
 
 	res, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return fmt.Errorf("Failed to download a release file from %s: %s", downloadURL, err)
+		return nil, fmt.Errorf("failed to download a release file from %s: %s", downloadURL, err)
 	}
 	defer res.Body.Close()
-
-	if res.StatusCode != 200 {
-		return fmt.Errorf("Failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode)
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode)
 	}
 
 	buf, err := ioutil.ReadAll(res.Body)
 	if err != nil {
-		return fmt.Errorf("Failed to create buffer for zip file: %s", err)
+		return nil, fmt.Errorf("failed to create buffer for zip file: %s", err)
 	}
 
 	reader := bytes.NewReader(buf)
 	z, err := zip.NewReader(reader, reader.Size())
 	if err != nil {
-		return fmt.Errorf("Failed to uncompress zip file: %s", err)
+		return nil, fmt.Errorf("failed to uncompress zip file: %s", err)
 	}
 
 	// Create the template folder if it doesn't exists
-	os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm)
+	err = os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create template base folder: %s", err)
+	}
 
+	results, err := r.compareAndWriteTemplates(z)
+	if err != nil {
+		return nil, fmt.Errorf("failed to write templates: %s", err)
+	}
+
+	r.printUpdateChangelog(results, version)
+	checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
+	err = writeTemplatesChecksum(checksumFile, results.checksums)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not write checksum")
+	}
+
+	// Write the additions to a cached file for new runs.
+	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	buffer := &bytes.Buffer{}
+	for _, addition := range results.additions {
+		buffer.WriteString(addition)
+		buffer.WriteString("\n")
+	}
+	err = ioutil.WriteFile(additionsFile, buffer.Bytes(), os.ModePerm)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not write new additions file")
+	}
+	return results, err
+}
+
+type templateUpdateResults struct {
+	additions     []string
+	deletions     []string
+	modifications []string
+	totalCount    int
+	checksums     map[string]string
+}
+
+// compareAndWriteTemplates compares and returns the stats of a template
+// update operations.
+func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults, error) {
+	results := &templateUpdateResults{
+		checksums: make(map[string]string),
+	}
+
+	// We use file-checksums that are md5 hashes to store the list of files->hashes
+	// that have been downloaded previously.
+	// If the path isn't found in new update after being read from the previous checksum,
+	// it is removed. This allows us fine-grained control over the download process
+	// as well as solves a long problem with nuclei-template updates.
+	checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
+	previousChecksum, _ := readPreviousTemplatesChecksum(checksumFile)
 	for _, file := range z.File {
 		directory, name := filepath.Split(file.Name)
 		if name == "" {
@@ -256,52 +316,147 @@ func (r *Runner) downloadReleaseAndUnzip(downloadURL string) error {
 		paths := strings.Split(directory, "/")
 		finalPath := strings.Join(paths[1:], "/")
 
+		if (!strings.EqualFold(name, ".nuclei-ignore") && strings.HasPrefix(name, ".")) || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") {
+			continue
+		}
+		results.totalCount++
 		templateDirectory := path.Join(r.templatesConfig.TemplatesDirectory, finalPath)
-		os.MkdirAll(templateDirectory, os.ModePerm)
+		err := os.MkdirAll(templateDirectory, os.ModePerm)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err)
+		}
+
+		templatePath := path.Join(templateDirectory, name)
 
-		f, err := os.Create(path.Join(templateDirectory, name))
+		isAddition := false
+		if _, statErr := os.Stat(templatePath); os.IsNotExist(statErr) {
+			isAddition = true
+		}
+		f, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777)
 		if err != nil {
-			return fmt.Errorf("Could not create uncompressed file: %s", err)
+			f.Close()
+			return nil, fmt.Errorf("could not create uncompressed file: %s", err)
 		}
-		defer f.Close()
 
 		reader, err := file.Open()
 		if err != nil {
-			return fmt.Errorf("Could not open archive to extract file: %s", err)
+			f.Close()
+			return nil, fmt.Errorf("could not open archive to extract file: %s", err)
 		}
-		io.Copy(f, reader)
+		hasher := md5.New()
+
+		// Save file and also read into hasher for md5
+		_, err = io.Copy(f, io.TeeReader(reader, hasher))
+		if err != nil {
+			f.Close()
+			return nil, fmt.Errorf("could not write template file: %s", err)
+		}
+		f.Close()
+
+		oldChecksum, checksumOK := previousChecksum[templatePath]
+
+		checksum := hex.EncodeToString(hasher.Sum(nil))
+		if isAddition {
+			results.additions = append(results.additions, path.Join(finalPath, name))
+		} else if checksumOK && oldChecksum[0] != checksum {
+			results.modifications = append(results.modifications, path.Join(finalPath, name))
+		}
+		results.checksums[templatePath] = checksum
+	}
+
+	// If we don't find a previous file in new download and it hasn't been
+	// changed on the disk, delete it.
+	for k, v := range previousChecksum {
+		_, ok := results.checksums[k]
+		if !ok && v[0] == v[1] {
+			os.Remove(k)
+			results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(k, r.templatesConfig.TemplatesDirectory), "/"))
+		}
+	}
+	return results, nil
+}
+
+// readPreviousTemplatesChecksum reads the previous checksum file from the disk.
+//
+// It reads two checksums, the first checksum is what we expect and the second is
+// the actual checksum of the file on disk currently.
+func readPreviousTemplatesChecksum(file string) (map[string][2]string, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+
+	checksum := make(map[string][2]string)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text == "" {
+			continue
+		}
+		parts := strings.Split(text, ",")
+		if len(parts) < 2 {
+			continue
+		}
+		values := [2]string{parts[1]}
+
+		f, err := os.Open(parts[0])
+		if err != nil {
+			return nil, err
+		}
+
+		hasher := md5.New()
+		if _, err := io.Copy(hasher, f); err != nil {
+			return nil, err
+		}
+		f.Close()
+
+		values[1] = hex.EncodeToString(hasher.Sum(nil))
+		checksum[parts[0]] = values
+	}
+	return checksum, nil
+}
+
+// writeTemplatesChecksum writes the nuclei-templates checksum data to disk.
+func writeTemplatesChecksum(file string, checksum map[string]string) error {
+	f, err := os.Create(file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	builder := &strings.Builder{}
+	for k, v := range checksum {
+		builder.WriteString(k)
+		builder.WriteString(",")
+		builder.WriteString(v)
+		builder.WriteString("\n")
+
+		if _, checksumErr := f.WriteString(builder.String()); checksumErr != nil {
+			return err
+		}
+		builder.Reset()
 	}
 	return nil
 }
 
-// isRelative checks if a given path is a relative path
-func (r *Runner) isRelative(path string) bool {
-	if strings.HasPrefix(path, "/") || strings.Contains(path, ":\\") {
-		return false
-	}
-	return true
-}
+func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version string) {
+	if len(results.additions) > 0 {
+		gologger.Print().Msgf("\nNewly added templates: \n\n")
 
-// resolvePath gets the absolute path to the template by either
-// looking in the current directory or checking the nuclei templates directory.
-//
-// Current directory is given preference over the nuclei-templates directory.
-func (r *Runner) resolvePath(templateName string) (string, error) {
-	curDirectory, err := os.Getwd()
-	if err != nil {
-		return "", err
-	}
-	templatePath := path.Join(curDirectory, templateName)
-	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
-		gologger.Infof("Found template in current directory: %s\n", templatePath)
-		return templatePath, nil
-	}
-	if r.templatesConfig != nil {
-		templatePath := path.Join(r.templatesConfig.TemplatesDirectory, templateName)
-		if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
-			gologger.Infof("Found template in nuclei-templates directory: %s\n", templatePath)
-			return templatePath, nil
+		for _, addition := range results.additions {
+			gologger.Print().Msgf("%s", addition)
 		}
 	}
-	return "", fmt.Errorf("no such path found: %s", templateName)
+
+	gologger.Print().Msgf("\nNuclei Templates v%s Changelog\n", version)
+	data := [][]string{
+		{strconv.Itoa(results.totalCount), strconv.Itoa(len(results.additions)), strconv.Itoa(len(results.deletions))},
+	}
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Total", "Added", "Removed"})
+	for _, v := range data {
+		table.Append(v)
+	}
+	table.Render()
 }

v2/internal/runner/update_test.go

@@ -0,0 +1,156 @@
+package runner
+
+import (
+	"archive/zip"
+	"context"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
+	gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{})
+
+	baseTemplates, err := ioutil.TempDir("", "old-temp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(baseTemplates)
+
+	err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create write base file")
+
+	err = zipFromDirectory("base.zip", baseTemplates)
+	require.Nil(t, err, "could not create zip from directory")
+	defer os.Remove("base.zip")
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "base.zip")
+	}))
+	defer ts.Close()
+
+	templatesDirectory, err := ioutil.TempDir("", "template-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(templatesDirectory)
+
+	r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}}
+
+	results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
+	require.Nil(t, err, "could not download release and unzip")
+	require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")
+
+	newTempDir, err := ioutil.TempDir("", "new-tmp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(newTempDir)
+
+	err = ioutil.WriteFile(path.Join(newTempDir, "base.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create base file")
+	err = ioutil.WriteFile(path.Join(newTempDir, "new.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create new file")
+
+	err = zipFromDirectory("new.zip", newTempDir)
+	require.Nil(t, err, "could not create new zip from directory")
+	defer os.Remove("new.zip")
+
+	ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "new.zip")
+	}))
+	defer ts2.Close()
+
+	results, err = r.downloadReleaseAndUnzip(context.Background(), "1.0.1", ts2.URL)
+	require.Nil(t, err, "could not download release and unzip")
+
+	require.Equal(t, "new.yaml", results.additions[0], "could not get correct new addition")
+}
+
+func TestDownloadReleaseAndUnzipDeletion(t *testing.T) {
+	gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{})
+
+	baseTemplates, err := ioutil.TempDir("", "old-temp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(baseTemplates)
+
+	err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create write base file")
+
+	err = zipFromDirectory("base.zip", baseTemplates)
+	require.Nil(t, err, "could not create zip from directory")
+	defer os.Remove("base.zip")
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "base.zip")
+	}))
+	defer ts.Close()
+
+	templatesDirectory, err := ioutil.TempDir("", "template-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(templatesDirectory)
+
+	r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}}
+
+	results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
+	require.Nil(t, err, "could not download release and unzip")
+	require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")
+
+	newTempDir, err := ioutil.TempDir("", "new-tmp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(newTempDir)
+
+	err = zipFromDirectory("new.zip", newTempDir)
+	require.Nil(t, err, "could not create new zip from directory")
+	defer os.Remove("new.zip")
+
+	ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "new.zip")
+	}))
+	defer ts2.Close()
+
+	results, err = r.downloadReleaseAndUnzip(context.Background(), "1.0.1", ts2.URL)
+	require.Nil(t, err, "could not download release and unzip")
+
+	require.Equal(t, "base.yaml", results.deletions[0], "could not get correct new deletions")
+}
+
+func zipFromDirectory(zipPath, directory string) error {
+	file, err := os.Create(zipPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	w := zip.NewWriter(file)
+	defer w.Close()
+
+	walker := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		file, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		f, err := w.Create(strings.TrimPrefix(path, directory))
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(f, file)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+	return filepath.Walk(directory, walker)
+}

v2/internal/runner/validate.go

@@ -1,57 +0,0 @@
-package runner
-
-import (
-	"errors"
-	"net/url"
-
-	"github.com/projectdiscovery/gologger"
-)
-
-// validateOptions validates the configuration options passed
-func (options *Options) validateOptions() error {
-	// Both verbose and silent flags were used
-	if options.Verbose && options.Silent {
-		return errors.New("both verbose and silent mode specified")
-	}
-
-	// Check if a list of templates was provided and it exists
-	if options.Templates == "" && !options.UpdateTemplates {
-		return errors.New("no template/templates provided")
-	}
-
-	if options.Targets == "" && !options.Stdin && options.Target == "" && !options.UpdateTemplates {
-		return errors.New("no target input provided")
-	}
-
-	// Validate proxy options if provided
-	if options.ProxyURL != "" && !isValidProxyURL(options.ProxyURL) {
-		return errors.New("invalid http proxy format (It should be http://username:password@host:port)")
-	}
-	if options.ProxySocksURL != "" && !isValidProxyURL(options.ProxySocksURL) {
-		return errors.New("invalid socks proxy format (It should be socks5://username:password@host:port)")
-	}
-
-	return nil
-}
-
-func isValidProxyURL(URL string) bool {
-	if _, err := url.Parse(URL); err != nil {
-		return false
-	}
-
-	return true
-}
-
-// configureOutput configures the output on the screen
-func (options *Options) configureOutput() {
-	// If the user desires verbose output, show verbose output
-	if options.Verbose {
-		gologger.MaxLevel = gologger.Verbose
-	}
-	if options.NoColor {
-		gologger.UseColors = false
-	}
-	if options.Silent {
-		gologger.MaxLevel = gologger.Silent
-	}
-}

v2/internal/testutils/integration.go

@@ -0,0 +1,96 @@
+package testutils
+
+import (
+	"net"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+// RunNucleiAndGetResults returns a list of results for a template
+func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
+	cmd := exec.Command("./nuclei", "-t", template, "-target", url)
+	if debug {
+		cmd = exec.Command("./nuclei", "-t", template, "-target", url, "-debug")
+		cmd.Stderr = os.Stderr
+	}
+	cmd.Args = append(cmd.Args, extra...)
+
+	data, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+	parts := []string{}
+	items := strings.Split(string(data), "\n")
+	for _, i := range items {
+		if i != "" {
+			parts = append(parts, i)
+		}
+	}
+	return parts, nil
+}
+
+// RunNucleiWorkflowAndGetResults returns a list of results for a workflow
+func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
+	cmd := exec.Command("./nuclei", "-w", template, "-target", url)
+	if debug {
+		cmd = exec.Command("./nuclei", "-w", template, "-target", url, "-debug")
+		cmd.Stderr = os.Stderr
+	}
+	cmd.Args = append(cmd.Args, extra...)
+
+	data, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+	parts := []string{}
+	items := strings.Split(string(data), "\n")
+	for _, i := range items {
+		if i != "" {
+			parts = append(parts, i)
+		}
+	}
+	return parts, nil
+}
+
+// TestCase is a single integration test case
+type TestCase interface {
+	// Execute executes a test case and returns any errors if occurred
+	Execute(filePath string) error
+}
+
+// TCPServer creates a new tcp server that returns a response
+type TCPServer struct {
+	URL      string
+	listener net.Listener
+}
+
+// NewTCPServer creates a new TCP server from a handler
+func NewTCPServer(handler func(conn net.Conn)) *TCPServer {
+	server := &TCPServer{}
+
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		panic(err)
+	}
+	server.URL = l.Addr().String()
+	server.listener = l
+
+	go func() {
+		for {
+			// Listen for an incoming connection.
+			conn, err := l.Accept()
+			if err != nil {
+				continue
+			}
+			// Handle connections in a new goroutine.
+			go handler(conn)
+		}
+	}()
+	return server
+}
+
+// Close closes the TCP server
+func (s *TCPServer) Close() {
+	s.listener.Close()
+}

v2/internal/testutils/testutils.go

@@ -0,0 +1,127 @@
+package testutils
+
+import (
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"go.uber.org/ratelimit"
+)
+
+// Init initializes the protocols and their configurations
+func Init(options *types.Options) {
+	_ = protocolinit.Init(options)
+}
+
+// DefaultOptions is the default options structure for nuclei during mocking.
+var DefaultOptions = &types.Options{
+	RandomAgent:          false,
+	Metrics:              false,
+	Debug:                false,
+	DebugRequests:        false,
+	DebugResponse:        false,
+	Silent:               false,
+	Version:              false,
+	Verbose:              false,
+	NoColor:              true,
+	UpdateTemplates:      false,
+	JSON:                 false,
+	JSONRequests:         false,
+	EnableProgressBar:    false,
+	TemplatesVersion:     false,
+	TemplateList:         false,
+	Stdin:                false,
+	StopAtFirstMatch:     false,
+	NoMeta:               false,
+	Project:              false,
+	MetricsPort:          0,
+	BulkSize:             25,
+	TemplateThreads:      10,
+	Timeout:              5,
+	Retries:              1,
+	RateLimit:            150,
+	BurpCollaboratorBiid: "",
+	ProjectPath:          "",
+	Severity:             []string{},
+	Target:               "",
+	Targets:              "",
+	Output:               "",
+	ProxyURL:             "",
+	ProxySocksURL:        "",
+	TemplatesDirectory:   "",
+	TraceLogFile:         "",
+	Templates:            []string{},
+	ExcludedTemplates:    []string{},
+	CustomHeaders:        []string{},
+}
+
+// MockOutputWriter is a mocked output writer.
+type MockOutputWriter struct {
+	aurora          aurora.Aurora
+	RequestCallback func(templateID, url, requestType string, err error)
+	WriteCallback   func(o *output.ResultEvent)
+}
+
+// NewMockOutputWriter creates a new mock output writer
+func NewMockOutputWriter() *MockOutputWriter {
+	return &MockOutputWriter{aurora: aurora.NewAurora(false)}
+}
+
+// Close closes the output writer interface
+func (m *MockOutputWriter) Close() {}
+
+// Colorizer returns the colorizer instance for writer
+func (m *MockOutputWriter) Colorizer() aurora.Aurora {
+	return m.aurora
+}
+
+// Write writes the event to file and/or screen.
+func (m *MockOutputWriter) Write(result *output.ResultEvent) error {
+	if m.WriteCallback != nil {
+		m.WriteCallback(result)
+	}
+	return nil
+}
+
+// Request writes a log the requests trace log
+func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) {
+	if m.RequestCallback != nil {
+		m.RequestCallback(templateID, url, requestType, err)
+	}
+}
+
+// TemplateInfo contains info for a mock executed template.
+type TemplateInfo struct {
+	ID   string
+	Info map[string]interface{}
+	Path string
+}
+
+// NewMockExecuterOptions creates a new mock executeroptions struct
+func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecuterOptions {
+	progressImpl, _ := progress.NewStatsTicker(0, false, false, 0)
+	executerOpts := &protocols.ExecuterOptions{
+		TemplateID:   info.ID,
+		TemplateInfo: info.Info,
+		TemplatePath: info.Path,
+		Output:       NewMockOutputWriter(),
+		Options:      options,
+		Progress:     progressImpl,
+		ProjectFile:  nil,
+		IssuesClient: nil,
+		Browser:      nil,
+		Catalog:      catalog.New(options.TemplatesDirectory),
+		RateLimiter:  ratelimit.New(options.RateLimit),
+	}
+	return executerOpts
+}
+
+// NoopWriter is a NooP gologger writer.
+type NoopWriter struct{}
+
+// Write writes the data to an output writer.
+func (n *NoopWriter) Write(data []byte, level levels.Level) {}

v2/pkg/catalog/catalogue.go

@@ -0,0 +1,14 @@
+package catalog
+
+// Catalog is a template catalog helper implementation
+type Catalog struct {
+	ignoreFiles        []string
+	templatesDirectory string
+}
+
+// New creates a new Catalog structure using provided input items
+func New(directory string) *Catalog {
+	catalog := &Catalog{templatesDirectory: directory}
+	catalog.readNucleiIgnoreFile()
+	return catalog
+}

v2/pkg/catalog/find.go

@@ -0,0 +1,155 @@
+package catalog
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/karrick/godirwalk"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+)
+
+// GetTemplatesPath returns a list of absolute paths for the provided template list.
+func (c *Catalog) GetTemplatesPath(definitions []string) []string {
+	// keeps track of processed dirs and files
+	processed := make(map[string]bool)
+	allTemplates := []string{}
+
+	for _, t := range definitions {
+		paths, err := c.GetTemplatePath(t)
+		if err != nil {
+			gologger.Error().Msgf("Could not find template '%s': %s\n", t, err)
+		}
+		for _, path := range paths {
+			if _, ok := processed[path]; !ok {
+				processed[path] = true
+				allTemplates = append(allTemplates, path)
+			}
+		}
+	}
+	if len(allTemplates) > 0 {
+		gologger.Verbose().Msgf("Identified %d templates", len(allTemplates))
+	}
+	return allTemplates
+}
+
+// GetTemplatePath parses the specified input template path and returns a compiled
+// list of finished absolute paths to the templates evaluating any glob patterns
+// or folders provided as in.
+func (c *Catalog) GetTemplatePath(target string) ([]string, error) {
+	processed := make(map[string]struct{})
+
+	absPath, err := c.convertPathToAbsolute(target)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not find template file")
+	}
+
+	// Template input includes a wildcard
+	if strings.Contains(absPath, "*") {
+		matches, findErr := c.findGlobPathMatches(absPath, processed)
+		if findErr != nil {
+			return nil, errors.Wrap(findErr, "could not find glob matches")
+		}
+		if len(matches) == 0 {
+			return nil, errors.Errorf("no templates found for path")
+		}
+		return matches, nil
+	}
+
+	// Template input is either a file or a directory
+	match, file, err := c.findFileMatches(absPath, processed)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not find file")
+	}
+	if file {
+		if match != "" {
+			return []string{match}, nil
+		}
+		return nil, nil
+	}
+
+	// Recursively walk down the Templates directory and run all
+	// the template file checks
+	matches, err := c.findDirectoryMatches(absPath, processed)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not find directory matches")
+	}
+	if len(matches) == 0 {
+		return nil, errors.Errorf("no templates found in path")
+	}
+	return matches, nil
+}
+
+// convertPathToAbsolute resolves the paths provided to absolute paths
+// before doing any operations on them regardless of them being blob, folders, files, etc.
+func (c *Catalog) convertPathToAbsolute(t string) (string, error) {
+	if strings.Contains(t, "*") {
+		file := path.Base(t)
+		absPath, err := c.ResolvePath(path.Dir(t), "")
+		if err != nil {
+			return "", err
+		}
+		return path.Join(absPath, file), nil
+	}
+	return c.ResolvePath(t, "")
+}
+
+// findGlobPathMatches returns the matched files from a glob path
+func (c *Catalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) {
+	matches, err := filepath.Glob(absPath)
+	if err != nil {
+		return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
+	}
+	results := make([]string, 0, len(matches))
+	for _, match := range matches {
+		if _, ok := processed[match]; !ok {
+			processed[match] = struct{}{}
+			results = append(results, match)
+		}
+	}
+	return results, nil
+}
+
+// findFileMatches finds if a path is an absolute file. If the path
+// is a file, it returns true otherwise false with no errors.
+func (c *Catalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) {
+	info, err := os.Stat(absPath)
+	if err != nil {
+		return "", false, err
+	}
+	if !info.Mode().IsRegular() {
+		return "", false, nil
+	}
+	if _, ok := processed[absPath]; !ok {
+		processed[absPath] = struct{}{}
+		return absPath, true, nil
+	}
+	return "", true, nil
+}
+
+// findDirectoryMatches finds matches for templates from a directory
+func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) {
+	var results []string
+	err := godirwalk.Walk(absPath, &godirwalk.Options{
+		Unsorted: true,
+		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
+			return godirwalk.SkipNode
+		},
+		Callback: func(path string, d *godirwalk.Dirent) error {
+			if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
+				if c.checkIfInNucleiIgnore(path) {
+					return nil
+				}
+
+				if _, ok := processed[path]; !ok {
+					results = append(results, path)
+					processed[path] = struct{}{}
+				}
+			}
+			return nil
+		},
+	})
+	return results, err
+}

v2/pkg/catalog/ignore.go

@@ -0,0 +1,84 @@
+package catalog
+
+import (
+	"bufio"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/projectdiscovery/gologger"
+)
+
+const nucleiIgnoreFile = ".nuclei-ignore"
+
+// readNucleiIgnoreFile reads the nuclei ignore file marking it in map
+func (c *Catalog) readNucleiIgnoreFile() {
+	file, err := os.Open(path.Join(c.templatesDirectory, nucleiIgnoreFile))
+	if err != nil {
+		return
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text == "" {
+			continue
+		}
+		if strings.HasPrefix(text, "#") {
+			continue
+		}
+		c.ignoreFiles = append(c.ignoreFiles, text)
+	}
+}
+
+// checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules.
+func (c *Catalog) checkIfInNucleiIgnore(item string) bool {
+	if c.templatesDirectory == "" {
+		return false
+	}
+
+	matched := false
+	for _, paths := range c.ignoreFiles {
+		if !strings.HasSuffix(paths, ".yaml") {
+			if strings.HasSuffix(strings.TrimSuffix(item, "/"), strings.TrimSuffix(paths, "/")) {
+				matched = true
+				break
+			}
+		} else if strings.HasSuffix(item, paths) {
+			matched = true
+			break
+		}
+	}
+	if matched {
+		gologger.Error().Msgf("Excluding %s due to nuclei-ignore filter", item)
+		return true
+	}
+	return false
+}
+
+// ignoreFilesWithExcludes ignores results with exclude paths
+func (c *Catalog) ignoreFilesWithExcludes(results, excluded []string) []string {
+	var templates []string
+
+	for _, result := range results {
+		matched := false
+		for _, paths := range excluded {
+			if !strings.HasSuffix(paths, ".yaml") {
+				if strings.HasSuffix(strings.TrimSuffix(result, "/"), strings.TrimSuffix(paths, "/")) {
+					matched = true
+					break
+				}
+			} else if strings.HasSuffix(result, paths) {
+				matched = true
+				break
+			}
+		}
+		if !matched {
+			templates = append(templates, result)
+		} else {
+			gologger.Error().Msgf("Excluding %s due to excludes filter", result)
+		}
+	}
+	return templates
+}

v2/pkg/catalog/ignore_test.go

@@ -0,0 +1,40 @@
+package catalog
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestIgnoreFilesIgnore(t *testing.T) {
+	c := &Catalog{
+		ignoreFiles:        []string{"workflows/", "cves/2020/cve-2020-5432.yaml"},
+		templatesDirectory: "test",
+	}
+	tests := []struct {
+		path   string
+		ignore bool
+	}{
+		{"workflows/", true},
+		{"misc", false},
+		{"cves/", false},
+		{"cves/2020/cve-2020-5432.yaml", true},
+		{"/Users/test/nuclei-templates/workflows/", true},
+		{"/Users/test/nuclei-templates/misc", false},
+		{"/Users/test/nuclei-templates/cves/", false},
+		{"/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", true},
+	}
+	for _, test := range tests {
+		require.Equal(t, test.ignore, c.checkIfInNucleiIgnore(test.path), fmt.Sprintf("could not ignore file correctly: %v", test))
+	}
+}
+
+func TestExcludeFilesIgnore(t *testing.T) {
+	c := &Catalog{}
+	excludes := []string{"workflows/", "cves/2020/cve-2020-5432.yaml"}
+	paths := []string{"/Users/test/nuclei-templates/workflows/", "/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", "/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"}
+
+	data := c.ignoreFilesWithExcludes(paths, excludes)
+	require.Equal(t, []string{"/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"}, data, "could not exclude correct files")
+}

v2/pkg/catalog/path.go

@@ -0,0 +1,45 @@
+package catalog
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+// ResolvePath resolves the path to an absolute one in various ways.
+//
+// It checks if the filename is an absolute path, looks in the current directory
+// or checking the nuclei templates directory. If a second path is given,
+// it also tries to find paths relative to that second path.
+func (c *Catalog) ResolvePath(templateName, second string) (string, error) {
+	if strings.HasPrefix(templateName, "/") || strings.Contains(templateName, ":\\") {
+		return templateName, nil
+	}
+
+	if second != "" {
+		secondBasePath := path.Join(filepath.Dir(second), templateName)
+		if _, err := os.Stat(secondBasePath); !os.IsNotExist(err) {
+			return secondBasePath, nil
+		}
+	}
+
+	curDirectory, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+
+	templatePath := path.Join(curDirectory, templateName)
+	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
+		return templatePath, nil
+	}
+
+	if c.templatesDirectory != "" {
+		templatePath := path.Join(c.templatesDirectory, templateName)
+		if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
+			return templatePath, nil
+		}
+	}
+	return "", fmt.Errorf("no such path found: %s", templateName)
+}

v2/pkg/executor/dns_utils.go

@@ -1,38 +0,0 @@
-package executor
-
-import (
-	"net/url"
-
-	"github.com/asaskevich/govalidator"
-)
-
-// isURL tests a string to determine if it is a well-structured url or not.
-func isURL(toTest string) bool {
-	_, err := url.ParseRequestURI(toTest)
-	if err != nil {
-		return false
-	}
-
-	u, err := url.Parse(toTest)
-	if err != nil || u.Scheme == "" || u.Host == "" {
-		return false
-	}
-	return true
-}
-
-// extractDomain extracts the domain name of a URL
-func extractDomain(URL string) string {
-	u, err := url.Parse(URL)
-	if err != nil {
-		return ""
-	}
-	hostname := u.Hostname()
-	return hostname
-}
-
-// isDNS tests a string to determine if it is a well-structured dns or not
-// even if it's oneliner, we leave it wrapped in a function call for
-// future improvements
-func isDNS(toTest string) bool {
-	return govalidator.IsDNSName(toTest)
-}

v2/pkg/executor/executer_http.go

@@ -1,282 +0,0 @@
-package executor
-
-import (
-	"bufio"
-	"crypto/tls"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/http/httputil"
-	"net/url"
-	"os"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/retryablehttp-go"
-	"golang.org/x/net/proxy"
-)
-
-// HTTPExecutor is client for performing HTTP requests
-// for a template.
-type HTTPExecutor struct {
-	debug         bool
-	results       uint32
-	jsonOutput    bool
-	httpClient    *retryablehttp.Client
-	template      *templates.Template
-	httpRequest   *requests.HTTPRequest
-	writer        *bufio.Writer
-	outputMutex   *sync.Mutex
-	customHeaders requests.CustomHeaders
-}
-
-// HTTPOptions contains configuration options for the HTTP executor.
-type HTTPOptions struct {
-	Template      *templates.Template
-	HTTPRequest   *requests.HTTPRequest
-	Writer        *bufio.Writer
-	Timeout       int
-	Retries       int
-	ProxyURL      string
-	ProxySocksURL string
-	Debug         bool
-	JSON          bool
-	CustomHeaders requests.CustomHeaders
-}
-
-// NewHTTPExecutor creates a new HTTP executor from a template
-// and a HTTP request query.
-func NewHTTPExecutor(options *HTTPOptions) (*HTTPExecutor, error) {
-	var proxyURL *url.URL
-	var err error
-
-	if options.ProxyURL != "" {
-		proxyURL, err = url.Parse(options.ProxyURL)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	// Create the HTTP Client
-	client := makeHTTPClient(proxyURL, options)
-	client.CheckRetry = retryablehttp.HostSprayRetryPolicy()
-
-	executer := &HTTPExecutor{
-		debug:         options.Debug,
-		jsonOutput:    options.JSON,
-		results:       0,
-		httpClient:    client,
-		template:      options.Template,
-		httpRequest:   options.HTTPRequest,
-		outputMutex:   &sync.Mutex{},
-		writer:        options.Writer,
-		customHeaders: options.CustomHeaders,
-	}
-	return executer, nil
-}
-
-// GotResults returns true if there were any results for the executor
-func (e *HTTPExecutor) GotResults() bool {
-	if atomic.LoadUint32(&e.results) == 0 {
-		return false
-	}
-	return true
-}
-
-// ExecuteHTTP executes the HTTP request on a URL
-func (e *HTTPExecutor) ExecuteHTTP(URL string) error {
-	// Compile each request for the template based on the URL
-	compiledRequest, err := e.httpRequest.MakeHTTPRequest(URL)
-	if err != nil {
-		return errors.Wrap(err, "could not make http request")
-	}
-
-	// Send the request to the target servers
-mainLoop:
-	for compiledRequest := range compiledRequest {
-		if compiledRequest.Error != nil {
-			return errors.Wrap(err, "could not make http request")
-		}
-		e.setCustomHeaders(compiledRequest)
-		req := compiledRequest.Request
-
-		if e.debug {
-			gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID)
-			dumpedRequest, err := httputil.DumpRequest(req.Request, true)
-			if err != nil {
-				return errors.Wrap(err, "could not dump http request")
-			}
-			fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
-		}
-
-		resp, err := e.httpClient.Do(req)
-		if err != nil {
-			if resp != nil {
-				resp.Body.Close()
-			}
-			gologger.Warningf("Could not do request: %s\n", err)
-			continue
-		}
-
-		if e.debug {
-			gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID)
-			dumpedResponse, err := httputil.DumpResponse(resp, true)
-			if err != nil {
-				return errors.Wrap(err, "could not dump http response")
-			}
-			fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
-		}
-
-		data, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			io.Copy(ioutil.Discard, resp.Body)
-			resp.Body.Close()
-			return errors.Wrap(err, "could not read http body")
-		}
-		resp.Body.Close()
-
-		// net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request
-		// so in case we have to manually do it
-		data, err = requests.HandleDecompression(compiledRequest.Request, data)
-		if err != nil {
-			return errors.Wrap(err, "could not decompress http body")
-		}
-
-		// Convert response body from []byte to string with zero copy
-		body := unsafeToString(data)
-
-		var headers string
-		matcherCondition := e.httpRequest.GetMatchersCondition()
-		for _, matcher := range e.httpRequest.Matchers {
-			headers = headersToString(resp.Header)
-			// Check if the matcher matched
-			if !matcher.Match(resp, body, headers) {
-				// If the condition is AND we haven't matched, try next request.
-				if matcherCondition == matchers.ANDCondition {
-					continue mainLoop
-				}
-			} else {
-				// If the matcher has matched, and its an OR
-				// write the first output then move to next matcher.
-				if matcherCondition == matchers.ORCondition && len(e.httpRequest.Extractors) == 0 {
-					e.writeOutputHTTP(compiledRequest, matcher, nil)
-					atomic.CompareAndSwapUint32(&e.results, 0, 1)
-				}
-			}
-		}
-
-		// All matchers have successfully completed so now start with the
-		// next task which is extraction of input from matchers.
-		var extractorResults []string
-		for _, extractor := range e.httpRequest.Extractors {
-			headers = headersToString(resp.Header)
-			for match := range extractor.Extract(body, headers) {
-				extractorResults = append(extractorResults, match)
-			}
-		}
-
-		// Write a final string of output if matcher type is
-		// AND or if we have extractors for the mechanism too.
-		if len(e.httpRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
-			e.writeOutputHTTP(compiledRequest, nil, extractorResults)
-			atomic.CompareAndSwapUint32(&e.results, 0, 1)
-		}
-	}
-
-	gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL)
-
-	return nil
-}
-
-// Close closes the http executor for a template.
-func (e *HTTPExecutor) Close() {
-	e.outputMutex.Lock()
-	e.writer.Flush()
-	e.outputMutex.Unlock()
-}
-
-// makeHTTPClient creates a http client
-func makeHTTPClient(proxyURL *url.URL, options *HTTPOptions) *retryablehttp.Client {
-	retryablehttpOptions := retryablehttp.DefaultOptionsSpraying
-	retryablehttpOptions.RetryWaitMax = 10 * time.Second
-	retryablehttpOptions.RetryMax = options.Retries
-	followRedirects := options.HTTPRequest.Redirects
-	maxRedirects := options.HTTPRequest.MaxRedirects
-
-	transport := &http.Transport{
-		MaxIdleConnsPerHost: -1,
-		TLSClientConfig: &tls.Config{
-			Renegotiation:      tls.RenegotiateOnceAsClient,
-			InsecureSkipVerify: true,
-		},
-		DisableKeepAlives: true,
-	}
-
-	// Attempts to overwrite the dial function with the socks proxied version
-	if options.ProxySocksURL != "" {
-		var proxyAuth *proxy.Auth
-		socksURL, err := url.Parse(options.ProxySocksURL)
-		if err == nil {
-			proxyAuth = &proxy.Auth{}
-			proxyAuth.User = socksURL.User.Username()
-			proxyAuth.Password, _ = socksURL.User.Password()
-		}
-		dialer, err := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct)
-		if err == nil {
-			transport.Dial = dialer.Dial
-		}
-	}
-
-	if proxyURL != nil {
-		transport.Proxy = http.ProxyURL(proxyURL)
-	}
-	return retryablehttp.NewWithHTTPClient(&http.Client{
-		Transport:     transport,
-		Timeout:       time.Duration(options.Timeout) * time.Second,
-		CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects),
-	}, retryablehttpOptions)
-}
-
-type checkRedirectFunc func(_ *http.Request, requests []*http.Request) error
-
-func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirectFunc {
-	return func(_ *http.Request, requests []*http.Request) error {
-		if !followRedirects {
-			return http.ErrUseLastResponse
-		}
-		if maxRedirects == 0 {
-			if len(requests) > 10 {
-				return http.ErrUseLastResponse
-			}
-			return nil
-		}
-		if len(requests) > maxRedirects {
-			return http.ErrUseLastResponse
-		}
-		return nil
-	}
-}
-
-func (e *HTTPExecutor) setCustomHeaders(r *requests.CompiledHTTP) {
-	for _, customHeader := range e.customHeaders {
-		// This should be pre-computed somewhere and done only once
-		tokens := strings.Split(customHeader, ":")
-		// if it's an invalid header skip it
-		if len(tokens) < 2 {
-			continue
-		}
-
-		headerName, headerValue := tokens[0], strings.Join(tokens[1:], "")
-		headerName = strings.TrimSpace(headerName)
-		headerValue = strings.TrimSpace(headerValue)
-		r.Request.Header.Set(headerName, headerValue)
-	}
-}

v2/pkg/executor/executor_dns.go

@@ -1,149 +0,0 @@
-package executor
-
-import (
-	"bufio"
-	"fmt"
-	"os"
-	"sync"
-	"sync/atomic"
-
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	retryabledns "github.com/projectdiscovery/retryabledns"
-)
-
-// DNSExecutor is a client for performing a DNS request
-// for a template.
-type DNSExecutor struct {
-	debug       bool
-	jsonOutput  bool
-	results     uint32
-	dnsClient   *retryabledns.Client
-	template    *templates.Template
-	dnsRequest  *requests.DNSRequest
-	writer      *bufio.Writer
-	outputMutex *sync.Mutex
-}
-
-// DefaultResolvers contains the list of resolvers known to be trusted.
-var DefaultResolvers = []string{
-	"1.1.1.1:53", // Cloudflare
-	"1.0.0.1:53", // Cloudflare
-	"8.8.8.8:53", // Google
-	"8.8.4.4:53", // Google
-}
-
-// DNSOptions contains configuration options for the DNS executor.
-type DNSOptions struct {
-	Debug      bool
-	JSON       bool
-	Template   *templates.Template
-	DNSRequest *requests.DNSRequest
-	Writer     *bufio.Writer
-}
-
-// NewDNSExecutor creates a new DNS executor from a template
-// and a DNS request query.
-func NewDNSExecutor(options *DNSOptions) *DNSExecutor {
-	dnsClient := retryabledns.New(DefaultResolvers, options.DNSRequest.Retries)
-
-	executer := &DNSExecutor{
-		debug:       options.Debug,
-		jsonOutput:  options.JSON,
-		results:     0,
-		dnsClient:   dnsClient,
-		template:    options.Template,
-		dnsRequest:  options.DNSRequest,
-		writer:      options.Writer,
-		outputMutex: &sync.Mutex{},
-	}
-	return executer
-}
-
-// GotResults returns true if there were any results for the executor
-func (e *DNSExecutor) GotResults() bool {
-	if atomic.LoadUint32(&e.results) == 0 {
-		return false
-	}
-	return true
-}
-
-// ExecuteDNS executes the DNS request on a URL
-func (e *DNSExecutor) ExecuteDNS(URL string) error {
-	// Parse the URL and return domain if URL.
-	var domain string
-	if isURL(URL) {
-		domain = extractDomain(URL)
-	} else {
-		domain = URL
-	}
-
-	// Compile each request for the template based on the URL
-	compiledRequest, err := e.dnsRequest.MakeDNSRequest(domain)
-	if err != nil {
-		return errors.Wrap(err, "could not make dns request")
-	}
-
-	if e.debug {
-		gologger.Infof("Dumped DNS request for %s (%s)\n\n", URL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s\n", compiledRequest.String())
-	}
-
-	// Send the request to the target servers
-	resp, err := e.dnsClient.Do(compiledRequest)
-	if err != nil {
-		return errors.Wrap(err, "could not send dns request")
-	}
-
-	gologger.Verbosef("Sent DNS request to %s\n", "dns-request", URL)
-
-	if e.debug {
-		gologger.Infof("Dumped DNS response for %s (%s)\n\n", URL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s\n", resp.String())
-	}
-
-	matcherCondition := e.dnsRequest.GetMatchersCondition()
-	for _, matcher := range e.dnsRequest.Matchers {
-		// Check if the matcher matched
-		if !matcher.MatchDNS(resp) {
-			// If the condition is AND we haven't matched, return.
-			if matcherCondition == matchers.ANDCondition {
-				return nil
-			}
-		} else {
-			// If the matcher has matched, and its an OR
-			// write the first output then move to next matcher.
-			if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 {
-				e.writeOutputDNS(domain, matcher, nil)
-				atomic.CompareAndSwapUint32(&e.results, 0, 1)
-			}
-		}
-	}
-
-	// All matchers have successfully completed so now start with the
-	// next task which is extraction of input from matchers.
-	var extractorResults []string
-	for _, extractor := range e.dnsRequest.Extractors {
-		for match := range extractor.ExtractDNS(resp.String()) {
-			extractorResults = append(extractorResults, match)
-		}
-	}
-
-	// Write a final string of output if matcher type is
-	// AND or if we have extractors for the mechanism too.
-	if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
-		e.writeOutputDNS(domain, nil, extractorResults)
-		atomic.CompareAndSwapUint32(&e.results, 0, 1)
-	}
-	return nil
-}
-
-// Close closes the dns executor for a template.
-func (e *DNSExecutor) Close() {
-	e.outputMutex.Lock()
-	e.writer.Flush()
-	e.outputMutex.Unlock()
-}

v2/pkg/executor/http_utils.go

@@ -1,41 +0,0 @@
-package executor
-
-import (
-	"net/http"
-	"strings"
-	"unsafe"
-)
-
-type jsonOutput struct {
-	Template         string   `json:"template"`
-	Type             string   `json:"type"`
-	Matched          string   `json:"matched"`
-	MatcherName      string   `json:"matcher_name,omitempty"`
-	ExtractedResults []string `json:"extracted_results,omitempty"`
-	Severity         string   `json:"severity"`
-	Author           string   `json:"author"`
-}
-
-// unsafeToString converts byte slice to string with zero allocations
-func unsafeToString(bs []byte) string {
-	return *(*string)(unsafe.Pointer(&bs))
-}
-
-// headersToString converts http headers to string
-func headersToString(headers http.Header) string {
-	builder := &strings.Builder{}
-
-	for header, values := range headers {
-		builder.WriteString(header)
-		builder.WriteString(": ")
-
-		for i, value := range values {
-			builder.WriteString(value)
-			if i != len(values)-1 {
-				builder.WriteRune(',')
-			}
-		}
-		builder.WriteRune('\n')
-	}
-	return builder.String()
-}

v2/pkg/executor/output_dns.go

@@ -1,76 +0,0 @@
-package executor
-
-import (
-	"strings"
-
-	jsoniter "github.com/json-iterator/go"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-)
-
-// writeOutputDNS writes dns output to streams
-func (e *DNSExecutor) writeOutputDNS(domain string, matcher *matchers.Matcher, extractorResults []string) {
-	if e.jsonOutput {
-		output := jsonOutput{
-			Template: e.template.ID,
-			Type:     "dns",
-			Matched:  domain,
-			Severity: e.template.Info.Severity,
-			Author:   e.template.Info.Author,
-		}
-		if matcher != nil && len(matcher.Name) > 0 {
-			output.MatcherName = matcher.Name
-		}
-		if len(extractorResults) > 0 {
-			output.ExtractedResults = extractorResults
-		}
-		data, err := jsoniter.Marshal(output)
-		if err != nil {
-			gologger.Warningf("Could not marshal json output: %s\n", err)
-		}
-
-		gologger.Silentf("%s", string(data))
-
-		if e.writer != nil {
-			e.outputMutex.Lock()
-			e.writer.Write(data)
-			e.writer.WriteRune('\n')
-			e.outputMutex.Unlock()
-		}
-		return
-	}
-
-	builder := &strings.Builder{}
-	builder.WriteRune('[')
-	builder.WriteString(e.template.ID)
-	if matcher != nil && len(matcher.Name) > 0 {
-		builder.WriteString(":")
-		builder.WriteString(matcher.Name)
-	}
-	builder.WriteString("] [dns] ")
-
-	builder.WriteString(domain)
-
-	// If any extractors, write the results
-	if len(extractorResults) > 0 {
-		builder.WriteString(" [")
-		for i, result := range extractorResults {
-			builder.WriteString(result)
-			if i != len(extractorResults)-1 {
-				builder.WriteRune(',')
-			}
-		}
-		builder.WriteString("]")
-	}
-	builder.WriteRune('\n')
-
-	// Write output to screen as well as any output file
-	message := builder.String()
-	gologger.Silentf("%s", message)
-
-	if e.writer != nil {
-		e.outputMutex.Lock()
-		e.writer.WriteString(message)
-		e.outputMutex.Unlock()
-	}
-}

v2/pkg/executor/output_http.go

@@ -1,94 +0,0 @@
-package executor
-
-import (
-	"strings"
-
-	jsoniter "github.com/json-iterator/go"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-)
-
-// writeOutputHTTP writes http output to streams
-func (e *HTTPExecutor) writeOutputHTTP(req *requests.CompiledHTTP, matcher *matchers.Matcher, extractorResults []string) {
-	URL := req.Request.URL.String()
-
-	if e.jsonOutput {
-		output := jsonOutput{
-			Template: e.template.ID,
-			Type:     "http",
-			Matched:  URL,
-			Severity: e.template.Info.Severity,
-			Author:   e.template.Info.Author,
-		}
-		if matcher != nil && len(matcher.Name) > 0 {
-			output.MatcherName = matcher.Name
-		}
-		if len(extractorResults) > 0 {
-			output.ExtractedResults = extractorResults
-		}
-		data, err := jsoniter.Marshal(output)
-		if err != nil {
-			gologger.Warningf("Could not marshal json output: %s\n", err)
-		}
-
-		gologger.Silentf("%s", string(data))
-
-		if e.writer != nil {
-			e.outputMutex.Lock()
-			e.writer.Write(data)
-			e.writer.WriteRune('\n')
-			e.outputMutex.Unlock()
-		}
-		return
-	}
-
-	builder := &strings.Builder{}
-
-	builder.WriteRune('[')
-	builder.WriteString(e.template.ID)
-	if matcher != nil && len(matcher.Name) > 0 {
-		builder.WriteString(":")
-		builder.WriteString(matcher.Name)
-	}
-	builder.WriteString("] [http] ")
-
-	// Escape the URL by replacing all % with %%
-	escapedURL := strings.Replace(URL, "%", "%%", -1)
-	builder.WriteString(escapedURL)
-
-	// If any extractors, write the results
-	if len(extractorResults) > 0 {
-		builder.WriteString(" [")
-		for i, result := range extractorResults {
-			builder.WriteString(result)
-			if i != len(extractorResults)-1 {
-				builder.WriteRune(',')
-			}
-		}
-		builder.WriteString("]")
-	}
-
-	// write meta if any
-	if len(req.Meta) > 0 {
-		builder.WriteString(" [")
-		var metas []string
-		for name, value := range req.Meta {
-			metas = append(metas, name+"="+value.(string))
-		}
-		builder.WriteString(strings.Join(metas, ","))
-		builder.WriteString("]")
-	}
-
-	builder.WriteRune('\n')
-
-	// Write output to screen as well as any output file
-	message := builder.String()
-	gologger.Silentf("%s", message)
-
-	if e.writer != nil {
-		e.outputMutex.Lock()
-		e.writer.WriteString(message)
-		e.outputMutex.Unlock()
-	}
-}

v2/pkg/extractors/extract.go

@@ -1,36 +0,0 @@
-package extractors
-
-// Extract extracts response from the parts of request using a regex
-func (e *Extractor) Extract(body, headers string) map[string]struct{} {
-	// Match the parts as required for regex check
-	if e.part == BodyPart {
-		return e.extractRegex(body)
-	} else if e.part == HeaderPart {
-		return e.extractRegex(headers)
-	} else {
-		matches := e.extractRegex(headers)
-		if len(matches) > 0 {
-			return matches
-		}
-		return e.extractRegex(body)
-	}
-}
-
-// ExtractDNS extracts response from dns message using a regex
-func (e *Extractor) ExtractDNS(msg string) map[string]struct{} {
-	// Match the parts as required for regex check
-	return e.extractRegex(msg)
-}
-
-// extractRegex extracts text from a corpus and returns it
-func (e *Extractor) extractRegex(corpus string) map[string]struct{} {
-	results := make(map[string]struct{})
-
-	for _, regex := range e.regexCompiled {
-		matches := regex.FindAllString(corpus, -1)
-		for _, match := range matches {
-			results[match] = struct{}{}
-		}
-	}
-	return results
-}

v2/pkg/generators/attack.go

@@ -1,20 +0,0 @@
-package generators
-
-// Type is type of attack
-type Type int
-
-const (
-	// Sniper attack - each variable replaced with values at a time
-	Sniper Type = iota + 1
-	// PitchFork attack - Each variable replaced with positional value in multiple wordlists
-	PitchFork
-	// ClusterBomb attack - Generate all possible combinations of values
-	ClusterBomb
-)
-
-// AttackTypes is an table for conversion of attack type from string.
-var AttackTypes = map[string]Type{
-	"sniper":      Sniper,
-	"pitchfork":   PitchFork,
-	"clusterbomb": ClusterBomb,
-}

v2/pkg/generators/clusterbomb.go

@@ -1,50 +0,0 @@
-package generators
-
-// ClusterbombGenerator Attack - Generate all possible combinations from an input map with all values listed
-// as slices of the same size
-func ClusterbombGenerator(payloads map[string][]string) (out chan map[string]interface{}) {
-	out = make(chan map[string]interface{})
-
-	// generator
-	go func() {
-		defer close(out)
-		var order []string
-		var parts [][]string
-		for name, wordlist := range payloads {
-			order = append(order, name)
-			parts = append(parts, wordlist)
-		}
-
-		var n = 1
-		for _, ar := range parts {
-			n *= len(ar)
-		}
-
-		var at = make([]int, len(parts))
-	loop:
-		for {
-			// increment position counters
-			for i := len(parts) - 1; i >= 0; i-- {
-				if at[i] > 0 && at[i] >= len(parts[i]) {
-					if i == 0 || (i == 1 && at[i-1] == len(parts[0])-1) {
-						break loop
-					}
-					at[i] = 0
-					at[i-1]++
-				}
-			}
-			// construct permutation
-			item := make(map[string]interface{})
-			for i, ar := range parts {
-				var p = at[i]
-				if p >= 0 && p < len(ar) {
-					item[order[i]] = ar[p]
-				}
-			}
-			out <- item
-			at[len(parts)-1]++
-		}
-	}()
-
-	return out
-}

v2/pkg/generators/dsl.go

@@ -1,111 +0,0 @@
-package generators
-
-import (
-	"crypto/md5"
-	"crypto/sha1"
-	"crypto/sha256"
-	"encoding/base64"
-	"encoding/hex"
-	"html"
-	"net/url"
-	"regexp"
-	"strings"
-
-	"github.com/Knetic/govaluate"
-)
-
-// HelperFunctions contains the dsl functions
-func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
-	functions = make(map[string]govaluate.ExpressionFunction)
-	// strings
-	functions["len"] = func(args ...interface{}) (interface{}, error) {
-		length := len(args[0].(string))
-		return (float64)(length), nil
-	}
-	functions["toupper"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ToUpper(args[0].(string)), nil
-	}
-	functions["tolower"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ToLower(args[0].(string)), nil
-	}
-	functions["replace"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Replace(args[0].(string), args[1].(string), args[2].(string), -1), nil
-	}
-	functions["trim"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Trim(args[0].(string), args[2].(string)), nil
-	}
-	functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimLeft(args[0].(string), args[1].(string)), nil
-	}
-	functions["trimright"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimRight(args[0].(string), args[1].(string)), nil
-	}
-	functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimSpace(args[0].(string)), nil
-	}
-	functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimPrefix(args[0].(string), args[1].(string)), nil
-	}
-	functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimSuffix(args[0].(string), args[1].(string)), nil
-	}
-	functions["reverse"] = func(args ...interface{}) (interface{}, error) {
-		return reverseString(args[0].(string)), nil
-	}
-	// encoding
-	functions["base64"] = func(args ...interface{}) (interface{}, error) {
-		sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
-		return sEnc, nil
-	}
-	functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
-		sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
-		return sEnc, nil
-	}
-	functions["url_encode"] = func(args ...interface{}) (interface{}, error) {
-		return url.PathEscape(args[0].(string)), nil
-	}
-	functions["url_decode"] = func(args ...interface{}) (interface{}, error) {
-		return url.PathUnescape(args[0].(string))
-	}
-	functions["hex_encode"] = func(args ...interface{}) (interface{}, error) {
-		return hex.EncodeToString([]byte(args[0].(string))), nil
-	}
-	functions["hex_decode"] = func(args ...interface{}) (interface{}, error) {
-		hx, _ := hex.DecodeString(args[0].(string))
-		return string(hx), nil
-	}
-	functions["html_escape"] = func(args ...interface{}) (interface{}, error) {
-		return html.EscapeString(args[0].(string)), nil
-	}
-	functions["html_unescape"] = func(args ...interface{}) (interface{}, error) {
-		return html.UnescapeString(args[0].(string)), nil
-	}
-	// hashing
-	functions["md5"] = func(args ...interface{}) (interface{}, error) {
-		hash := md5.Sum([]byte(args[0].(string)))
-		return hex.EncodeToString(hash[:]), nil
-	}
-	functions["sha256"] = func(args ...interface{}) (interface{}, error) {
-		h := sha256.New()
-		h.Write([]byte(args[0].(string)))
-		return hex.EncodeToString(h.Sum(nil)), nil
-	}
-	functions["sha1"] = func(args ...interface{}) (interface{}, error) {
-		h := sha1.New()
-		h.Write([]byte(args[0].(string)))
-		return hex.EncodeToString(h.Sum(nil)), nil
-	}
-	// search
-	functions["contains"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Contains(args[0].(string), args[1].(string)), nil
-	}
-	functions["regex"] = func(args ...interface{}) (interface{}, error) {
-		compiled, err := regexp.Compile(args[0].(string))
-		if err != nil {
-			return nil, err
-		}
-		return compiled.MatchString(args[1].(string)), nil
-	}
-
-	return
-}

v2/pkg/generators/pitchfork.go

@@ -1,38 +0,0 @@
-package generators
-
-// PitchforkGenerator Attack - Generate positional combinations from an input map with all values listed
-// as slices of the same size
-func PitchforkGenerator(payloads map[string][]string) (out chan map[string]interface{}) {
-	out = make(chan map[string]interface{})
-
-	size := 0
-
-	// check if all wordlists have the same size
-	for _, wordlist := range payloads {
-		if size == 0 {
-			size = len(wordlist)
-		}
-
-		if len(wordlist) != size {
-			//set size = 0 and exit the cycle
-			size = 0
-			break
-		}
-	}
-
-	// generator
-	go func() {
-		defer close(out)
-
-		for i := 0; i < size; i++ {
-			element := make(map[string]interface{})
-			for name, wordlist := range payloads {
-				element[name] = wordlist[i]
-			}
-
-			out <- element
-		}
-	}()
-
-	return out
-}

v2/pkg/generators/sniper.go

@@ -1,21 +0,0 @@
-package generators
-
-// SniperGenerator Attack - Generate sequential combinations
-func SniperGenerator(payloads map[string][]string) (out chan map[string]interface{}) {
-	out = make(chan map[string]interface{})
-
-	// generator
-	go func() {
-		defer close(out)
-
-		for name, wordlist := range payloads {
-			for _, value := range wordlist {
-				element := CopyMapWithDefaultValue(payloads, "")
-				element[name] = value
-				out <- element
-			}
-		}
-	}()
-
-	return out
-}

v2/pkg/generators/util.go

@@ -1,117 +0,0 @@
-package generators
-
-import (
-	"bufio"
-	"os"
-	"strings"
-)
-
-// LoadWordlists creating proper data structure
-func LoadWordlists(payloads map[string]string) map[string][]string {
-	wordlists := make(map[string][]string)
-	// load all wordlists
-	for name, filepath := range payloads {
-		wordlists[name] = LoadFile(filepath)
-	}
-
-	return wordlists
-}
-
-// LoadFile into slice of strings
-func LoadFile(filepath string) (lines []string) {
-	for line := range StreamFile(filepath) {
-		lines = append(lines, line)
-	}
-
-	return
-}
-
-// StreamFile content to a chan
-func StreamFile(filepath string) (content chan string) {
-	content = make(chan string)
-
-	go func() {
-		defer close(content)
-		file, err := os.Open(filepath)
-		if err != nil {
-			return
-		}
-		defer file.Close()
-
-		// yql filter applied
-		scanner := bufio.NewScanner(file)
-		for scanner.Scan() {
-			content <- scanner.Text()
-		}
-
-		if err := scanner.Err(); err != nil {
-			return
-		}
-	}()
-
-	return
-}
-
-// MergeMaps into a new one
-func MergeMaps(m1, m2 map[string]interface{}) (m map[string]interface{}) {
-	m = make(map[string]interface{})
-	for k, v := range m1 {
-		m[k] = v
-	}
-	for k, v := range m2 {
-		m[k] = v
-	}
-
-	return m
-}
-
-func reverseString(s string) string {
-	runes := []rune(s)
-	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
-		runes[i], runes[j] = runes[j], runes[i]
-	}
-	return string(runes)
-}
-
-// CopyMap creates a new copy of an existing map
-func CopyMap(originalMap map[string]interface{}) map[string]interface{} {
-	newMap := make(map[string]interface{})
-	for key, value := range originalMap {
-		newMap[key] = value
-	}
-	return newMap
-}
-
-// CopyMapWithDefaultValue creates a new copy of an existing map and set a default value
-func CopyMapWithDefaultValue(originalMap map[string][]string, defaultValue interface{}) map[string]interface{} {
-	newMap := make(map[string]interface{})
-	for key := range originalMap {
-		newMap[key] = defaultValue
-	}
-	return newMap
-}
-
-// StringContainsAnyMapItem verifies is a string contains any value of a map
-func StringContainsAnyMapItem(m map[string]interface{}, s string) bool {
-	for key := range m {
-		if strings.Contains(s, key) {
-			return true
-		}
-	}
-
-	return false
-}
-
-// TrimDelimiters removes trailing brackets
-func TrimDelimiters(s string) string {
-	return strings.TrimSuffix(strings.TrimPrefix(s, "{{"), "}}")
-}
-
-// FileExists checks if a file exists and is not a directory
-func FileExists(filename string) bool {
-	info, err := os.Stat(filename)
-	if os.IsNotExist(err) {
-		return false
-	}
-	return !info.IsDir()
-}

v2/pkg/matchers/util.go

@@ -1,62 +0,0 @@
-package matchers
-
-import (
-	"fmt"
-	"net/http"
-	"net/http/httputil"
-	"strings"
-
-	"github.com/miekg/dns"
-)
-
-func httpToMap(resp *http.Response, body, headers string) (m map[string]interface{}) {
-	m = make(map[string]interface{})
-
-	m["content_length"] = resp.ContentLength
-	m["status_code"] = resp.StatusCode
-	for k, v := range resp.Header {
-		k = strings.ToLower(strings.TrimSpace(strings.Replace(k, "-", "_", -1)))
-		m[k] = strings.Join(v, " ")
-	}
-	m["all_headers"] = headers
-
-	m["body"] = body
-	if r, err := httputil.DumpResponse(resp, true); err == nil {
-		m["raw"] = string(r)
-	}
-
-	return m
-}
-
-func dnsToMap(msg *dns.Msg) (m map[string]interface{}) {
-	m = make(map[string]interface{})
-
-	m["rcode"] = msg.Rcode
-	var qs string
-	for _, question := range msg.Question {
-		qs += fmt.Sprintln(question.String())
-	}
-	m["question"] = qs
-
-	var exs string
-	for _, extra := range msg.Extra {
-		exs += fmt.Sprintln(extra.String())
-	}
-	m["extra"] = exs
-
-	var ans string
-	for _, answer := range msg.Answer {
-		ans += fmt.Sprintln(answer.String())
-	}
-	m["answer"] = ans
-
-	var nss string
-	for _, ns := range msg.Ns {
-		nss += fmt.Sprintln(ns.String())
-	}
-	m["ns"] = nss
-
-	m["raw"] = msg.String()
-
-	return m
-}

v2/pkg/operators/common/dsl/dsl.go

@@ -0,0 +1,316 @@
+package dsl
+
+import (
+	"bytes"
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"html"
+	"math"
+	"math/rand"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/Knetic/govaluate"
+	"github.com/projectdiscovery/nuclei/v2/internal/collaborator"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"github.com/spaolacci/murmur3"
+)
+
+const (
+	numbers              = "1234567890"
+	letters              = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+	withCutSetArgsSize   = 2
+	withBaseRandArgsSize = 3
+	withMaxRandArgsSize  = withCutSetArgsSize
+)
+
+// HelperFunctions contains the dsl helper functions
+func HelperFunctions() map[string]govaluate.ExpressionFunction {
+	functions := make(map[string]govaluate.ExpressionFunction)
+
+	functions["len"] = func(args ...interface{}) (interface{}, error) {
+		length := len(types.ToString(args[0]))
+		return float64(length), nil
+	}
+
+	functions["toupper"] = func(args ...interface{}) (interface{}, error) {
+		return strings.ToUpper(types.ToString(args[0])), nil
+	}
+
+	functions["tolower"] = func(args ...interface{}) (interface{}, error) {
+		return strings.ToLower(types.ToString(args[0])), nil
+	}
+
+	functions["replace"] = func(args ...interface{}) (interface{}, error) {
+		return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil
+	}
+
+	functions["replace_regex"] = func(args ...interface{}) (interface{}, error) {
+		compiled, err := regexp.Compile(types.ToString(args[1]))
+		if err != nil {
+			return nil, err
+		}
+		return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil
+	}
+
+	functions["trim"] = func(args ...interface{}) (interface{}, error) {
+		return strings.Trim(types.ToString(args[0]), types.ToString(args[2])), nil
+	}
+
+	functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
+		return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil
+	}
+
+	functions["trimright"] = func(args ...interface{}) (interface{}, error) {
+		return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil
+	}
+
+	functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
+		return strings.TrimSpace(types.ToString(args[0])), nil
+	}
+
+	functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
+		return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil
+	}
+
+	functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
+		return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil
+	}
+
+	functions["reverse"] = func(args ...interface{}) (interface{}, error) {
+		return reverseString(types.ToString(args[0])), nil
+	}
+
+	// encoding
+	functions["base64"] = func(args ...interface{}) (interface{}, error) {
+		sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
+
+		return sEnc, nil
+	}
+
+	// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
+	functions["base64_py"] = func(args ...interface{}) (interface{}, error) {
+		sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
+		return insertInto(sEnc, 76, '\n'), nil
+	}
+
+	functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
+		return base64.StdEncoding.DecodeString(types.ToString(args[0]))
+	}
+
+	functions["url_encode"] = func(args ...interface{}) (interface{}, error) {
+		return url.PathEscape(types.ToString(args[0])), nil
+	}
+
+	functions["url_decode"] = func(args ...interface{}) (interface{}, error) {
+		return url.PathUnescape(types.ToString(args[0]))
+	}
+
+	functions["hex_encode"] = func(args ...interface{}) (interface{}, error) {
+		return hex.EncodeToString([]byte(types.ToString(args[0]))), nil
+	}
+
+	functions["hex_decode"] = func(args ...interface{}) (interface{}, error) {
+		hx, _ := hex.DecodeString(types.ToString(args[0]))
+		return string(hx), nil
+	}
+
+	functions["html_escape"] = func(args ...interface{}) (interface{}, error) {
+		return html.EscapeString(types.ToString(args[0])), nil
+	}
+
+	functions["html_unescape"] = func(args ...interface{}) (interface{}, error) {
+		return html.UnescapeString(types.ToString(args[0])), nil
+	}
+
+	// hashing
+	functions["md5"] = func(args ...interface{}) (interface{}, error) {
+		hash := md5.Sum([]byte(types.ToString(args[0])))
+
+		return hex.EncodeToString(hash[:]), nil
+	}
+
+	functions["sha256"] = func(args ...interface{}) (interface{}, error) {
+		h := sha256.New()
+		_, err := h.Write([]byte(types.ToString(args[0])))
+
+		if err != nil {
+			return nil, err
+		}
+		return hex.EncodeToString(h.Sum(nil)), nil
+	}
+
+	functions["sha1"] = func(args ...interface{}) (interface{}, error) {
+		h := sha1.New()
+		_, err := h.Write([]byte(types.ToString(args[0])))
+
+		if err != nil {
+			return nil, err
+		}
+		return hex.EncodeToString(h.Sum(nil)), nil
+	}
+
+	functions["mmh3"] = func(args ...interface{}) (interface{}, error) {
+		return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil
+	}
+
+	// search
+	functions["contains"] = func(args ...interface{}) (interface{}, error) {
+		return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil
+	}
+
+	functions["regex"] = func(args ...interface{}) (interface{}, error) {
+		compiled, err := regexp.Compile(types.ToString(args[0]))
+		if err != nil {
+			return nil, err
+		}
+		return compiled.MatchString(types.ToString(args[1])), nil
+	}
+
+	// random generators
+	functions["rand_char"] = func(args ...interface{}) (interface{}, error) {
+		chars := letters + numbers
+		bad := ""
+		if len(args) >= 1 {
+			chars = types.ToString(args[0])
+		}
+		if len(args) >= withCutSetArgsSize {
+			bad = types.ToString(args[1])
+		}
+		chars = trimAll(chars, bad)
+		return chars[rand.Intn(len(chars))], nil
+	}
+
+	functions["rand_base"] = func(args ...interface{}) (interface{}, error) {
+		l := 0
+		bad := ""
+		base := letters + numbers
+
+		if len(args) >= 1 {
+			l = args[0].(int)
+		}
+		if len(args) >= withCutSetArgsSize {
+			bad = types.ToString(args[1])
+		}
+		if len(args) >= withBaseRandArgsSize {
+			base = types.ToString(args[2])
+		}
+		base = trimAll(base, bad)
+		return randSeq(base, l), nil
+	}
+
+	functions["rand_text_alphanumeric"] = func(args ...interface{}) (interface{}, error) {
+		l := 0
+		bad := ""
+		chars := letters + numbers
+
+		if len(args) >= 1 {
+			l = args[0].(int)
+		}
+		if len(args) >= withCutSetArgsSize {
+			bad = types.ToString(args[1])
+		}
+		chars = trimAll(chars, bad)
+		return randSeq(chars, l), nil
+	}
+
+	functions["rand_text_alpha"] = func(args ...interface{}) (interface{}, error) {
+		l := 0
+		bad := ""
+		chars := letters
+
+		if len(args) >= 1 {
+			l = args[0].(int)
+		}
+		if len(args) >= withCutSetArgsSize {
+			bad = types.ToString(args[1])
+		}
+		chars = trimAll(chars, bad)
+		return randSeq(chars, l), nil
+	}
+
+	functions["rand_text_numeric"] = func(args ...interface{}) (interface{}, error) {
+		l := 0
+		bad := ""
+		chars := numbers
+
+		if len(args) >= 1 {
+			l = args[0].(int)
+		}
+		if len(args) >= withCutSetArgsSize {
+			bad = types.ToString(args[1])
+		}
+		chars = trimAll(chars, bad)
+		return randSeq(chars, l), nil
+	}
+
+	functions["rand_int"] = func(args ...interface{}) (interface{}, error) {
+		min := 0
+		max := math.MaxInt32
+
+		if len(args) >= 1 {
+			min = args[0].(int)
+		}
+		if len(args) >= withMaxRandArgsSize {
+			max = args[1].(int)
+		}
+		return rand.Intn(max-min) + min, nil
+	}
+
+	// Time Functions
+	functions["waitfor"] = func(args ...interface{}) (interface{}, error) {
+		seconds := args[0].(float64)
+		time.Sleep(time.Duration(seconds) * time.Second)
+		return true, nil
+	}
+
+	// Collaborator
+	functions["collab"] = func(args ...interface{}) (interface{}, error) {
+		// check if collaborator contains a specific pattern
+		return collaborator.DefaultCollaborator.Has(types.ToString(args[0])), nil
+	}
+	return functions
+}
+
+func reverseString(s string) string {
+	runes := []rune(s)
+	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
+		runes[i], runes[j] = runes[j], runes[i]
+	}
+	return string(runes)
+}
+
+func trimAll(s, cutset string) string {
+	for _, c := range cutset {
+		s = strings.ReplaceAll(s, string(c), "")
+	}
+	return s
+}
+
+func randSeq(base string, n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = rune(base[rand.Intn(len(base))])
+	}
+	return string(b)
+}
+
+func insertInto(s string, interval int, sep rune) string {
+	var buffer bytes.Buffer
+	before := interval - 1
+	last := len(s) - 1
+	for i, char := range s {
+		buffer.WriteRune(char)
+		if i%interval == before && i != last {
+			buffer.WriteRune(sep)
+		}
+	}
+	buffer.WriteRune(sep)
+	return buffer.String()
+}

v2/pkg/operators/extractors/compile.go

@@ -3,12 +3,14 @@ package extractors
 import (
 	"fmt"
 	"regexp"
+	"strings"
 )
 
 // CompileExtractors performs the initial setup operation on a extractor
 func (e *Extractor) CompileExtractors() error {
-	// Setup the matcher type
-	_, ok := ExtractorTypes[e.Type]
+	var ok bool
+	// Setup the extractor type
+	e.extractorType, ok = ExtractorTypes[e.Type]
 	if !ok {
 		return fmt.Errorf("unknown extractor type specified: %s", e.Type)
 	}
@@ -22,14 +24,13 @@ func (e *Extractor) CompileExtractors() error {
 		e.regexCompiled = append(e.regexCompiled, compiled)
 	}
 
+	for i, kval := range e.KVal {
+		e.KVal[i] = strings.ToLower(kval)
+	}
+
 	// Setup the part of the request to match, if any.
-	if e.Part != "" {
-		e.part, ok = PartTypes[e.Part]
-		if !ok {
-			return fmt.Errorf("unknown matcher part specified: %s", e.Part)
-		}
-	} else {
-		e.part = BodyPart
+	if e.Part == "" {
+		e.Part = "body"
 	}
 	return nil
 }

v2/pkg/operators/extractors/extract.go

@@ -0,0 +1,44 @@
+package extractors
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+// ExtractRegex extracts text from a corpus and returns it
+func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} {
+	results := make(map[string]struct{})
+
+	groupPlusOne := e.RegexGroup + 1
+	for _, regex := range e.regexCompiled {
+		matches := regex.FindAllStringSubmatch(corpus, -1)
+
+		for _, match := range matches {
+			if len(match) < groupPlusOne {
+				continue
+			}
+			matchString := match[e.RegexGroup]
+
+			if _, ok := results[matchString]; !ok {
+				results[matchString] = struct{}{}
+			}
+		}
+	}
+	return results
+}
+
+// ExtractKval extracts key value pairs from a data map
+func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{} {
+	results := make(map[string]struct{})
+
+	for _, k := range e.KVal {
+		item, ok := data[k]
+		if !ok {
+			continue
+		}
+		itemString := types.ToString(item)
+		if _, ok := results[itemString]; !ok {
+			results[itemString] = struct{}{}
+		}
+	}
+	return results
+}

v2/pkg/operators/extractors/extractors.go

@@ -4,20 +4,29 @@ import "regexp"
 
 // Extractor is used to extract part of response using a regex.
 type Extractor struct {
-	// Type is the type of the matcher
+	// Name is the extractor's name
+	Name string `yaml:"name,omitempty"`
+	// Type is the type of the extractor
 	Type string `yaml:"type"`
+	// extractorType is the internal type of the extractor
+	extractorType ExtractorType
 
 	// Regex are the regex pattern required to be present in the response
 	Regex []string `yaml:"regex"`
+	// RegexGroup specifies a group to extract from the regex
+	RegexGroup int `yaml:"group"`
 	// regexCompiled is the compiled variant
 	regexCompiled []*regexp.Regexp
 
+	// KVal are the kval to be present in the response headers/cookies
+	KVal []string `yaml:"kval,omitempty"`
+
 	// Part is the part of the request to match
 	//
 	// By default, matching is performed in request body.
 	Part string `yaml:"part,omitempty"`
-	// part is the part of the request to match
-	part Part
+	// Internal defines if this is used internally
+	Internal bool `yaml:"internal,omitempty"`
 }
 
 // ExtractorType is the type of the extractor specified
@@ -26,33 +35,17 @@ type ExtractorType = int
 const (
 	// RegexExtractor extracts responses with regexes
 	RegexExtractor ExtractorType = iota + 1
+	// KValExtractor extracts responses with key:value
+	KValExtractor
 )
 
 // ExtractorTypes is an table for conversion of extractor type from string.
 var ExtractorTypes = map[string]ExtractorType{
 	"regex": RegexExtractor,
+	"kval":  KValExtractor,
 }
 
-// Part is the part of the request to match
-type Part int
-
-const (
-	// BodyPart matches body of the response.
-	BodyPart Part = iota + 1
-	// HeaderPart matches headers of the response.
-	HeaderPart
-	// AllPart matches both response body and headers of the response.
-	AllPart
-)
-
-// PartTypes is an table for conversion of part type from string.
-var PartTypes = map[string]Part{
-	"body":   BodyPart,
-	"header": HeaderPart,
-	"all":    AllPart,
-}
-
-// GetPart returns the part of the matcher
-func (e *Extractor) GetPart() Part {
-	return e.part
+// GetType returns the type of the matcher
+func (e *Extractor) GetType() ExtractorType {
+	return e.extractorType
 }

v2/pkg/operators/matchers/compile.go

@@ -1,22 +1,36 @@
 package matchers
 
 import (
+	"encoding/hex"
 	"fmt"
 	"regexp"
 
 	"github.com/Knetic/govaluate"
-	"github.com/projectdiscovery/nuclei/v2/pkg/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
 )
 
 // CompileMatchers performs the initial setup operation on a matcher
 func (m *Matcher) CompileMatchers() error {
 	var ok bool
 
+	// Support hexadecimal encoding for matchers too.
+	if m.Encoding == "hex" {
+		for i, word := range m.Words {
+			if decoded, err := hex.DecodeString(word); err == nil && len(decoded) > 0 {
+				m.Words[i] = string(decoded)
+			}
+		}
+	}
+
 	// Setup the matcher type
 	m.matcherType, ok = MatcherTypes[m.Type]
 	if !ok {
 		return fmt.Errorf("unknown matcher type specified: %s", m.Type)
 	}
+	// By default, match on body if user hasn't provided any specific items
+	if m.Part == "" {
+		m.Part = "body"
+	}
 
 	// Compile the regexes
 	for _, regex := range m.Regex {
@@ -24,17 +38,15 @@ func (m *Matcher) CompileMatchers() error {
 		if err != nil {
 			return fmt.Errorf("could not compile regex: %s", regex)
 		}
-
 		m.regexCompiled = append(m.regexCompiled, compiled)
 	}
 
 	// Compile the dsl expressions
-	for _, dsl := range m.DSL {
-		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(dsl, generators.HelperFunctions())
+	for _, expr := range m.DSL {
+		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
 		if err != nil {
-			return fmt.Errorf("could not compile dsl: %s", dsl)
+			return fmt.Errorf("could not compile dsl: %s", expr)
 		}
-
 		m.dslCompiled = append(m.dslCompiled, compiled)
 	}
 
@@ -47,15 +59,5 @@ func (m *Matcher) CompileMatchers() error {
 	} else {
 		m.condition = ORCondition
 	}
-
-	// Setup the part of the request to match, if any.
-	if m.Part != "" {
-		m.part, ok = PartTypes[m.Part]
-		if !ok {
-			return fmt.Errorf("unknown matcher part specified: %s", m.Part)
-		}
-	} else {
-		m.part = BodyPart
-	}
 	return nil
 }