Merge pull request #886 from projectdiscovery/dev

2.4.2 Bugfix Release

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,14 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: "[feature]"
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.

.github/ISSUE_TEMPLATE/issue-report.md

@@ -0,0 +1,18 @@
+---
+name: Issue report
+about: Create a report to help us improve
+title: "[issue]"
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Nuclei version**
+Please share the version of the nuclei you are running with `nuclei -version` 
+
+
+**Screenshot of the error or bug**
+please add the screenshot showing bug or issue you are facing.

.github/dependabot.yml

@@ -0,0 +1,37 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    target-branch: "dev"
+    commit-message:
+      prefix: "chore"
+      include: "scope"
+
+  # Maintain dependencies for go modules
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    target-branch: "dev"
+    commit-message:
+      prefix: "chore"
+      include: "scope"
+
+  # Maintain dependencies for docker
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    target-branch: "dev"
+    commit-message:
+      prefix: "chore"
+      include: "scope"

.github/workflows/build-test.yml

@@ -0,0 +1,31 @@
+name: 🔨 Build Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+
+jobs:  
+  build:
+    name: Test Builds
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.15
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Test
+        run: go test ./...
+        working-directory: v2/
+
+      - name: Integration Tests
+        run: bash run.sh
+        working-directory: integration_tests/
+
+      - name: Build
+        run: go build .
+        working-directory: v2/cmd/nuclei/

.github/workflows/build.yaml

@@ -1,27 +0,0 @@
-name: Build
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-  
-jobs:          
-  build:
-    name: Build
-    runs-on: ubuntu-latest 
-    steps:
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.14
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Test
-        run: go test .
-        working-directory: v2/cmd/nuclei/
-
-      - name: Build
-        run: go build .
-        working-directory: v2/cmd/nuclei/

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,38 @@
+name: 🚨 CodeQL Analysis
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - dev
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/dockerhub-push.yml

@@ -1,17 +1,34 @@
-# dockerhub-push pushes docker build to dockerhub automatically
-# on the creation of a new release
-name: Publish to Dockerhub on creation of a new release
-on: 
+name: 🌥 Docker Push
+
+on:
   release:
     types: [published]
+  workflow_dispatch:
+
 jobs:
-  build:
+  docker:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@master
-    - name: Publish to Dockerhub Registry
-      uses: elgohr/Publish-Docker-Github-Action@master
-      with:
-        name: projectdiscovery/nuclei
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64,linux/arm
+          push: true
+          tags: projectdiscovery/nuclei:latest

.github/workflows/lint-test.yml

@@ -0,0 +1,19 @@
+name: 🙏🏻 Lint Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  lint:
+    name: Lint Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          args: --timeout 5m
+          working-directory: v2/

.github/workflows/release-binary.yml

@@ -0,0 +1,30 @@
+name: 🎉 Release Binary
+on:
+  create:
+    tags:
+      - v*
+  workflow_dispatch:
+
+jobs: 
+  release: 
+    runs-on: ubuntu-latest
+    steps: 
+      - 
+        name: "Check out code"
+        uses: actions/checkout@v2
+        with: 
+          fetch-depth: 0
+      - 
+        name: "Set up Go"
+        uses: actions/setup-go@v2
+        with: 
+          go-version: 1.16
+      - 
+        env: 
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        name: "Create release on GitHub"
+        uses: goreleaser/goreleaser-action@v2
+        with: 
+          args: "release --rm-dist"
+          version: latest
+          workdir: v2/

.github/workflows/release.yml

@@ -1,25 +0,0 @@
-name: Release
-on:
-  create:
-    tags:
-      - v*
-
-jobs:
-  release:
-    name: Release on GitHub
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v1
-
-      - name: Validates GO releaser config
-        uses: docker://goreleaser/goreleaser:latest
-        with:
-          args: check
-
-      - name: Create release on GitHub
-        uses: docker://goreleaser/goreleaser:latest
-        with:
-          args: release
-        env:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.gitignore

@@ -1,2 +1,10 @@
 cmd/nuclei/nuclei*
 v2/cmd/nuclei/nuclei
+.idea
+integration_tests/integration-test
+integration_tests/nuclei
+v2/cmd/integration-test/integration-test
+bin
+v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class
+v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class
+v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser

.golangci.yml

@@ -0,0 +1,108 @@
+linters-settings:
+  dupl:
+    threshold: 100
+  exhaustive:
+    default-signifies-exhaustive: false
+  # funlen:
+  #   lines: 100
+  #   statements: 50
+  #goconst:
+  #  min-len: 2
+  #  min-occurrences: 2
+  gocritic:
+    enabled-tags:
+      - diagnostic
+      - experimental
+      - opinionated
+      - performance
+      - style
+    disabled-checks:
+      - dupImport # https://github.com/go-critic/go-critic/issues/845
+      - ifElseChain
+  # gocyclo:
+  #   min-complexity: 15
+  goimports:
+    local-prefixes: github.com/golangci/golangci-lint
+  golint:
+    min-confidence: 0
+  gomnd:
+    settings:
+      mnd:
+        # don't include the "operation" and "assign"
+        checks: argument,case,condition,return
+  govet:
+    check-shadowing: true
+    settings:
+      printf:
+        funcs:
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
+  # lll:
+  #   line-length: 140
+  maligned:
+    suggest-new: true
+  misspell:
+    locale: US
+  nolintlint:
+    allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
+    allow-unused: false # report any unused nolint directives
+    require-explanation: false # don't require an explanation for nolint directives
+    require-specific: false # don't require nolint directives to be specific about which linter is being skipped
+
+linters:
+  # please, do not use `enable-all`: it's deprecated and will be removed soon.
+  # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
+  disable-all: true
+  enable:
+    #- bodyclose
+    - deadcode
+    - dogsled
+    - errcheck
+    - exhaustive
+    - gochecknoinits
+    #- goconst
+    - gocritic
+    - gofmt
+    - goimports
+    #- gomnd
+    - goprintffuncname
+    - gosimple
+    - govet
+    - ineffassign
+    - interfacer
+    - maligned
+    - misspell
+    - nakedret
+    - noctx
+    - nolintlint
+    - rowserrcheck
+    - scopelint
+    - staticcheck
+    - structcheck
+    - stylecheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+    - varcheck
+    - whitespace
+    - revive
+
+  # don't enable:
+  # - depguard
+  # - asciicheck
+  # - funlen
+  # - gochecknoglobals
+  # - gocognit
+  # - gocyclo
+  # - godot
+  # - godox
+  # - goerr113
+  # - gosec
+  # - lll
+  # - nestif
+  # - prealloc
+  # - testpackage
+  # - wsl

.goreleaser.yml

@@ -1,21 +0,0 @@
-builds:
-    - binary: nuclei
-      main: v2/cmd/nuclei/main.go
-      goos:
-        - linux
-        - windows
-        - darwin
-      goarch:
-        - amd64
-        - 386
-        - arm
-        - arm64
-  
-archives:
-    - id: tgz
-      format: tar.gz
-      replacements:
-          darwin: macOS
-      format_overrides:
-          - goos: windows
-            format: zip

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at contact@projectdiscovery.io. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

Dockerfile

@@ -1,10 +1,7 @@
-FROM golang:1.13.11-alpine3.10 AS build-env
+FROM golang:1.16.6-alpine as build-env
+RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
 
-RUN apk add --no-cache --upgrade git openssh-client ca-certificates
-RUN go get -u github.com/golang/dep/cmd/dep
-WORKDIR /go/src/app
-
-# Install
-RUN GO111MODULE=on go get -u github.com/projectdiscovery/nuclei/v2/cmd/nuclei
-
-ENTRYPOINT ["nuclei"]
+FROM alpine:latest
+RUN apk add --no-cache bind-tools ca-certificates
+COPY --from=build-env /go/bin/nuclei /usr/local/bin/nuclei
+ENTRYPOINT ["nuclei"]

LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Exposed Atoms
+Copyright (c) 2021 ProjectDiscovery, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,177 +1,286 @@
-<h1 align="left">
-  <img src="static/nuclei-logo.png" alt="nuclei" width="200px"></a>
+<h1 align="center">
   <br>
+  <a href="https://nuclei.projectdiscovery.io"><img src="static/nuclei-logo.png" width="200px" alt="Nuclei"></a>
 </h1>
 
-[![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT)
-[![Go Report Card](https://goreportcard.com/badge/github.com/projectdiscovery/nuclei)](https://goreportcard.com/report/github.com/projectdiscovery/nuclei)
-[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectdiscovery/nuclei/issues)
+<h4 align="center">Fast and customisable vulnerability scanner based on simple YAML based DSL.</h4>
 
-Nuclei is a fast tool for configurable targeted scanning based on templates offering massive extensibility and ease of use. 
 
-Nuclei is used to send requests across targets based on a template leading to zero false positives and providing effective scanning for known paths. Main use cases for nuclei are during initial reconnaissance phase to quickly check for low hanging fruits or CVEs across targets that are known and easily detectable. It uses [retryablehttp-go library](https://github.com/projectdiscovery/retryablehttp-go) designed to handle various errors and retries in case of blocking by WAFs, this is also one of our core modules from custom-queries.
+<p align="center">
+<a href="https://goreportcard.com/report/github.com/projectdiscovery/nuclei"><img src="https://goreportcard.com/badge/github.com/projectdiscovery/nuclei"></a>
+<a href="https://github.com/projectdiscovery/nuclei/issues"><img src="https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat"></a>
+<a href="https://github.com/projectdiscovery/nuclei/releases"><img src="https://img.shields.io/github/release/projectdiscovery/nuclei"></a>
+<a href="https://twitter.com/pdnuclei"><img src="https://img.shields.io/twitter/follow/pdnuclei.svg?logo=twitter"></a>
+<a href="https://discord.gg/projectdiscovery"><img src="https://img.shields.io/discord/695645237418131507.svg?logo=discord"></a>
+<a href="https://github.com/projectdiscovery/nuclei/actions/workflows/build-test.yml"><img src="https://github.com/projectdiscovery/nuclei/actions/workflows/build-test.yml/badge.svg?branch=master"></a>
+</p>
+      
+<p align="center">
+  <a href="#how-it-works">How</a> •
+  <a href="#install-nuclei">Install</a> •
+  <a href="#for-security-engineers">For Security Engineers</a> •
+  <a href="#for-developers-and-organisations">For Developers</a> •
+  <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">Documentation</a> •
+  <a href="#credits">Credits</a> •
+  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">FAQs</a> •
+  <a href="https://discord.gg/projectdiscovery">Join Discord</a>
+</p>
 
-We have also [open-sourced a dedicated repository](https://github.com/projectdiscovery/nuclei-templates) to maintain various type of templates, we hope that you will contribute there too. Templates are provided in hopes that these will be useful and will allow everyone to build their own templates for the scanner. Checkout the guide at [**GUIDE.md**](https://github.com/projectdiscovery/nuclei-templates/blob/master/GUIDE.md) for a primer on nuclei templates.
+---
 
-# Resources
-- [Resources](#resources)
-- [Features](#features)
-- [Usage](#usage)
-- [Installation Instructions](#installation-instructions)
-    - [From Binary](#from-binary)
-    - [From Source](#from-source)
-    - [Running in a Docker Container](#running-in-a-docker-container)
-- [Nuclei templates](#nuclei-templates)
-- [Running nuclei](#running-nuclei)
-    - [1. Running nuclei with a single template.](#1-running-nuclei-with-a-single-template)
-    - [2. Running nuclei with multiple templates.](#2-running-nuclei-with-multiple-templates)
-    - [3. Automating nuclei with subfinder and any other similar tool.](#3-automating-nuclei-with-subfinder-and-any-other-similar-tool)
-- [Thanks](#thanks)
+Nuclei is used to send requests across targets based on a template leading to zero false positives and providing fast scanning on large number of hosts. Nuclei offers scanning for a variety of protocols including TCP, DNS, HTTP, File, etc. With powerful and flexible templating, all kinds of security checks can be modelled with Nuclei.
 
- # Features
+We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-templates) that houses various type of vulnerability templates contributed by **more than 100** security researchers and engineers. It is preloaded with ready to use templates using `-update-templates` flag.
 
-<h1 align="left">
-  <img src="static/nuclei-run.png" alt="nuclei" width="700px"></a>
-  <br>
-</h1>
 
- - Simple and modular code base making it easy to contribute.
- - Fast And fully configurable using a template based engine.
- - Handles edge cases doing retries, backoffs etc for handling WAFs.
- - Smart matching functionality for zero false positive scanning.
 
-# Usage
+## How it works
 
-```bash
+
+<h3 align="center">
+  <img src="static/nuclei-flow.jpg" alt="nuclei-flow" width="700px"></a>
+</h3>
+
+
+# Install Nuclei
+
+```sh
+GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+```
+
+**More installation [methods can be found here](https://nuclei.projectdiscovery.io/nuclei/get-started/).**
+
+<table>
+<tr>
+<td>  
+
+### Nuclei Templates
+
+Nuclei has had built-in support for automatic update/download templates since version [v2.4.0](https://github.com/projectdiscovery/nuclei/releases/tag/v2.4.0). [**Nuclei-Templates**](https://github.com/projectdiscovery/nuclei-templates) project provides a community-contributed list of ready-to-use templates that is constantly updated.
+
+You may still use the `update-templates` flag to update the nuclei templates at any time; automatic updates happen every 24 hours. You can write your own checks for your individual workflow and needs following Nuclei's [templating guide](https://nuclei.projectdiscovery.io/templating-guide/).
+
+</td>
+</tr>
+</table>
+
+### Usage
+
+```sh
 nuclei -h
 ```
 
 This will display help for the tool. Here are all the switches it supports.
 
-| Flag              | Description                                           | Example                                            |
-|-------------------|-------------------------------------------------------|----------------------------------------------------|
-| -c                | Number of concurrent requests (default 10)            | nuclei -c 100                                      |
-| -l                | List of urls to run templates                         | nuclei -l urls.txt                                 |
-| -target           | Target to scan using templates                        | nuclei -target hxxps://example.com                 |
-| -t                | Templates input file/files to check across hosts      | nuclei -t git-core.yaml                            |
-| -t                | Templates input file/files to check across hosts      | nuclei -t nuclei-templates/cves/                   |
-| -nC               | Don't Use colors in output                            | nuclei -nC                                         |
-| -json             | Prints and write output in json format                | nuclei -json                                       |
-| -o                | File to save output result (optional)                 | nuclei -o output.txt                               |
-| -silent           | Show only found results in output                     | nuclei -silent                                     |
-| -retries          | Number of times to retry a failed request (default 1) | nuclei -retries 1                                  |
-| -timeout          | Seconds to wait before timeout (default 5)            | nuclei -timeout 5                                  |
-| -debug            | Allow debugging of request/responses.                 | nuclei -debug                                      |
-| -update-templates | Download and updates nuclei templates                 | nuclei -update-templates                           |
-| -update-directory | Directory for storing nuclei-templates(optional)      | nuclei -update-directory templates                 |
-| -v                | Shows verbose output of all sent requests             | nuclei -v                                          |
-| -version          | Show version of nuclei                                | nuclei -version                                    |
-| -proxy-url        | Proxy URL                                             | nuclei -proxy-url hxxp://127.0.0.1:8080            |
-| -proxy-socks-url  | Socks proxy  URL                                      | nuclei -proxy-socks-url socks5://127.0.0.1:8080    |
-| -H                | Custom Header                                         | nuclei -H "x-bug-bounty: hacker"                   |
 
+```yaml
+Nuclei is a fast, template based vulnerability scanner focusing
+on extensive configurability, massive extensibility and ease of use.
 
-# Installation Instructions
+Usage:
+  ./nuclei [flags]
 
+Flags:
+TARGET:
+   -u, -target string  target URL/host to scan
+   -l, -list string    path to file containing a list of target URLs/hosts to scan (one per line)
 
-### From Binary
+TEMPLATES:
+   -tl                      list all available templates
+   -t, -templates string[]  template or template directory paths to include in the scan
+   -w, -workflows string[]  list of workflows to run
+   -nt, -new-templates      run newly added templates only
+   -validate                validate the passed templates to nuclei
 
-The installation is easy. You can download the pre-built binaries for your platform from the [Releases](https://github.com/projectdiscovery/nuclei/releases/) page. Extract them using tar, move it to your `$PATH`and you're ready to go.
+FILTERING:
+   -tags string[]                         execute a subset of templates that contain the provided tags
+   -include-tags string[]                 tags from the default deny list that permit executing more intrusive templates
+   -etags, -exclude-tags string[]         exclude templates with the provided tags
+   -include-templates string[]            templates to be executed even if they are excluded either by default or configuration
+   -exclude-templates, -exclude string[]  template or template directory paths to exclude
+   -severity, -impact string[]            execute templates that match the provided severities only
+   -author string[]                       execute templates that are (co-)created by the specified authors
 
-```bash
-> tar -xzvf nuclei-linux-amd64.tar.gz
-> mv nuclei /usr/bin/
-> nuclei -h
+OUTPUT:
+   -o, -output string            output file to write found issues/vulnerabilities
+   -silent                       display findings only
+   -v, -verbose                  show verbose output
+   -vv                           display extra verbose information
+   -nc, -no-color                disable output content coloring (ANSI escape codes)
+   -json                         write output in JSONL(ines) format
+   -irr, -include-rr             include request/response pairs in the JSONL output (for findings only)
+   -nm, -no-meta                 don't display match metadata
+   -rdb, -report-db string       local nuclei reporting database (always use this to persist report data)
+   -me, -markdown-export string  directory to export results in markdown format
+   -se, -sarif-export string     file to export results in SARIF format
+
+CONFIGURATIONS:
+   -config string              path to the nuclei configuration file
+   -rc, -report-config string  nuclei reporting module configuration file
+   -H, -header string[]        custom headers in header:value format
+   -r, -resolvers string       file containing resolver list for nuclei
+   -system-resolvers           use system DNS resolving as error fallback
+   -passive                    enable passive HTTP response processing mode
+
+INTERACTSH:
+   -no-interactsh                     do not use interactsh server for blind interaction polling
+   -interactsh-url string             self-hosted Interactsh Server URL (default "https://interact.sh")
+   -interactions-cache-size int       number of requests to keep in the interactions cache (default 5000)
+   -interactions-eviction int         number of seconds to wait before evicting requests from cache (default 60)
+   -interactions-poll-duration int    number of seconds to wait before each interaction poll request (default 5)
+   -interactions-cooldown-period int  extra time for interaction polling before exiting (default 5)
+
+RATE-LIMIT:
+   -rl, -rate-limit int  maximum number of requests to send per second (default 150)
+   -bs, -bulk-size int   maximum number of hosts to be analyzed in parallel per template (default 25)
+   -c, -concurrency int  maximum number of templates to be executed in parallel (default 10)
+
+OPTIMIZATIONS:
+   -timeout int               time to wait in seconds before timeout (default 5)
+   -retries int               number of times to retry a failed request (default 1)
+   -project                   use a project folder to avoid sending same request multiple times
+   -project-path string       set a specific project path (default "/var/folders/ml/m31ysb5x73l1s3kjlyn5g4180000gn/T/")
+   -spm, -stop-at-first-path  stop processing HTTP requests after the first match (may break template/workflow logic)
+
+HEADLESS:
+   -headless          enable templates that require headless browser support
+   -page-timeout int  seconds to wait for each page in headless mode (default 20)
+   -show-browser      show the browser on the screen when running templates with headless mode
+
+DEBUG:
+   -debug                     show all requests and responses
+   -debug-req                 show all sent requests
+   -debug-resp                show all received responses
+   -proxy, -proxy-url string  URL of the HTTP proxy server
+   -proxy-socks-url string    URL of the SOCKS proxy server
+   -trace-log string          file to write sent requests trace log
+   -version                   show nuclei version
+   -tv, -templates-version    shows the version of the installed nuclei-templates
+
+UPDATE:
+   -update                        update nuclei to the latest released version
+   -ut, -update-templates         update the community templates to latest released version
+   -ud, -update-directory string  overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates")
+
+STATISTICS:
+   -stats                    display statistics about the running scan
+   -stats-json               write statistics data to an output file in JSONL(ines) format
+   -si, -stats-interval int  number of seconds to wait between showing a statistics update (default 5)
+   -metrics                  expose nuclei metrics on a port
+   -metrics-port int         port to expose nuclei metrics on (default 9092)
 ```
 
-### From Source
+### Running Nuclei
 
-nuclei requires go1.13+ to install successfully. Run the following command to get the repo - 
+Scanning target domain with [community-curated](https://github.com/projectdiscovery/nuclei-templates) nuclei templates.
 
-```bash
-> GO111MODULE=on go get -u -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+```sh
+nuclei -u https://example.com
 ```
 
-In order to update the tool, you can use -u flag with `go get` command.
+Scanning target URLs with [community-curated](https://github.com/projectdiscovery/nuclei-templates) nuclei templates.
 
-### Running in a Docker Container
-
-You can use the [nuclei dockerhub image](https://hub.docker.com/r/projectdiscovery/nuclei). Simply run -
-
-```bash
-> docker pull projectdiscovery/nuclei
+```sh
+nuclei -list urls.txt
 ```
 
-- After downloading or building the container, run the following:
-```bash
-> docker run -it projectdiscovery/nuclei
+Example of `urls.txt`:
+
+```yaml
+http://example.com
+http://app.example.com
+http://test.example.com
+http://uat.example.com
 ```
 
-For example, this will run the tool against all the hosts in `urls.txt` and output the results to your host file system:
-```bash
-> cat urls.txt | docker run -v /path-to-nuclei-templates:/go/src/app/ -i projectdiscovery/nuclei -t ./files/git-config.yaml > results.txt
-```
-Remember to change `/path-to-nuclei-templates` to the real path on your host file system.
+**More detailed examples of running nuclei can be found [here](https://nuclei.projectdiscovery.io/nuclei/get-started/#running-nuclei).**
 
-# Nuclei templates
+# For Security Engineers
 
-You can download or update the nuclei templates using `update-templates` flag. 
+Nuclei offers great number of features that are helpful for security engineers to customise workflow in their organisation. With the varieties of scan capabilities (like DNS, HTTP, TCP), security engineers can easily create their suite of custom checks with Nuclei.
 
-```bash
-> nuclei -update-templates
-```
+- Varieties of protocols supported: TCP, DNS, HTTP, File, etc
+- Achieve complex vulnerability steps with workflows and [dynamic requests.](https://blog.projectdiscovery.io/nuclei-unleashed-quickly-write-complex-exploits/)
+- Easy to integrate into CI/CD, designed to be easily integrated into regression cycle to actively check the fix and re-appearance of vulnerability. 
 
-or download it from [nuclei templates](https://github.com/projectdiscovery/nuclei-templates) Github project.
+<h1 align="left">
+  <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/"><img src="static/learn-more-button.png" width="170px" alt="Learn More"></a>
+</h1>
 
-```bash
-> git clone https://github.com/projectdiscovery/nuclei-templates.git
-```
+<table>
+<tr>
+<td>  
 
-# Running nuclei
+**For bugbounty hunters:**
 
-### 1. Running nuclei with a single template. 
+Nuclei allows you to customise your testing approach with your own suite of checks and easily run across your bug bounty programs. Moreover, Nuclei can be easily integrated into any continuous scanning workflow.
 
-This will run the tool against all the hosts in `urls.txt` and returns the matched results. 
+- Designed to be easily integrated into other tool workflow.
+- Can process thousands of hosts in few minutes.
+- Easily automate your custom testing approach with our simple YAML DSL.
 
-```bash
-> nuclei -l urls.txt -t files/git-core.yaml -o results.txt
-```
+Please check our other open-source projects that might fit into your bug bounty workflow: [github.com/projectdiscovery](http://github.com/projectdiscovery), we also host daily [refresh of DNS data at Chaos](http://chaos.projectdiscovery.io).
 
-You can also pass the list of hosts at standard input (STDIN). This allows for easy integration in automation pipelines.
+</td>
+</tr>
+</table>
 
-This will run the tool against all the hosts in `urls.txt` and returns the matched results. 
+<table>
+<tr>
+<td>
+  
+**For pentesters:**
 
-```bash
-> cat urls.txt | nuclei -t files/git-core.yaml -o results.txt
-```
+Nuclei immensely improve how you approach security assessment by augmenting the manual repetitve processes. Consultancies are already converting their manual assessment steps with Nuclei, it allows them to run set of their custom assessment approach across thousands of hosts in an automated manner. 
 
-### 2. Running nuclei with multiple templates. 
+Pen-testers get the full power of our public templates and customization capabilities to speed-up their assessment process, and specifically with the regression cycle where you can easily verify the fix.
 
-This will run the tool against all the hosts in `urls.txt` with all the templates in the `path-to-templates` directory and returns the matched results. 
+- Easily create your compliance, standards suite (e.g. OWASP Top 10) checklist.
+- With capabilities like [fuzz](https://nuclei.projectdiscovery.io/templating-guide/#advance-fuzzing) and [workflows](https://nuclei.projectdiscovery.io/templating-guide/#workflows), complex manual steps and repetitive assessment can be easily automated with Nuclei.
+- Easy to re-test vulnerability-fix by just re-running the template.
 
-```bash
-> nuclei -l urls.txt -t cves/ -o results.txt 
-```
-
-### 3. Automating nuclei with subfinder and any other similar tool.
+</td>
+</tr>
+</table>
 
 
-```bash
-> subfinder -d hackerone.com -silent | httpx -silent | nuclei -t cves/ -o results.txt
-```
+# For Developers and Organisations
 
--------
+Nuclei is built with simplicity in mind, with the community backed templates by hundreds of security researchers, it allows you to stay updated with latest security threats using continuous Nuclei scanning on the hosts. It is designed to be easily integrated into regression tests cycle, to verify the fixes and  eliminate vulnerabilities from occuring in future.
 
-**Please refer to [this guide](https://github.com/projectdiscovery/nuclei-templates/blob/master/GUIDE.md) to build your own custom templates.**
+- **CI/CD:** Engineers are already utilising Nuclei within their CI/CD pipeline, it allows them to constantly monitor their staging and production environments with customised templates.
+- **Continuous Regression Cycle:** With Nuclei, you can create your custom template on every new identified vulnerability and put into Nuclei engine to eliminate in the continuous regression cycle.
 
+We have [a discussion thread around this](https://github.com/projectdiscovery/nuclei-templates/discussions/693), there are already some bug bounty programs giving incentives to hackers on writing nuclei templates with every submission, that helps them to eliminate the vulnerability across all their assets, as well as to eliminate future risk in reappearing on productions. If you're interested in implementing it in your organisation, feel free to [reach out to us](mailto:[email protected]). We will be more than happy to help you in the getting started process, or you can also post into the [discussion thread for any help](https://github.com/projectdiscovery/nuclei-templates/discussions/693).
 
-# Thanks
+<h3 align="center">
+  <img src="static/regression-with-nuclei.jpg" alt="regression-cycle-with-nuclei" width="1100px"></a>
+</h3>
 
-nuclei is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)** file for more details. Do also check out these similar awesome projects that may fit in your workflow:
+<h1 align="left">
+  <a href="https://github.com/projectdiscovery/nuclei-action"><img src="static/learn-more-button.png" width="170px" alt="Learn More"></a>
+</h1>
 
-[https://portswigger.net/burp](https://portswigger.net/burp)</br>
-[https://github.com/ffuf/ffuf](https://github.com/ffuf/ffuf)</br>
-[https://github.com/jaeles-project/jaeles](https://github.com/jaeles-project/jaeles)</br>
-[https://github.com/ameenmaali/qsfuzz](https://github.com/ameenmaali/qsfuzz)</br>
-[https://github.com/proabiral/inception](https://github.com/proabiral/inception)</br>
-[https://github.com/hannob/snallygaster](https://github.com/hannob/snallygaster)</br>
+### Resources
+
+- [Community Powered Scanning with Nuclei](https://blog.projectdiscovery.io/community-powered-scanning-with-nuclei/)
+- [Nuclei Unleashed - Quickly write complex exploits](https://blog.projectdiscovery.io/nuclei-unleashed-quickly-write-complex-exploits/)
+- [Nuclei - Fuzz all the things](https://blog.projectdiscovery.io/nuclei-fuzz-all-the-things/)
+- [Nuclei + Interactsh Integration for Automating OOB Testing](https://blog.projectdiscovery.io/nuclei-interactsh-integration/)
+- [Weaponizes nuclei Workflows to Pwn All the Things](https://medium.com/@dwisiswant0/weaponizes-nuclei-workflows-to-pwn-all-the-things-cd01223feb77) by [@dwisiswant0](https://github.com/dwisiswant0)
+- [How to Scan Continuously with Nuclei?](https://medium.com/@dwisiswant0/how-to-scan-continuously-with-nuclei-fcb7e9d8b8b9) by [@dwisiswant0](https://github.com/dwisiswant0)
+- [Hack with Automation !!!](https://dhiyaneshgeek.github.io/web/security/2021/07/19/hack-with-automation/) by [@DhiyaneshGeek](https://github.com/DhiyaneshGeek)
+
+### Credits
+
+Thanks to all the amazing community [contributors for sending PRs](https://github.com/projectdiscovery/nuclei/graphs/contributors). Do also check out the below similar open-source projects that may fit in your workflow:
+
+[FFuF](https://github.com/ffuf/ffuf), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [Jaeles](https://github.com/jaeles-project/jaeles), [ChopChop](https://github.com/michelin/ChopChop)
+
+### License
+
+Nuclei is distributed under [MIT License](https://github.com/projectdiscovery/nuclei/blob/master/LICENSE.md)
+
+<h1 align="left">
+  <a href="https://discord.gg/projectdiscovery"><img src="static/Join-Discord.png" width="380" alt="Join Discord"></a> <a href="https://nuclei.projectdiscovery.io"><img src="static/check-nuclei-documentation.png" width="380" alt="Check Nuclei Documentation"></a>
+</h1>

README_CN.md

@@ -0,0 +1,253 @@
+<h1 align="center">
+  <img src="static/nuclei-logo.png" alt="nuclei" width="200px"></a>
+  <br>
+</h1>
+
+[![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT)
+[![Go Report Card](https://goreportcard.com/badge/github.com/projectdiscovery/nuclei)](https://goreportcard.com/report/github.com/projectdiscovery/nuclei)
+[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectdiscovery/nuclei/issues)
+[![GitHub Release](https://img.shields.io/github/release/projectdiscovery/nuclei)](https://github.com/projectdiscovery/nuclei/releases)
+[![Follow on Twitter](https://img.shields.io/twitter/follow/pdnuclei.svg?logo=twitter)](https://twitter.com/pdnuclei)
+[![Docker Images](https://img.shields.io/docker/pulls/projectdiscovery/nuclei.svg)](https://hub.docker.com/r/projectdiscovery/nuclei)
+[![Chat on Discord](https://img.shields.io/discord/695645237418131507.svg?logo=discord)](https://discord.gg/KECAGdH)
+
+<p align="center">
+<a href="https://nuclei.projectdiscovery.io/templating-guide/" target="_blank"><img src="static/read-the-docs-button.png" height="42px"/></center></a>  <a href="https://github.com/projectdiscovery/nuclei-templates" target="_blank"><img src="static/download-templates-button.png" height="42px"/></a>
+</p>
+
+
+Nuclei是一个基于模板的、可配置攻击目标的扫描快速工具，同时还提供了强大的可扩展性和易用性。
+
+基于模板的nuclei被用来发送请求给目标，有着实现零误报的优点，并且可以对已知的路径进行有效的扫描。nuclei的主要用于在初期的探测阶段快速地对已知的且易于检测的漏洞或者CVE进行扫描。如果存在WAF的话，nuclei使用[retryablehttp-go库](https://github.com/projectdiscovery/retryablehttp-go)来处理各种错误，并且重新尝试攻击，这也是我们自定义功能的核心模块之一。
+
+我们也维护一个具有各个类型的模板的[开源库](https://github.com/projectdiscovery/nuclei-templates)，我们希望你也能贡献一些模板，贡献的这些模板最好是有效的，并且能允许每个人基于你的模板重新构建。查看[**nuclei.projectdiscovery.io**](https://nuclei.projectdiscovery.io/templating-guide/)这个网站去学习制作模板的入门知识。
+
+## 目录
+
+- [目录](#目录)
+- [功能](#功能)
+- [安装](#安装)
+- [Nuclei模板](#nuclei模板)
+- [用法](#用法)
+- [运行Nuclei](#运行nuclei)
+- [排除模板](#排除模板)
+- [致谢](#致谢)
+
+## 功能
+
+<h1 align="left">
+  <img src="static/nuclei-run.png" alt="nuclei" width="700px"></a>
+  <br>
+</h1>
+
+- 有着易于开发的、简单的、模块化的代码库
+- 使用了基于模板的引擎，运行速度极快，可以修改所以配置
+- 可以对特殊情况处理、重试、绕过等，可以绕过WAF
+- 智能匹配，零误报
+
+## 安装
+
+### 二进制文件安装
+
+二进制文件安装很简单，你可以从[Releases](https://github.com/projectdiscovery/nuclei/releases/)页面下载已经构建好的二进制文件压缩包，使用解压工具提取下载的压缩包，并将解压的文件夹移动到$PATH目录，就可以直接使用了。
+
+```sh
+Download latest binary from https://github.com/projectdiscovery/nuclei/releases
+
+▶ tar -xzvf nuclei-linux-amd64.tar.gz
+▶ mv nuclei /usr/local/bin/
+▶ nuclei -version
+```
+
+### 源码安装
+
+nuclei需要**go1.14+**才能成功安装，运行以下命令获取repo
+
+```sh
+▶ GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+```
+
+### GitHub安装
+
+```sh
+▶ git clone https://github.com/projectdiscovery/nuclei.git; cd nuclei/v2/cmd/nuclei/; go build; mv nuclei /usr/local/bin/; nuclei -version
+```
+
+## Nuclei模板
+
+你可以使用`update-templates`来下载和更新nuclei模板，该命令会从unclei的[模板库]()中下载最新版本，这个由社区来维护的库是可以随时使用的。
+
+```sh
+▶ nuclei -update-templates
+```
+
+此外您可以根据自己的工作情况或者需求编写模板，请参阅**nuclei[模板向导](https://nuclei.projectdiscovery.io/templating-guide/)去编写自定义模板**
+
+## 用法
+
+```sh
+nuclei -h
+```
+
+这条命令会显示帮助，以下是nuclei支持的所有命令
+
+|命令|描述|例子|
+|-----|-----|-----|
+|bulk-size|每个模板最大并行的主机数(默认25)|nuclei -bulk-size 25|
+|burp-collaborator-biid|使用burp-collaborator插件|nuclei -burp-collaborator-biid XXXX|
+|c|并行的最大模板数量(默认10)|nuclei -c 10|
+|l|对URL列表进行测试|nuclei -l urls.txt|
+|target|对目标进行测试|nuclei -target hxxps://example.com|
+|t|要检测的模板种类|nuclei -t git-core.yaml -t cves/|
+|no-color|输出不显示颜色|nuclei -no-color|
+|no-meta|不显示匹配的元数据|nuclei -no-meta|
+|json|输出为json格式|nuclei -json|
+|include-rr|json输出格式中包含请求和响应数据|nuclei -json -include-rr|
+|o|输出为文件|nuclei -o output.txt|
+|project|避免发送相同的请求|nuclei -project|
+|stats|使用进度条|nuclei -stats|
+|silent|只输出测试成功的结果|nuclei -silent|
+|retries|失败后的重试次数|nuclei -retries 1|
+|timeout|超时时间(默认为5秒)|nuclei -timeout 5|
+|trace-log|输出日志到log文件|nuclei -trace-log logs|
+|rate-limit|每秒最大请求数(默认150)|nuclei -rate-limit 150|
+|severity|根据严重性选择模板|nuclei  -severity critical,high|
+|stop-at-first-match|第一次匹配不要处理HTTP请求|nuclei -stop-at-frst-match|
+|exclude|排除的模板或文件夹|nuclei -exclude panels -exclude tokens|
+|debug|调试请求或者响应|nuclei -debug|
+|update-templates|下载或者升级模板|nuclei -update-templates|
+|update-directory|选择储存模板的目录(可选)|nuclei -update-directory templates|
+|tl|列出可用的模板|nuclei -tl|
+|templates-version|显示已安装的模板版本|nuclei -templates-version|
+|v|显示发送请求的详细信息|nuclei -v|
+|version|显示nuclei的版本号|nuclei -version|
+|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080|
+|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080|
+|random-agent|使用随机的UA|nuclei -random-agent|
+|H|自定义请求头|nuclei -H “x-bug-bounty:hacker”|
+
+## 运行Nuclei
+
+### 运行单个模板
+
+这将对`urls.txt`中所有的主机运行`git-core.yaml`并返回结果到`results.txt`
+
+```sh
+▶ nuclei -l urls.txt -t files/git-core.yaml -o results.txt
+```
+
+你可以轻松的通过管道使用标准的输入(STDIN)传递URL列表。
+
+```sh
+▶ cat urls.txt | nuclei -t files/git-core.yaml -o results.txt
+```
+
+💡 Nuclei可以接受如下列表的URL作为输入，例如以下URL：
+
+```
+https://test.some-site.com
+http://vuls-testing.com
+https://test.com
+```
+### 运行多个模板
+
+这将会对`urls.txt`中所有的URL运行`cves`和`files`模板检查，并返回输出到`results.txt`
+
+```sh
+▶ nuclei -l urls.txt -t cves/ -t files/ -o results.txt
+```
+
+### 使用subfinder运行
+
+```sh
+▶ subfinder -d hackerone.com -silent | httpx -silent | nuclei -t cves/ -o results.txt
+```
+
+### 在docker中运行
+
+你需要使用[nuclei的docker镜像](https://hub.docker.com/r/projectdiscovery/nuclei)来运行
+
+```sh
+▶ docker pull projectdiscovery/nuclei
+```
+
+下载并构建完成后，运行以下命令：
+
+```sh
+▶ docker run -it projectdiscovery/nuclei
+```
+
+这将会对`urls.txt`中的URL通过docker中的nuclei进行检测，并将结果输出到本机的`results.txt`文件的：
+
+```sh
+▶ cat urls.txt | docker run -v /path/to/nuclei-templates:/app/nuclei-templates -v /path/to/nuclei/config:/app/.nuclei-config.json -i projectdiscovery/nuclei -t /app/nuclei-templates/files/git-config.yaml > results.txt
+```
+记住更改的模板路径到本机
+
+### 速率限制
+
+Nuclei有多种控制速率的方法，包括并行执行多个模板、并行检查多个主机，以及使nuclei限制全局的请求速率，下面就是示例。
+
+- `-c`参数 => 限制并行的模板数
+- `-bulk-size`参数 => 限制并行的主机数
+- `-rate-limit`参数 => 全局速率限制
+
+如果你想快速扫描或者控制扫描，请使用这些标志并输入限制数，`速率限制`只保证控制传出的请求，与其他参数无关。
+
+### 排除模板
+
+[Nuclei模板](https://github.com/projectdiscovery/nuclei-templates)包含多种检查，其中有许多对攻击有用的检查，但并不是都有用的。如果您只希望扫描少数特定的模板或目录，则可以使用如下的参数筛选模板，或将某些模板排除。
+
+#### 排除模板运行
+
+我们不建议同时运行所有的nuclei模板，如果要排除模板，可以使用`exclude`参数来排除特定的目录或模板。
+
+```sh
+nuclei -l urls.txt -t nuclei-templates -exclude panels/ -exclude technologies -exclude files/wp-xmlrpc.yaml
+```
+
+注意：如上述示例中显示的那样，目录和特定模板都将不会扫描
+
+#### 基于严重性运行模板
+
+您可以根据模板的严重性运行模板，扫描时可以选择单个严重性或多个严重性。
+
+```sh
+nuclei -l urls.txt -t cves/ -severity critical,medium
+```
+
+上面的例子将运行`cves`目录下所有`严重`和`中等`的模板。
+
+```sh
+nuclei -l urls.txt -t panels/ -t technologies -severity info
+```
+
+上面的例子将运行`panels`和`technologies`目录下严重性标记为`info`的模板
+
+#### 使用`.nuclei-ignore`文件排除模板
+
+自从nuclei的[v2.1.1版本](https://github.com/projectdiscovery/nuclei/releases/tag/v2.1.1)以来，我们添加了对`.nuclei-ignore`文件的支持，该文件与`update-templates`参数一起使用，在 **.nuclei-ignore** 文件中，您可以定义要从nuclei扫描中排除的所有模板目录或者模板路径，要开始使用此功能，请确保使用`nuclei-update-templates`参数安装nuclei模板，现在可以根据`.nuclei-ignore`的文件来添加、更新、删除模板文件。
+
+```
+nano ~/nuclei-templates/.nuclei-ignore
+```
+
+默认的**nuclei忽略**列表可以访问[这里]((https://github.com/projectdiscovery/nuclei-templates/blob/master/.nuclei-ignore)，如果不想排除任何内容，只需要删除`.nuclei-ignore`文件。
+
+* * *
+
+### 📋 笔记
+
+- 进度条是实验性功能，在某些情况下可能无法使用。
+- 进度条不适用于工作流，因为是条件执行，所以不准确。
+
+
+## 致谢
+
+也要看看这些类似的好项目，或许它们也适合你：
+
+[Burp Suite](https://portswigger.net/burp), [FFuF](https://github.com/ffuf/ffuf), [Jaeles](https://github.com/jaeles-project/jaeles), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [ChopChop](https://github.com/michelin/ChopChop)
+
+--------
+
+Nuclei是由[projectdiscovery](https://projectdiscovery.io)团队用🖤制作的，当然社区也贡献了很多，通过 **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)**文件以获取更多详细信息。

SECURITY.md

@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+DO NOT CREATE AN ISSUE to report a security problem. Instead, please send an email to security@projectdiscovery.io and we will acknowledge it within 3 working days.

THANKS.md

@@ -1 +1,17 @@
 
+### Thanks
+
+Many people have contributed to **nuclei** making it a wonderful tool either by making a pull request fixing some stuff. Here, we recognize these persons and thank them. 
+
+- All the contributors at [CONTRIBUTORS](https://github.com/projectdiscovery/nuclei/graphs/contributors) who made Nuclei what it is.
+
+We'd like to thank some additional amazing people, who contributed a lot in nuclei's journey - 
+
+- [@manuelbua](https://www.github.com/manuelbua) - Multiple feature additions including progress bar. 
+- [@dwisiswant0](https://www.github.com/dwisiswant0) - For fixing multiple bugs with HTTP Raw requests. 
+- [@toufik-airane](https://www.github.com/toufik-airane) - For adding binary matchers.
+- [@lc](https://www.github.com/lc) - For adding directory support for templates.
+- [@wdahlenburg](https://www.github.com/wdahlenburg) - For adding wildcard globing support to run templates.
+- [@Marmelatze](https://www.github.com/Marmelatze) - For adding support to save matched request/response in JSON format. 
+- [@ankh2054](https://www.github.com/ankh2054) - For adding description field for template information.
+- [@rotemreiss](https://www.github.com/rotemreiss) - For adding docker support.

go.mod

@@ -1,22 +0,0 @@
-module github.com/projectdiscovery/nuclei
-
-go 1.14
-
-require (
-	github.com/Knetic/govaluate v3.0.0+incompatible
-	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
-	github.com/blang/semver v3.5.1+incompatible
-	github.com/d5/tengo v1.24.8
-	github.com/d5/tengo/v2 v2.6.0
-	github.com/google/go-github v17.0.0+incompatible
-	github.com/google/go-github/v32 v32.0.0
-	github.com/json-iterator/go v1.1.10
-	github.com/karrick/godirwalk v1.15.6
-	github.com/miekg/dns v1.1.29
-	github.com/pkg/errors v0.9.1
-	github.com/projectdiscovery/gologger v1.0.0
-	github.com/projectdiscovery/retryabledns v1.0.4
-	github.com/projectdiscovery/retryablehttp-go v1.0.1
-	golang.org/x/net v0.0.0-20200528225125-3c3fba18258b
-	gopkg.in/yaml.v2 v2.3.0
-)

go.sum

@@ -1,71 +0,0 @@
-github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4=
-github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
-github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
-github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg=
-github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
-github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
-github.com/d5/tengo v1.24.8/go.mod h1:VhLq8Q2QFhCIJO3NhvM934qOThykMqJi9y9Siqd1ocQ=
-github.com/d5/tengo/v2 v2.6.0 h1:D0cJtpiBzaLJ/Smv6nnUc/LIfO46oKwDx85NZtIRNRI=
-github.com/d5/tengo/v2 v2.6.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
-github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
-github.com/google/go-github/v32 v32.0.0 h1:q74KVb22spUq0U5HqZ9VCYqQz8YRuOtL/39ZnfwO+NM=
-github.com/google/go-github/v32 v32.0.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
-github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
-github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
-github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA=
-github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
-github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
-github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s=
-github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
-github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60=
-github.com/projectdiscovery/retryabledns v1.0.4/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
-github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
-github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200528225125-3c3fba18258b h1:IYiJPiJfzktmDAO1HQiwjMjwjlYKHAL7KzeD544RJPs=
-golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

integration_tests/dns/basic.yaml

@@ -0,0 +1,17 @@
+id: basic-dns-example
+
+info:
+  name: Test DNS Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: A
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        words:
+          - "1.1.1.1"

integration_tests/http/get-headers.yaml

@@ -0,0 +1,17 @@
+id: basic-get-headers
+
+info:
+  name: Basic GET Headers Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    headers:
+      test: nuclei
+    matchers:
+      - type: word
+        words:
+          - "This is test headers matcher text"

integration_tests/http/get-query-string.yaml

@@ -0,0 +1,15 @@
+id: basic-get-querystring
+
+info:
+  name: Basic GET QueryString Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}?test=nuclei"
+    matchers:
+      - type: word
+        words:
+          - "This is test querystring matcher text"

integration_tests/http/get-redirects.yaml

@@ -0,0 +1,17 @@
+id: basic-get-redirects
+
+info:
+  name: Basic GET Redirects Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    redirects: true
+    max-redirects: 3
+    matchers:
+      - type: word
+        words:
+          - "This is test redirects matcher text"

integration_tests/http/get.yaml

@@ -0,0 +1,15 @@
+id: basic-get
+
+info:
+  name: Basic GET Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "This is test matcher text"

integration_tests/http/post-body.yaml

@@ -0,0 +1,19 @@
+id: basic-post-body
+
+info:
+  name: Basic POST Body Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: POST
+    path:
+      - "{{BaseURL}}"
+    headers: 
+      Content-Type: application/x-www-form-urlencoded
+      Content-Length: 1 # as long as there is a value, nuclei will auto-recalculate it.
+    body: username=test&password=nuclei
+    matchers:
+      - type: word
+        words:
+          - "This is test post-body matcher text"

integration_tests/http/post-json-body.yaml

@@ -0,0 +1,19 @@
+id: basic-post-json-body
+
+info:
+  name: Basic POST JSON Body Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: POST
+    path:
+      - "{{BaseURL}}"
+    headers: 
+      Content-Type: application/json
+      Content-Length: 1
+    body: '{"username":"test","password":"nuclei"}'
+    matchers:
+      - type: word
+        words:
+          - "This is test post-json-body matcher text"

integration_tests/http/post-multipart-body.yaml

@@ -0,0 +1,29 @@
+id: basic-post-multipart-body
+
+info:
+  name: Basic POST Multipart Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: POST
+    path:
+      - "{{BaseURL}}"
+    headers: 
+      Content-Type: multipart/form-data; boundary=d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8
+      Content-Length: 1
+    body: |
+      --d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8
+      Content-Disposition: form-data; name="username"; filename="username"
+      Content-Type: application/octet-stream
+
+      test
+      --d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8
+      Content-Disposition: form-data; name="password"
+
+      nuclei
+      --d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8--
+    matchers:
+      - type: word
+        words:
+          - "This is test post-multipart matcher text"

integration_tests/http/raw-cookie-reuse.yaml

@@ -0,0 +1,34 @@
+id: cookiereuse-raw-example
+info:
+  name: Test CookieReuse RAW Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        POST / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+
+        testing=parameter
+      - |
+        GET / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+        
+    cookie-reuse: true
+    matchers:
+      - type: word
+        words:
+          - "Test is test-cookie-reuse matcher text"

integration_tests/http/raw-dynamic-extractor.yaml

@@ -0,0 +1,43 @@
+id: dynamic-extractor-raw-example
+
+info:
+  name: Test Dynamic Extractor RAW Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        POST / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+
+        testing=parameter
+      - |
+        GET /?username={{randkey}} HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+    extractors:
+      - type: regex
+        name: randkey
+        part: body
+        group: 1
+        internal: true
+        regex:
+          - "Token: '([A-Za-z0-9]+)'"
+
+    cookie-reuse: true
+    matchers:
+      - type: word
+        words:
+          - "Test is test-dynamic-extractor-raw matcher text"

integration_tests/http/raw-get-query.yaml

@@ -0,0 +1,18 @@
+id: basic-raw-query-example
+
+info:
+  name: Test RAW GET Query Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        GET ?test=nuclei HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+
+    matchers:
+      - type: word
+        words:
+          - "Test is test raw-get-query-matcher text"

integration_tests/http/raw-get.yaml

@@ -0,0 +1,18 @@
+id: basic-raw-http-example
+
+info:
+  name: Test RAW GET Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        GET / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+
+    matchers:
+      - type: word
+        words:
+          - "Test is test raw-get-matcher text"

integration_tests/http/raw-payload.yaml

@@ -0,0 +1,29 @@
+id: payload-raw-example
+info:
+  name: Test RAW With Payload Template
+  author: pdteam
+  severity: info
+
+requests:
+  - payloads:
+      username:
+        - test
+      password:
+        - nuclei
+        - guest
+    attack: clusterbomb
+    raw:
+      - |
+        POST / HTTP/1.1
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5)
+        Host: {{Hostname}}
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+        another_header: {{base64('§password§')}}
+        Accept: */*
+
+        username=§username§&password={{password}}
+    matchers:
+      - type: word
+        words:
+          - "Test is raw-payload matcher text"

integration_tests/http/raw-post-body.yaml

@@ -0,0 +1,21 @@
+id: basic-raw-http-body-example
+
+info:
+  name: Test RAW POST Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        POST / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+
+        username=test&password=nuclei
+    matchers:
+      - type: word
+        words:
+          - "Test is test raw-post-body-matcher text"

integration_tests/http/raw-unsafe-request.yaml

@@ -0,0 +1,20 @@
+id: basic-raw-unsafe-request-example
+
+info:
+  name: Test RAW Unsafe Request Template
+  author: pd-team
+  severity: info
+
+requests:
+  - raw:
+      - |
+        GET / HTTP/1.1
+        Host: 
+        Content-Length: 4
+
+    unsafe: true
+    matchers-condition: and
+    matchers:
+      - type: word
+        words:
+          - "This is test raw-unsafe-matcher test"

integration_tests/http/request-condition-new.yaml

@@ -0,0 +1,19 @@
+id: request-condition-new
+
+info:
+  name: request-condition-new
+  author: pd-team
+  severity: info
+
+requests:
+  - method: GET
+    id: first
+    path:
+      - "{{BaseURL}}/200"
+  - method: GET
+    path:
+      - "{{BaseURL}}/400"
+    matchers:
+      - type: dsl
+        dsl:
+          - "first_status_code==200 && status_code==400"

integration_tests/http/request-condition.yaml

@@ -0,0 +1,18 @@
+id: request-condition
+
+info:
+  name: request-condition
+  author: pd-team
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}/200"
+      - "{{BaseURL}}/400"
+ 
+    req-condition: true
+    matchers:
+      - type: dsl
+        dsl:
+          - "status_code_1==200 && status_code_2==400"

integration_tests/network/basic.yaml

@@ -0,0 +1,18 @@
+id: basic-network-request
+
+info:
+  name: Basic Network Request
+  author: pdteam
+  severity: info
+
+network:
+  - host: 
+      - "{{Hostname}}"
+    inputs:
+      - data: "PING\r\n"
+    read-size: 4
+    matchers:
+      - type: word
+        part: data
+        words:
+          - "PONG"

integration_tests/network/hex.yaml

@@ -0,0 +1,22 @@
+id: hex-network-request
+
+info:
+  name: Hex Input Network Request
+  author: pdteam
+  severity: info
+
+network:
+  - host: 
+      - "{{Hostname}}"
+    inputs:
+      - data: "50494e47"
+        type: hex
+      - data: "\r\n"
+
+    read-size: 4
+    matchers:
+      - type: word
+        part: data
+        encoding: hex
+        words:
+          - "504f4e47"

integration_tests/network/multi-step.yaml

@@ -0,0 +1,32 @@
+id: multi-step
+
+info:
+  name: Multi-Step Network Template
+  author: pd-team
+  severity: info
+
+network:
+  - inputs:
+      - data: "FIRST"
+        read: 4
+        name: first
+      - data: "SECOND"
+        read: 4
+        name: second
+    host:
+      - "{{Hostname}}"
+    read-size: 6
+    matchers:
+      - type: word
+        part: first
+        words:
+          - "PING" 
+      - type: word
+        part: second
+        words:
+          - "PONG" 
+      - type: word
+        part: data
+        words:
+          - "NUCLEI" 
+    matchers-condition: and

integration_tests/run.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+cd ../v2/cmd/nuclei
+go build
+mv nuclei ../../../integration_tests/nuclei 
+cd ../integration-test
+go build
+mv integration-test ../../../integration_tests/integration-test 
+cd ../../../integration_tests
+./integration-test
+if [ $? -eq 0 ]
+then
+  exit 0
+else
+  exit 1
+fi

integration_tests/workflow/basic.yaml

@@ -0,0 +1,10 @@
+id: workflow-example
+
+info:
+  name: Test Workflow Template
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-1.yaml
+  - template: workflow/match-2.yaml

integration_tests/workflow/condition-matched.yaml

@@ -0,0 +1,11 @@
+id: condition-matched-workflow
+
+info:
+  name: Condition Matched Workflow
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-1.yaml
+    subtemplates:
+      - template: workflow/match-2.yaml

integration_tests/workflow/condition-unmatched.yaml

@@ -0,0 +1,11 @@
+id: condition-unmatched-workflow
+
+info:
+  name: Condition UnMatched Workflow
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/nomatch-1.yaml
+    subtemplates:
+      - template: workflow/match-2.yaml

integration_tests/workflow/match-1.yaml

@@ -0,0 +1,15 @@
+id: basic-get
+
+info:
+  name: Basic GET Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "This is test matcher text"

integration_tests/workflow/match-2.yaml

@@ -0,0 +1,16 @@
+id: basic-get-another
+
+info:
+  name: Basic Another GET Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        name: test-matcher
+        words:
+          - "This is test matcher text"

integration_tests/workflow/matcher-name.yaml

@@ -0,0 +1,13 @@
+id: matcher-name-workflow
+
+info:
+  name: Matcher Name Workflow
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-2.yaml
+    matchers: 
+      - name: test-matcher
+        subtemplates:
+          - template: workflow/match-1.yaml

integration_tests/workflow/nomatch-1.yaml

@@ -0,0 +1,15 @@
+id: basic-get-nomatch
+
+info:
+  name: Basic GET Request NoMatch
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "Random"

static/regression-cycle.md

@@ -0,0 +1 @@
+

v2/.goreleaser.yml

@@ -0,0 +1,33 @@
+before:
+  hooks:
+    - go mod tidy
+
+builds:
+- env:
+  - CGO_ENABLED=0
+  goos:
+    - windows
+    - linux
+    - darwin
+  goarch:
+    - amd64
+    - 386
+    - arm
+    - arm64
+
+  ignore:
+    - goos: darwin
+      goarch: '386'
+    - goos: windows
+      goarch: 'arm'
+
+  binary: '{{ .ProjectName }}'
+  main: cmd/nuclei/main.go
+
+archives:
+- format: zip
+  replacements:
+      darwin: macOS
+
+checksum:
+  algorithm: sha256

v2/Makefile

@@ -0,0 +1,14 @@
+# Go parameters
+GOCMD=go
+GOBUILD=$(GOCMD) build
+GOMOD=$(GOCMD) mod
+GOTEST=$(GOCMD) test
+GOGET=$(GOCMD) get
+    
+all: build
+build:
+		$(GOBUILD) -v -ldflags="-extldflags=-static" -o "nuclei" cmd/nuclei/main.go
+test: 
+		$(GOTEST) -v ./...
+tidy:
+		$(GOMOD) tidy

v2/cmd/integration-test/dns.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var dnsTestCases = map[string]testutils.TestCase{
+	"dns/basic.yaml": &dnsBasic{},
+}
+
+type dnsBasic struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *dnsBasic) Execute(filePath string) error {
+	var routerErr error
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, "one.one.one.one", debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/integration-test/http.go

@@ -0,0 +1,523 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"strings"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var httpTestcases = map[string]testutils.TestCase{
+	"http/get-headers.yaml":           &httpGetHeaders{},
+	"http/get-query-string.yaml":      &httpGetQueryString{},
+	"http/get-redirects.yaml":         &httpGetRedirects{},
+	"http/get.yaml":                   &httpGet{},
+	"http/post-body.yaml":             &httpPostBody{},
+	"http/post-json-body.yaml":        &httpPostJSONBody{},
+	"http/post-multipart-body.yaml":   &httpPostMultipartBody{},
+	"http/raw-cookie-reuse.yaml":      &httpRawCookieReuse{},
+	"http/raw-dynamic-extractor.yaml": &httpRawDynamicExtractor{},
+	"http/raw-get-query.yaml":         &httpRawGetQuery{},
+	"http/raw-get.yaml":               &httpRawGet{},
+	"http/raw-payload.yaml":           &httpRawPayload{},
+	"http/raw-post-body.yaml":         &httpRawPostBody{},
+	"http/raw-unsafe-request.yaml":    &httpRawUnsafeRequest{},
+	"http/request-condition.yaml":     &httpRequestCondition{},
+	"http/request-condition-new.yaml": &httpRequestCondition{},
+}
+
+func httpDebugRequestDump(r *http.Request) {
+	if debug {
+		if dump, err := httputil.DumpRequest(r, true); err == nil {
+			fmt.Printf("\nRequest dump: \n%s\n\n", string(dump))
+		}
+	}
+}
+
+type httpGetHeaders struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGetHeaders) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.Header.Get("test"), "nuclei") {
+			fmt.Fprintf(w, "This is test headers matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpGetQueryString struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGetQueryString) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") {
+			fmt.Fprintf(w, "This is test querystring matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpGetRedirects struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGetRedirects) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		http.Redirect(w, r, "/redirected", http.StatusFound)
+	}))
+	router.GET("/redirected", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test redirects matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpGet struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGet) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpPostBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpPostBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("username"), "test") && strings.EqualFold(r.Form.Get("password"), "nuclei") {
+			fmt.Fprintf(w, "This is test post-body matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpPostJSONBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpPostJSONBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+
+		type doc struct {
+			Username string `json:"username"`
+			Password string `json:"password"`
+		}
+		obj := &doc{}
+		if err := json.NewDecoder(r.Body).Decode(obj); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(obj.Username, "test") && strings.EqualFold(obj.Password, "nuclei") {
+			fmt.Fprintf(w, "This is test post-json-body matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpPostMultipartBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpPostMultipartBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseMultipartForm(1 * 1024); err != nil {
+			routerErr = err
+			return
+		}
+		password, ok := r.MultipartForm.Value["password"]
+		if !ok || len(password) != 1 {
+			routerErr = errors.New("no password in request")
+			return
+		}
+		file := r.MultipartForm.File["username"]
+		if len(file) != 1 {
+			routerErr = errors.New("no file in request")
+			return
+		}
+		if strings.EqualFold(password[0], "nuclei") && strings.EqualFold(file[0].Filename, "username") {
+			fmt.Fprintf(w, "This is test post-multipart matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawDynamicExtractor struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawDynamicExtractor) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("testing"), "parameter") {
+			fmt.Fprintf(w, "Token: 'nuclei'")
+		}
+	}))
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.URL.Query().Get("username"), "nuclei") {
+			fmt.Fprintf(w, "Test is test-dynamic-extractor-raw matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawGetQuery struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawGetQuery) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") {
+			fmt.Fprintf(w, "Test is test raw-get-query-matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawGet struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawGet) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+
+		fmt.Fprintf(w, "Test is test raw-get-matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawPayload struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawPayload) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if !(strings.EqualFold(r.Header.Get("another_header"), "bnVjbGVp") || strings.EqualFold(r.Header.Get("another_header"), "Z3Vlc3Q=")) {
+			return
+		}
+		if strings.EqualFold(r.Form.Get("username"), "test") && (strings.EqualFold(r.Form.Get("password"), "nuclei") || strings.EqualFold(r.Form.Get("password"), "guest")) {
+			fmt.Fprintf(w, "Test is raw-payload matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 2 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawPostBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawPostBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("username"), "test") && strings.EqualFold(r.Form.Get("password"), "nuclei") {
+			fmt.Fprintf(w, "Test is test raw-post-body-matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawCookieReuse struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawCookieReuse) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("testing"), "parameter") {
+			http.SetCookie(w, &http.Cookie{Name: "nuclei", Value: "test"})
+		}
+	}))
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		cookie, err := r.Cookie("nuclei")
+		if err != nil {
+			routerErr = err
+			return
+		}
+
+		if strings.EqualFold(cookie.Value, "test") {
+			fmt.Fprintf(w, "Test is test-cookie-reuse matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawUnsafeRequest struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawUnsafeRequest) Execute(filePath string) error {
+	var routerErr error
+
+	ts := testutils.NewTCPServer(func(conn net.Conn) {
+		defer conn.Close()
+		_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 36\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nThis is test raw-unsafe-matcher test"))
+	})
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, "http://"+ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRequestCondition struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRequestCondition) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.GET("/200", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		w.WriteHeader(200)
+	}))
+	router.GET("/400", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		w.WriteHeader(400)
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/integration-test/integration-test.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var (
+	debug      = os.Getenv("DEBUG") == "true"
+	customTest = os.Getenv("TEST")
+	protocol   = os.Getenv("PROTO")
+)
+
+func main() {
+	success := aurora.Green("[✓]").String()
+	failed := aurora.Red("[✘]").String()
+
+	protocolTests := map[string]map[string]testutils.TestCase{
+		"http":     httpTestcases,
+		"network":  networkTestcases,
+		"dns":      dnsTestCases,
+		"workflow": workflowTestcases,
+	}
+	for proto, tests := range protocolTests {
+		if protocol == "" || protocol == proto {
+			fmt.Printf("Running test cases for \"%s\" protocol\n", aurora.Blue(proto))
+
+			for file, test := range tests {
+				if customTest != "" && !strings.Contains(file, customTest) {
+					continue // only run tests user asked
+				}
+				err := test.Execute(file)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, file, err)
+					os.Exit(1)
+				} else {
+					fmt.Printf("%s Test \"%s\" passed!\n", success, file)
+				}
+			}
+		}
+	}
+}
+
+func errIncorrectResultsCount(results []string) error {
+	return fmt.Errorf("incorrect number of results %s", strings.Join(results, "\n\t"))
+}

v2/cmd/integration-test/network.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"net"
+
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var networkTestcases = map[string]testutils.TestCase{
+	"network/basic.yaml":      &networkBasic{},
+	"network/hex.yaml":        &networkBasic{},
+	"network/multi-step.yaml": &networkMultiStep{},
+}
+
+type networkBasic struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *networkBasic) Execute(filePath string) error {
+	var routerErr error
+
+	ts := testutils.NewTCPServer(func(conn net.Conn) {
+		defer conn.Close()
+
+		data := make([]byte, 4)
+		if _, err := conn.Read(data); err != nil {
+			routerErr = err
+			return
+		}
+		if string(data) == "PING" {
+			_, _ = conn.Write([]byte("PONG"))
+		}
+	})
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type networkMultiStep struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *networkMultiStep) Execute(filePath string) error {
+	var routerErr error
+
+	ts := testutils.NewTCPServer(func(conn net.Conn) {
+		defer conn.Close()
+
+		data := make([]byte, 5)
+		if _, err := conn.Read(data); err != nil {
+			routerErr = err
+			return
+		}
+		if string(data) == "FIRST" {
+			_, _ = conn.Write([]byte("PING"))
+		}
+
+		data = make([]byte, 6)
+		if _, err := conn.Read(data); err != nil {
+			routerErr = err
+			return
+		}
+		if string(data) == "SECOND" {
+			_, _ = conn.Write([]byte("PONG"))
+		}
+		_, _ = conn.Write([]byte("NUCLEI"))
+	})
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/integration-test/workflow.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var workflowTestcases = map[string]testutils.TestCase{
+	"workflow/basic.yaml":               &workflowBasic{},
+	"workflow/condition-matched.yaml":   &workflowConditionMatched{},
+	"workflow/condition-unmatched.yaml": &workflowConditionUnmatch{},
+	"workflow/matcher-name.yaml":        &workflowMatcherName{},
+}
+
+type workflowBasic struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowBasic) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 2 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type workflowConditionMatched struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowConditionMatched) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type workflowConditionUnmatch struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowConditionUnmatch) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 0 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type workflowMatcherName struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowMatcherName) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/nuclei/issue-tracker-config.yaml

@@ -0,0 +1,49 @@
+#allow-list:
+#  severity: high,critical
+#deny-list:
+#  severity: low
+
+# github contains configuration options for github issue tracker
+#github: 
+#  # base-url is the optional self-hosted github application url
+#  base-url: ""
+#  # username is the username of the github user
+#  username: ""
+#  # owner is the owner name of the repository for issues.
+#  owner: ""
+#  # token is the token for github account.
+#  token: ""
+#  # project-name is the name of the repository.
+#  project-name: ""
+#  # issue-label is the label of the created issue type
+#  issue-label: ""
+
+# gitlab contains configuration options for gitlab issue tracker
+#gitlab: 
+#  # base-url is the optional self-hosted gitlab application url
+#  base-url: ""
+#  # username is the username of the gitlab user
+#  username: ""
+#  # token is the token for gitlab account.
+#  token: ""
+#  # project-id is the ID of the repository.
+#  project-id: ""
+#  # issue-label is the label of the created issue type
+#  issue-label: ""
+
+# jira contains configuration options for jira issue tracker
+#jira:
+#  # Cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
+#  cloud: true
+#  # URL is the jira application url
+#  url: ""
+#  # account-id is the account-id of the jira user or username in case of on-prem Jira
+#  account-id: ""
+#  # email is the email of the user for jira instance
+#  email: ""
+#  # token is the token for jira instance or password in case of on-prem Jira
+#  token: ""
+#  # project-name is the name of the project.
+#  project-name: ""
+#  # issue-type is the name of the created issue type
+#  issue-type: ""

v2/cmd/nuclei/main.go

@@ -1,19 +1,178 @@
 package main
 
 import (
+	"os"
+	"path"
+
+	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/internal/runner"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+var (
+	cfgFile string
+	options = &types.Options{}
 )
 
 func main() {
-	// Parse the command line flags and read config files
-	options := runner.ParseOptions()
+	readConfig()
 
-	runner, err := runner.New(options)
+	runner.ParseOptions(options)
+
+	nucleiRunner, err := runner.New(options)
 	if err != nil {
-		gologger.Fatalf("Could not create runner: %s\n", err)
+		gologger.Fatal().Msgf("Could not create runner: %s\n", err)
+	}
+	if nucleiRunner == nil {
+		return
+	}
+	if err := nucleiRunner.RunEnumeration(); err != nil {
+		gologger.Fatal().Msgf("Could not run nuclei: %s\n", err)
+	}
+	nucleiRunner.Close()
+}
+
+func readConfig() {
+	home, _ := os.UserHomeDir()
+	templatesDirectory := path.Join(home, "nuclei-templates")
+
+	flagSet := goflags.NewFlagSet()
+	flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing
+on extensive configurability, massive extensibility and ease of use.`)
+
+	createGroup(flagSet, "input", "Target",
+		flagSet.StringVarP(&options.Target, "target", "u", "", "target URL/host to scan"),
+		flagSet.StringVarP(&options.Targets, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
+	)
+
+	createGroup(flagSet, "templates", "Templates",
+		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
+
+		flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"),
+		flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "list of workflows to run"),
+
+		flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run newly added templates only"),
+		flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
+	)
+
+	createGroup(flagSet, "filters", "Filtering",
+		flagSet.NormalizedStringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"),
+		flagSet.NormalizedStringSliceVar(&options.IncludeTags, "include-tags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list
+		flagSet.NormalizedStringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"),
+
+		flagSet.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "templates to be executed even if they are excluded either by default or configuration"),
+		flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"),
+
+		flagSet.NormalizedStringSliceVarP(&options.Severity, "impact", "severity", []string{}, "execute templates that match the provided severities only"),
+		flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"),
+	)
+
+	createGroup(flagSet, "output", "Output",
+		flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write found issues/vulnerabilities"),
+
+		flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"),
+		flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"),
+		flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display extra verbose information"),
+		flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"),
+
+		flagSet.BoolVar(&options.JSON, "json", false, "write output in JSONL(ines) format"),
+		flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"),
+
+		flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "don't display match metadata"),
+		flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "local nuclei reporting database (always use this to persist report data)"),
+
+		flagSet.StringVarP(&options.DiskExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"),
+		flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"),
+	)
+
+	createGroup(flagSet, "configs", "Configurations",
+		flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"),
+		flagSet.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking
+
+		flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "custom headers in header:value format"),
+
+		flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"),
+		flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"),
+		flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"),
+	)
+
+	createGroup(flagSet, "interactsh", "interactsh",
+		flagSet.BoolVar(&options.NoInteractsh, "no-interactsh", false, "do not use interactsh server for blind interaction polling"),
+		flagSet.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "self-hosted Interactsh Server URL"),
+
+		flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"),
+		flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"),
+		flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"),
+		flagSet.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"),
+	)
+
+	createGroup(flagSet, "rate-limit", "Rate-Limit",
+		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
+		flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
+		flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "maximum number of templates to be executed in parallel"),
+	)
+
+	createGroup(flagSet, "optimization", "Optimizations",
+		flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"),
+		flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"),
+
+		flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"),
+		flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"),
+
+		flagSet.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "stop processing HTTP requests after the first match (may break template/workflow logic)"),
+	)
+
+	createGroup(flagSet, "headless", "Headless",
+		flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support"),
+		flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"),
+		flagSet.BoolVar(&options.ShowBrowser, "show-browser", false, "show the browser on the screen when running templates with headless mode"),
+	)
+
+	createGroup(flagSet, "debug", "Debug",
+		flagSet.BoolVar(&options.Debug, "debug", false, "show all requests and responses"),
+		flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"),
+		flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"),
+
+		/* TODO why the separation? http://proxy:port vs socks5://proxy:port etc
+		   TODO should auto-set the HTTP_PROXY variable for the process? */
+		flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"),
+		flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"),
+
+		flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"),
+
+		flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"),
+		flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"),
+	)
+
+	createGroup(flagSet, "update", "Update",
+		flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei to the latest released version"),
+		flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update the community templates to latest released version"),
+		flagSet.BoolVarP(&options.NoUpdateTemplates, "no-update-templates", "nut", false, "Do not check for nuclei-templates updates"),
+		flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default nuclei-templates directory"),
+	)
+
+	createGroup(flagSet, "stats", "Statistics",
+		flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"),
+		flagSet.BoolVar(&options.StatsJSON, "stats-json", false, "write statistics data to an output file in JSONL(ines) format"),
+		flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"),
+
+		flagSet.BoolVar(&options.Metrics, "metrics", false, "expose nuclei metrics on a port"),
+		flagSet.IntVar(&options.MetricsPort, "metrics-port", 9092, "port to expose nuclei metrics on"),
+	)
+
+	_ = flagSet.Parse()
+
+	if cfgFile != "" {
+		if err := flagSet.MergeConfigFile(cfgFile); err != nil {
+			gologger.Fatal().Msgf("Could not read config: %s\n", err)
+		}
+	}
+}
+
+func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) {
+	flagSet.SetGroup(groupName, description)
+	for _, currentFlag := range flags {
+		currentFlag.Group(groupName)
 	}
-
-	runner.RunEnumeration()
-	runner.Close()
 }

v2/go.mod

@@ -1,22 +1,63 @@
 module github.com/projectdiscovery/nuclei/v2
 
-go 1.14
+go 1.15
 
 require (
 	github.com/Knetic/govaluate v3.0.0+incompatible
-	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
+	github.com/andygrunwald/go-jira v1.13.0
+	github.com/apex/log v1.9.0
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/d5/tengo v1.24.8
-	github.com/d5/tengo/v2 v2.6.0
+	github.com/c4milo/unpackit v0.1.0 // indirect
+	github.com/corpix/uarand v0.1.1
+	github.com/fatih/structs v1.1.0 // indirect
+	github.com/go-rod/rod v0.91.1
+	github.com/golang/protobuf v1.4.3 // indirect
 	github.com/google/go-github v17.0.0+incompatible
-	github.com/google/go-github/v32 v32.0.0
+	github.com/gosuri/uilive v0.0.4 // indirect
+	github.com/gosuri/uiprogress v0.0.1 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
 	github.com/json-iterator/go v1.1.10
-	github.com/karrick/godirwalk v1.15.6
-	github.com/miekg/dns v1.1.29
+	github.com/julienschmidt/httprouter v1.3.0
+	github.com/karlseguin/ccache v2.0.3+incompatible
+	github.com/karrick/godirwalk v1.16.1
+	github.com/logrusorgru/aurora v2.0.3+incompatible
+	github.com/mattn/go-runewidth v0.0.10 // indirect
+	github.com/miekg/dns v1.1.38
+	github.com/olekukonko/tablewriter v0.0.5
+	github.com/owenrumney/go-sarif v1.0.4
 	github.com/pkg/errors v0.9.1
-	github.com/projectdiscovery/gologger v1.0.0
-	github.com/projectdiscovery/retryabledns v1.0.4
-	github.com/projectdiscovery/retryablehttp-go v1.0.1
-	golang.org/x/net v0.0.0-20200528225125-3c3fba18258b
-	gopkg.in/yaml.v2 v2.3.0
+	github.com/projectdiscovery/clistats v0.0.8
+	github.com/projectdiscovery/fastdialer v0.0.8
+	github.com/projectdiscovery/goflags v0.0.6
+	github.com/projectdiscovery/gologger v1.1.4
+	github.com/projectdiscovery/hmap v0.0.1
+	github.com/projectdiscovery/interactsh v0.0.3
+	github.com/projectdiscovery/rawhttp v0.0.7
+	github.com/projectdiscovery/retryabledns v1.0.10
+	github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727
+	github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d
+	github.com/remeh/sizedwaitgroup v1.0.0
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/rs/xid v1.2.1
+	github.com/segmentio/ksuid v1.0.3
+	github.com/shirou/gopsutil/v3 v3.21.5
+	github.com/spaolacci/murmur3 v1.1.0
+	github.com/spf13/cast v1.3.1
+	github.com/stretchr/testify v1.7.0
+	github.com/syndtr/goleveldb v1.0.0
+	github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible
+	github.com/trivago/tgo v1.0.7 // indirect
+	github.com/valyala/fasttemplate v1.2.1
+	github.com/xanzy/go-gitlab v0.44.0
+	go.uber.org/atomic v1.7.0
+	go.uber.org/multierr v1.6.0
+	go.uber.org/ratelimit v0.1.0
+	golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect
+	golang.org/x/net v0.0.0-20210521195947-fe42d452be8f
+	golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
+	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	gopkg.in/yaml.v2 v2.4.0
 )

v2/go.sum

@@ -1,69 +1,663 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
+github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
+github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE=
+github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I=
+github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
+github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA=
+github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
+github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
+github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
+github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
+github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
-github.com/d5/tengo v1.24.8/go.mod h1:VhLq8Q2QFhCIJO3NhvM934qOThykMqJi9y9Siqd1ocQ=
-github.com/d5/tengo/v2 v2.6.0 h1:D0cJtpiBzaLJ/Smv6nnUc/LIfO46oKwDx85NZtIRNRI=
-github.com/d5/tengo/v2 v2.6.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
+github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
+github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og=
+github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I=
+github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r7ZRSxEo=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
+github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4=
+github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
+github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
+github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
+github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
+github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
+github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
+github.com/eggsampler/acme/v3 v3.2.1 h1:Lfsrg3M2zt00QRnizOFzdpSfsS9oDvPsGrodXS/w1KI=
+github.com/eggsampler/acme/v3 v3.2.1/go.mod h1:/qh0rKC/Dh7Jj+p4So7DbWmFNzC4dpcpK53r226Fhuo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
+github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-rod/rod v0.91.1 h1:7xIlC/bXCXosZqZUl2x6GVB8tv4yMQ4W/ZVdGVa1qYI=
+github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
+github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
-github.com/google/go-github/v32 v32.0.0 h1:q74KVb22spUq0U5HqZ9VCYqQz8YRuOtL/39ZnfwO+NM=
-github.com/google/go-github/v32 v32.0.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
+github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY=
+github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI=
+github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw=
+github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
+github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs=
+github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
+github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA=
-github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU=
+github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w=
+github.com/karlseguin/ccache/v2 v2.0.7 h1:y5Pfi4eiyYCOD6LS/Kj+o6Nb4M5Ngpw9qFQs+v44ZYM=
+github.com/karlseguin/ccache/v2 v2.0.7/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ=
+github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA=
+github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
+github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
+github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
+github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
-github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
+github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
+github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
+github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw=
+github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/owenrumney/go-sarif v1.0.4 h1:0LFC5eHP6amc/9ajM1jDiE52UfXFcl/oozay+X3KgV4=
+github.com/owenrumney/go-sarif v1.0.4/go.mod h1:DXUGbHwQcCMvqcvZbxh8l/7diHsJVztOKZgmPt88RNI=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s=
-github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
-github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60=
-github.com/projectdiscovery/retryabledns v1.0.4/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
-github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
+github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20=
+github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
+github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0=
+github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4=
+github.com/projectdiscovery/goflags v0.0.6 h1:4ErduTfSC55cRR3TmUg+TQirBlCuBdBadrluAsy1pew=
+github.com/projectdiscovery/goflags v0.0.6/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
+github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E=
+github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
+github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
+github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
+github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
+github.com/projectdiscovery/interactsh v0.0.3 h1:PUkWk+NzSyd5glMqfORmuqizhsd7c3WdTYBOto/MQIU=
+github.com/projectdiscovery/interactsh v0.0.3/go.mod h1:dWnKO14d2FLP3kLhI9DecEsiAC/aZiJoUBGFjGhDskY=
+github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg=
+github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
+github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
+github.com/projectdiscovery/retryabledns v1.0.10 h1:xJZ2aKoqrNg/OZEw1+4+QIOH40V/WkZDYY1ZZc+uphE=
+github.com/projectdiscovery/retryabledns v1.0.10/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
+github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727 h1:CJHP3CLCc/eqdXQEvZy8KiiqtAk9kEsd1URtPyPAQ1s=
+github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
+github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nlOAex7twmrEqD5i6WLnugF9uO3DQ6jDEKN9gevrTAk=
+github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
+github.com/prologic/smtpd v0.0.0-20210126001904-0893ad18168e h1:ZT3wZ92sp/EHEE/HcFCWCsYS3ROLjHb6EqSX8qYrgXw=
+github.com/prologic/smtpd v0.0.0-20210126001904-0893ad18168e/go.mod h1:GkLsdH1RZj6RDKeI9A05NGZYmEZQ/PbQcZPnZoSZuYI=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
+github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY=
+github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shirou/gopsutil/v3 v3.21.5 h1:YUBf0w/KPLk7w1803AYBnH7BmA+1Z/Q5MEZxpREUaB4=
+github.com/shirou/gopsutil/v3 v3.21.5/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw=
+github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
+github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
+github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
+github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
+github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc=
+github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
+github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
+github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
+github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible h1:guTq1YxwB8XSILkI9q4IrOmrCOS6Hc1L3hmOhi4Swcs=
+github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible/go.mod h1:waFwwyiAhGey2e+dNoYQ/iLhIcFqhCW7zL/+vDU1WLo=
+github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M=
+github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek=
+github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc=
+github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8=
+github.com/trivago/tgo v1.0.1/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
+github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM=
+github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
+github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
+github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
+github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
+github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
+github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
+github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
+github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
+github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
+github.com/xanzy/go-gitlab v0.44.0 h1:cEiGhqu7EpFGuei2a2etAwB+x6403E5CvpLn35y+GPs=
+github.com/xanzy/go-gitlab v0.44.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
+github.com/ysmood/goob v0.3.0 h1:XZ51cZJ4W3WCoCiUktixzMIQF86W7G5VFL4QQ/Q2uS0=
+github.com/ysmood/goob v0.3.0/go.mod h1:S3lq113Y91y1UBf1wj1pFOxeahvfKkCk6mTWTWbDdWs=
+github.com/ysmood/got v0.9.3 h1:qx51X49jL/WAiqZzPTkPZ0zp5pTmrWJa4zYFTYo0gHI=
+github.com/ysmood/got v0.9.3/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
+github.com/ysmood/gotrace v0.2.0 h1:IkTC6rJREwXSaG8yWK+NFwIJGIsxA1DjC6/gxYyQttE=
+github.com/ysmood/gotrace v0.2.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
+github.com/ysmood/gson v0.6.3 h1:4cU+5oOdsyundXHy00t99H0rLXLthuseD3x6W+xmCiU=
+github.com/ysmood/gson v0.6.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
+github.com/ysmood/leakless v0.6.12 h1:XxtRYl97bJklfv4BZVdyGnd/y42p6w8lu1hUzfCkT4M=
+github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/zclconf/go-cty v1.8.2 h1:u+xZfBKgpycDnTNjPhGiTEYZS5qS/Sb5MqSfm7vzcjg=
+github.com/zclconf/go-cty v1.8.2/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
+go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df h1:y7QZzfUiTwWam+xBn29Ulb8CBwVN5UdzmMDavl9Whlw=
+golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200528225125-3c3fba18258b h1:IYiJPiJfzktmDAO1HQiwjMjwjlYKHAL7KzeD544RJPs=
-golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U=
+golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 h1:5vD4XjIc0X5+kHZjx4UecYdjA6mJo+XXNoaW0EjU5Os=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE=
+golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/corvus-ch/zbase32.v1 v1.0.0 h1:K4u1NprbDNvKPczKfHLbwdOWHTZ0zfv2ow71H1nRnFU=
+gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DSo0nK9U+c0Y=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

v2/internal/colorizer/colorizer.go

@@ -0,0 +1,24 @@
+package colorizer
+
+import "github.com/logrusorgru/aurora"
+
+// Colorizer returns a colorized severity printer
+type Colorizer struct {
+	Data map[string]string
+}
+
+const (
+	fgOrange uint8 = 208
+)
+
+// New returns a new severity based colorizer
+func New(colorizer aurora.Aurora) *Colorizer {
+	severityMap := map[string]string{
+		"info":     colorizer.Blue("info").String(),
+		"low":      colorizer.Green("low").String(),
+		"medium":   colorizer.Yellow("medium").String(),
+		"high":     colorizer.Index(fgOrange, "high").String(),
+		"critical": colorizer.Red("critical").String(),
+	}
+	return &Colorizer{Data: severityMap}
+}

v2/internal/runner/banner.go

@@ -1,23 +1,25 @@
 package runner
 
-import "github.com/projectdiscovery/gologger"
+import (
+	"fmt"
 
-const banner = `
-                       __     _ 
-     ____  __  _______/ /__  (_)
-    / __ \/ / / / ___/ / _ \/ / 
-   / / / / /_/ / /__/ /  __/ /  
-  /_/ /_/\__,_/\___/_/\___/_/   v2									  
-`
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+)
 
-// Version is the current version of nuclei
-const Version = `2.0.2`
+var banner = fmt.Sprintf(`
+                     __     _
+   ____  __  _______/ /__  (_)
+  / __ \/ / / / ___/ / _ \/ /
+ / / / / /_/ / /__/ /  __/ /
+/_/ /_/\__,_/\___/_/\___/_/   %s
+`, config.Version)
 
 // showBanner is used to show the banner to the user
 func showBanner() {
-	gologger.Printf("%s\n", banner)
-	gologger.Printf("\t\tprojectdiscovery.io\n\n")
+	gologger.Print().Msgf("%s\n", banner)
+	gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n")
 
-	gologger.Labelf("Use with caution. You are responsible for your actions\n")
-	gologger.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n")
+	gologger.Warning().Msgf("Use with caution. You are responsible for your actions\n")
+	gologger.Warning().Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
 }

v2/internal/runner/config.go

@@ -0,0 +1 @@
+package runner

v2/internal/runner/options.go

@@ -1,93 +1,151 @@
 package runner
 
 import (
-	"flag"
+	"bufio"
+	"errors"
+	"net/url"
 	"os"
+	"strings"
 
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
+	"github.com/projectdiscovery/gologger/formatter"
+	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
-// Options contains the configuration options for tuning
-// the template requesting process.
-type Options struct {
-	Debug              bool                   // Debug mode allows debugging request/responses for the engine
-	Templates          string                 // Signature specifies the template/templates to use
-	Target             string                 // Target is a single URL/Domain to scan usng a template
-	Targets            string                 // Targets specifies the targets to scan using templates.
-	Threads            int                    // Thread controls the number of concurrent requests to make.
-	Timeout            int                    // Timeout is the seconds to wait for a response from the server.
-	Retries            int                    // Retries is the number of times to retry the request
-	Output             string                 // Output is the file to write found subdomains to.
-	ProxyURL           string                 // ProxyURL is the URL for the proxy server
-	ProxySocksURL      string                 // ProxySocksURL is the URL for the proxy socks server
-	Silent             bool                   // Silent suppresses any extra text and only writes found URLs on screen.
-	Version            bool                   // Version specifies if we should just show version and exit
-	Verbose            bool                   // Verbose flag indicates whether to show verbose output or not
-	NoColor            bool                   // No-Color disables the colored output.
-	CustomHeaders      requests.CustomHeaders // Custom global headers
-	UpdateTemplates    bool                   // UpdateTemplates updates the templates installed at startup
-	TemplatesDirectory string                 // TemplatesDirectory is the directory to use for storing templates
-	JSON               bool                   // JSON writes json output to files
-
-	Stdin bool // Stdin specifies whether stdin input was given to the process
-}
-
 // ParseOptions parses the command line flags provided by a user
-func ParseOptions() *Options {
-	options := &Options{}
-
-	flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template")
-	flag.StringVar(&options.Templates, "t", "", "Template input file/files to run on host")
-	flag.StringVar(&options.Targets, "l", "", "List of URLs to run templates on")
-	flag.StringVar(&options.Output, "o", "", "File to write output to (optional)")
-	flag.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server")
-	flag.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the proxy socks server")
-	flag.BoolVar(&options.Silent, "silent", false, "Show only results in output")
-	flag.BoolVar(&options.Version, "version", false, "Show version of nuclei")
-	flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output")
-	flag.BoolVar(&options.NoColor, "nC", false, "Don't Use colors in output")
-	flag.IntVar(&options.Threads, "c", 10, "Number of concurrent requests to make")
-	flag.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout")
-	flag.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
-	flag.Var(&options.CustomHeaders, "H", "Custom Header.")
-	flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses")
-	flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)")
-	flag.StringVar(&options.TemplatesDirectory, "update-directory", "", "Directory to use for storing nuclei-templates")
-	flag.BoolVar(&options.JSON, "json", false, "Write json output to files")
-
-	flag.Parse()
-
+func ParseOptions(options *types.Options) {
 	// Check if stdin pipe was given
 	options.Stdin = hasStdin()
 
 	// Read the inputs and configure the logging
-	options.configureOutput()
+	configureOutput(options)
 
 	// Show the user the banner
 	showBanner()
 
 	if options.Version {
-		gologger.Infof("Current Version: %s\n", Version)
+		gologger.Info().Msgf("Current Version: %s\n", config.Version)
+		os.Exit(0)
+	}
+	if options.TemplatesVersion {
+		configuration, err := config.ReadConfiguration()
+		if err != nil {
+			gologger.Fatal().Msgf("Could not read template configuration: %s\n", err)
+		}
+		gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", configuration.CurrentVersion, configuration.TemplatesDirectory)
 		os.Exit(0)
 	}
 
 	// Validate the options passed by the user and if any
 	// invalid options have been used, exit.
-	err := options.validateOptions()
+	if err := validateOptions(options); err != nil {
+		gologger.Fatal().Msgf("Program exiting: %s\n", err)
+	}
+
+	// Auto adjust rate limits when using headless mode if the user
+	// hasn't specified any custom limits.
+	if options.Headless && options.BulkSize == 25 && options.TemplateThreads == 10 {
+		options.BulkSize = 2
+		options.TemplateThreads = 2
+	}
+
+	// Load the resolvers if user asked for them
+	loadResolvers(options)
+
+	err := protocolinit.Init(options)
 	if err != nil {
-		gologger.Fatalf("Program exiting: %s\n", err)
+		gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err)
 	}
-	return options
 }
 
+// hasStdin returns true if we have stdin input
 func hasStdin() bool {
-	fi, err := os.Stdin.Stat()
+	stat, err := os.Stdin.Stat()
 	if err != nil {
 		return false
 	}
-	if fi.Mode()&os.ModeNamedPipe == 0 {
-		return false
+
+	isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0
+	isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0
+
+	return isPipedFromChrDev || isPipedFromFIFO
+}
+
+// validateOptions validates the configuration options passed
+func validateOptions(options *types.Options) error {
+	// Both verbose and silent flags were used
+	if options.Verbose && options.Silent {
+		return errors.New("both verbose and silent mode specified")
 	}
-	return true
+
+	// Validate proxy options if provided
+	err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)")
+	if err != nil {
+		return err
+	}
+
+	err = validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)")
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func validateProxyURL(proxyURL, message string) error {
+	if proxyURL != "" && !isValidURL(proxyURL) {
+		return errors.New(message)
+	}
+
+	return nil
+}
+
+func isValidURL(urlString string) bool {
+	_, err := url.Parse(urlString)
+	return err == nil
+}
+
+// configureOutput configures the output on the screen
+func configureOutput(options *types.Options) {
+	// If the user desires verbose output, show verbose output
+	if options.Verbose {
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
+	}
+	if options.Debug {
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
+	}
+	if options.NoColor {
+		gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true))
+	}
+	if options.Silent {
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
+	}
+}
+
+// loadResolvers loads resolvers from both user provided flag and file
+func loadResolvers(options *types.Options) {
+	if options.ResolversFile == "" {
+		return
+	}
+
+	file, err := os.Open(options.ResolversFile)
+	if err != nil {
+		gologger.Fatal().Msgf("Could not open resolvers file: %s\n", err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		part := scanner.Text()
+		if part == "" {
+			continue
+		}
+		if strings.Contains(part, ":") {
+			options.InternalResolversList = append(options.InternalResolversList, part)
+		} else {
+			options.InternalResolversList = append(options.InternalResolversList, part+":53")
+		}
+	}
 }

v2/internal/runner/processor.go

@@ -0,0 +1,50 @@
+package runner
+
+import (
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/remeh/sizedwaitgroup"
+	"go.uber.org/atomic"
+)
+
+// processTemplateWithList process a template on the URL list
+func (r *Runner) processTemplateWithList(template *templates.Template) bool {
+	results := &atomic.Bool{}
+	wg := sizedwaitgroup.New(r.options.BulkSize)
+	r.hostMap.Scan(func(k, _ []byte) error {
+		URL := string(k)
+
+		wg.Add()
+		go func(URL string) {
+			defer wg.Done()
+
+			match, err := template.Executer.Execute(URL)
+			if err != nil {
+				gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err)
+			}
+			results.CAS(false, match)
+		}(URL)
+		return nil
+	})
+	wg.Wait()
+	return results.Load()
+}
+
+// processTemplateWithList process a template on the URL list
+func (r *Runner) processWorkflowWithList(template *templates.Template) bool {
+	results := &atomic.Bool{}
+	wg := sizedwaitgroup.New(r.options.BulkSize)
+
+	r.hostMap.Scan(func(k, _ []byte) error {
+		URL := string(k)
+		wg.Add()
+		go func(URL string) {
+			defer wg.Done()
+			match := template.CompiledWorkflow.RunWorkflow(URL)
+			results.CAS(false, match)
+		}(URL)
+		return nil
+	})
+	wg.Wait()
+	return results.Load()
+}

v2/internal/runner/runner.go

@@ -2,472 +2,510 @@ package runner
 
 import (
 	"bufio"
-	"context"
-	"errors"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"os"
+	"path"
 	"strings"
-	"sync"
+	"time"
 
-	"github.com/d5/tengo/v2"
-	"github.com/karrick/godirwalk"
+	"github.com/logrusorgru/aurora"
+	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/executor"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
+	"github.com/projectdiscovery/hmap/store/hybrid"
+	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"github.com/remeh/sizedwaitgroup"
+	"github.com/rs/xid"
+	"go.uber.org/atomic"
+	"go.uber.org/ratelimit"
+	"gopkg.in/yaml.v2"
 )
 
 // Runner is a client for running the enumeration process.
 type Runner struct {
-	// output is the output file to write if any
-	output      *os.File
-	outputMutex *sync.Mutex
-
-	tempFile        string
-	templatesConfig *nucleiConfig
-	// options contains configuration options for runner
-	options *Options
+	hostMap         *hybrid.HybridMap
+	output          output.Writer
+	interactsh      *interactsh.Client
+	inputCount      int64
+	templatesConfig *config.Config
+	options         *types.Options
+	projectFile     *projectfile.ProjectFile
+	catalog         *catalog.Catalog
+	progress        progress.Progress
+	colorizer       aurora.Aurora
+	issuesClient    *reporting.Client
+	severityColors  *colorizer.Colorizer
+	browser         *engine.Browser
+	ratelimiter     ratelimit.Limiter
 }
 
 // New creates a new client for running enumeration process.
-func New(options *Options) (*Runner, error) {
+func New(options *types.Options) (*Runner, error) {
 	runner := &Runner{
-		outputMutex: &sync.Mutex{},
-		options:     options,
+		options: options,
+	}
+	if options.UpdateNuclei {
+		if err := updateNucleiVersionToLatest(runner.options.Verbose); err != nil {
+			return nil, err
+		}
+		return nil, nil
+	}
+	if err := runner.updateTemplates(); err != nil {
+		gologger.Warning().Msgf("Could not update templates: %s\n", err)
+	}
+	if options.Headless {
+		browser, err := engine.New(options)
+		if err != nil {
+			return nil, err
+		}
+		runner.browser = browser
 	}
 
-	if err := runner.updateTemplates(); err != nil {
-		gologger.Warningf("Could not update templates: %s\n", err)
+	runner.catalog = catalog.New(runner.options.TemplatesDirectory)
+	var reportingOptions *reporting.Options
+	if options.ReportingConfig != "" {
+		file, err := os.Open(options.ReportingConfig)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not open reporting config file")
+		}
+
+		reportingOptions = &reporting.Options{}
+		if parseErr := yaml.NewDecoder(file).Decode(reportingOptions); parseErr != nil {
+			file.Close()
+			return nil, errors.Wrap(parseErr, "could not parse reporting config file")
+		}
+		file.Close()
 	}
-	if (options.Templates == "" || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
+	if options.DiskExportDirectory != "" {
+		if reportingOptions != nil {
+			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
+		} else {
+			reportingOptions = &reporting.Options{}
+			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
+		}
+	}
+	if options.SarifExport != "" {
+		if reportingOptions != nil {
+			reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}
+		} else {
+			reportingOptions = &reporting.Options{}
+			reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}
+		}
+	}
+	if reportingOptions != nil {
+		client, err := reporting.New(reportingOptions, options.ReportingDB)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create issue reporting client")
+		}
+		runner.issuesClient = client
+	}
+
+	// output coloring
+	useColor := !options.NoColor
+	runner.colorizer = aurora.NewAurora(useColor)
+	runner.severityColors = colorizer.New(runner.colorizer)
+
+	if options.TemplateList {
+		runner.listAvailableTemplates()
 		os.Exit(0)
 	}
 
-	// If we have stdin, write it to a new file
-	if options.Stdin {
-		tempInput, err := ioutil.TempFile("", "stdin-input-*")
-		if err != nil {
-			return nil, err
-		}
-		if _, err := io.Copy(tempInput, os.Stdin); err != nil {
-			return nil, err
-		}
-		runner.tempFile = tempInput.Name()
-		tempInput.Close()
+	if (len(options.Templates) == 0 || !options.NewTemplates || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
+		os.Exit(0)
 	}
-	// If we have single target, write it to a new file
+	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create temporary input file")
+	}
+	runner.hostMap = hm
+
+	runner.inputCount = 0
+	dupeCount := 0
+
+	// Handle single target
 	if options.Target != "" {
-		tempInput, err := ioutil.TempFile("", "stdin-input-*")
-		if err != nil {
-			return nil, err
+		runner.inputCount++
+		// nolint:errcheck // ignoring error
+		runner.hostMap.Set(options.Target, nil)
+	}
+
+	// Handle stdin
+	if options.Stdin {
+		scanner := bufio.NewScanner(os.Stdin)
+		for scanner.Scan() {
+			url := strings.TrimSpace(scanner.Text())
+			if url == "" {
+				continue
+			}
+			if _, ok := runner.hostMap.Get(url); ok {
+				dupeCount++
+				continue
+			}
+			runner.inputCount++
+			// nolint:errcheck // ignoring error
+			runner.hostMap.Set(url, nil)
 		}
-		tempInput.WriteString(options.Target)
-		runner.tempFile = tempInput.Name()
-		tempInput.Close()
+	}
+
+	// Handle taget file
+	if options.Targets != "" {
+		input, inputErr := os.Open(options.Targets)
+		if inputErr != nil {
+			return nil, errors.Wrap(inputErr, "could not open targets file")
+		}
+		scanner := bufio.NewScanner(input)
+		for scanner.Scan() {
+			url := strings.TrimSpace(scanner.Text())
+			if url == "" {
+				continue
+			}
+			if _, ok := runner.hostMap.Get(url); ok {
+				dupeCount++
+				continue
+			}
+			runner.inputCount++
+			// nolint:errcheck // ignoring error
+			runner.hostMap.Set(url, nil)
+		}
+		input.Close()
+	}
+
+	if dupeCount > 0 {
+		gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", dupeCount)
 	}
 
 	// Create the output file if asked
-	if options.Output != "" {
-		output, err := os.Create(options.Output)
-		if err != nil {
-			gologger.Fatalf("Could not create output file '%s': %s\n", options.Output, err)
+	outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.JSON, options.Output, options.TraceLogFile)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create output file")
+	}
+	runner.output = outputWriter
+
+	if options.JSON && options.EnableProgressBar {
+		options.StatsJSON = true
+	}
+	if options.StatsJSON {
+		options.EnableProgressBar = true
+	}
+	// Creates the progress tracking object
+	var progressErr error
+	runner.progress, progressErr = progress.NewStatsTicker(options.StatsInterval, options.EnableProgressBar, options.StatsJSON, options.Metrics, options.MetricsPort)
+	if progressErr != nil {
+		return nil, progressErr
+	}
+
+	// create project file if requested or load existing one
+	if options.Project {
+		var projectFileErr error
+		runner.projectFile, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""})
+		if projectFileErr != nil {
+			return nil, projectFileErr
 		}
-		runner.output = output
+	}
+
+	if !options.NoInteractsh {
+		interactshClient, err := interactsh.New(&interactsh.Options{
+			ServerURL:      options.InteractshURL,
+			CacheSize:      int64(options.InteractionsCacheSize),
+			Eviction:       time.Duration(options.InteractionsEviction) * time.Second,
+			ColldownPeriod: time.Duration(options.InteractionsColldownPeriod) * time.Second,
+			PollDuration:   time.Duration(options.InteractionsPollDuration) * time.Second,
+			Output:         runner.output,
+			IssuesClient:   runner.issuesClient,
+			Progress:       runner.progress,
+		})
+		if err != nil {
+			gologger.Error().Msgf("Could not create interactsh client: %s", err)
+		} else {
+			runner.interactsh = interactshClient
+		}
+	}
+
+	if options.RateLimit > 0 {
+		runner.ratelimiter = ratelimit.New(options.RateLimit)
+	} else {
+		runner.ratelimiter = ratelimit.NewUnlimited()
 	}
 	return runner, nil
 }
 
 // Close releases all the resources and cleans up
 func (r *Runner) Close() {
-	r.output.Close()
-	os.Remove(r.tempFile)
+	if r.output != nil {
+		r.output.Close()
+	}
+	r.hostMap.Close()
+	if r.projectFile != nil {
+		r.projectFile.Close()
+	}
+	protocolinit.Close()
 }
 
 // RunEnumeration sets up the input layer for giving input nuclei.
 // binary and runs the actual enumeration
-func (r *Runner) RunEnumeration() {
-	var err error
+func (r *Runner) RunEnumeration() error {
+	defer r.Close()
 
-	// Check if the template is an absolute path or relative path.
-	// If the path is absolute, use it. Otherwise,
-	if r.isRelative(r.options.Templates) {
-		newPath, err := r.resolvePath(r.options.Templates)
+	// If user asked for new templates to be executed, collect the list from template directory.
+	if r.options.NewTemplates {
+		templatesLoaded, err := r.readNewTemplatesFile()
 		if err != nil {
-			gologger.Errorf("Could not find template file '%s': %s\n", r.options.Templates, err)
-			return
+			return errors.Wrap(err, "could not get newly added templates")
 		}
-		r.options.Templates = newPath
+		r.options.Templates = append(r.options.Templates, templatesLoaded...)
 	}
+	ignoreFile := config.ReadIgnoreFile()
+	r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
+	r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
 
-	// Single yaml provided
-	if strings.HasSuffix(r.options.Templates, ".yaml") {
-		t, err := r.parse(r.options.Templates)
-		switch t.(type) {
-		case *templates.Template:
-			var results bool
-			template := t.(*templates.Template)
-			// process http requests
-			for _, request := range template.RequestsHTTP {
-				results = r.processTemplateRequest(template, request)
-			}
-			// process dns requests
-			for _, request := range template.RequestsDNS {
-				dnsResults := r.processTemplateRequest(template, request)
-				if !results {
-					results = dnsResults
-				}
-			}
-
-			if !results {
-				if r.output != nil {
-					outputFile := r.output.Name()
-					r.output.Close()
-					os.Remove(outputFile)
-				}
-				gologger.Infof("No results found for the template. Happy hacking!")
-			}
-		case *workflows.Workflow:
-			workflow := t.(*workflows.Workflow)
-			r.ProcessWorkflowWithList(workflow)
-		default:
-			gologger.Errorf("Could not parse file '%s': %s\n", r.options.Templates, err)
-		}
-		return
+	executerOpts := protocols.ExecuterOptions{
+		Output:       r.output,
+		Options:      r.options,
+		Progress:     r.progress,
+		Catalog:      r.catalog,
+		IssuesClient: r.issuesClient,
+		RateLimiter:  r.ratelimiter,
+		Interactsh:   r.interactsh,
+		ProjectFile:  r.projectFile,
+		Browser:      r.browser,
 	}
-
-	// If the template passed is a directory
-	matches := []string{}
-
-	// Recursively walk down the Templates directory and run all the template file checks
-	err = godirwalk.Walk(r.options.Templates, &godirwalk.Options{
-		Callback: func(path string, d *godirwalk.Dirent) error {
-			if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
-				matches = append(matches, path)
-			}
-			return nil
-		},
-		ErrorCallback: func(path string, err error) godirwalk.ErrorAction {
-			return godirwalk.SkipNode
-		},
-		Unsorted: true,
-	})
+	loaderConfig := loader.Config{
+		Templates:          r.options.Templates,
+		Workflows:          r.options.Workflows,
+		ExcludeTemplates:   r.options.ExcludedTemplates,
+		Tags:               r.options.Tags,
+		ExcludeTags:        r.options.ExcludeTags,
+		IncludeTemplates:   r.options.IncludeTemplates,
+		Authors:            r.options.Author,
+		Severities:         r.options.Severity,
+		IncludeTags:        r.options.IncludeTags,
+		TemplatesDirectory: r.options.TemplatesDirectory,
+		Catalog:            r.catalog,
+		ExecutorOptions:    executerOpts,
+	}
+	store, err := loader.New(&loaderConfig)
 	if err != nil {
-		gologger.Fatalf("Could not find templates in directory '%s': %s\n", r.options.Templates, err)
+		return errors.Wrap(err, "could not load templates from config")
 	}
-	// 0 matches means no templates were found in directory
-	if len(matches) == 0 {
-		gologger.Fatalf("Error, no templates found in directory: '%s'\n", r.options.Templates)
-	}
-
-	var results bool
-	for _, match := range matches {
-		t, err := r.parse(match)
-		switch t.(type) {
-		case *templates.Template:
-			template := t.(*templates.Template)
-			for _, request := range template.RequestsDNS {
-				dnsResults := r.processTemplateRequest(template, request)
-				if dnsResults {
-					results = dnsResults
-				}
-			}
-			for _, request := range template.RequestsHTTP {
-				httpResults := r.processTemplateRequest(template, request)
-				if httpResults {
-					results = httpResults
-				}
-			}
-		case *workflows.Workflow:
-			workflow := t.(*workflows.Workflow)
-			r.ProcessWorkflowWithList(workflow)
-		default:
-			gologger.Errorf("Could not parse file '%s': %s\n", r.options.Templates, err)
+	if r.options.Validate {
+		if !store.ValidateTemplates(r.options.Templates, r.options.Workflows) {
+			return errors.New("an error occurred during templates validation")
 		}
+		gologger.Info().Msgf("All templates validated successfully\n")
+		return nil // exit
 	}
-	if !results {
-		if r.output != nil {
-			outputFile := r.output.Name()
-			r.output.Close()
-			os.Remove(outputFile)
+	store.Load()
+
+	builder := &strings.Builder{}
+	if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" {
+		builder.WriteString(" (")
+
+		if strings.Contains(config.Version, "-dev") {
+			builder.WriteString(r.colorizer.Blue("development").String())
+		} else if config.Version == r.templatesConfig.NucleiLatestVersion {
+			builder.WriteString(r.colorizer.Green("latest").String())
+		} else {
+			builder.WriteString(r.colorizer.Red("outdated").String())
 		}
-		gologger.Infof("No results found for the template. Happy hacking!")
+		builder.WriteString(")")
 	}
-	return
-}
+	messageStr := builder.String()
+	builder.Reset()
 
-// processTemplate processes a template and runs the enumeration on all the targets
-func (r *Runner) processTemplateRequest(template *templates.Template, request interface{}) bool {
-	var file *os.File
-	var err error
+	gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr)
 
-	// Handle a list of hosts as argument
-	if r.options.Targets != "" {
-		file, err = os.Open(r.options.Targets)
-	} else if r.options.Stdin || r.options.Target != "" {
-		file, err = os.Open(r.tempFile)
+	if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" {
+		builder.WriteString(" (")
+
+		if r.templatesConfig.CurrentVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
+			builder.WriteString(r.colorizer.Green("latest").String())
+		} else {
+			builder.WriteString(r.colorizer.Red("outdated").String())
+		}
+		builder.WriteString(")")
 	}
-	if err != nil {
-		gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err)
-	}
-	results := r.processTemplateWithList(template, request, file)
-	file.Close()
-	return results
-}
+	messageStr = builder.String()
+	builder.Reset()
 
-// processDomain processes the list with a template
-func (r *Runner) processTemplateWithList(template *templates.Template, request interface{}, reader io.Reader) bool {
-	// Display the message for the template
-	message := fmt.Sprintf("[%s] Loaded template %s (@%s)", template.ID, template.Info.Name, template.Info.Author)
-	if template.Info.Severity != "" {
-		message += " [" + template.Info.Severity + "]"
-	}
-	gologger.Infof("%s\n", message)
+	gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.CurrentVersion, messageStr)
 
-	var writer *bufio.Writer
-	if r.output != nil {
-		writer = bufio.NewWriter(r.output)
-		defer writer.Flush()
+	if r.interactsh != nil {
+		gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL)
+	}
+	if len(store.Templates()) > 0 {
+		gologger.Info().Msgf("Templates loaded: %d (New: %d)", len(store.Templates()), r.countNewTemplates())
+	}
+	if len(store.Workflows()) > 0 {
+		gologger.Info().Msgf("Workflows loaded: %d", len(store.Workflows()))
 	}
 
-	var httpExecutor *executor.HTTPExecutor
-	var dnsExecutor *executor.DNSExecutor
-	var err error
+	// pre-parse all the templates, apply filters
+	finalTemplates := []*templates.Template{}
 
-	// Create an executor based on the request type.
-	switch value := request.(type) {
-	case *requests.DNSRequest:
-		dnsExecutor = executor.NewDNSExecutor(&executor.DNSOptions{
-			Debug:      r.options.Debug,
-			Template:   template,
-			DNSRequest: value,
-			Writer:     writer,
-			JSON:       r.options.JSON,
-		})
-	case *requests.HTTPRequest:
-		httpExecutor, err = executor.NewHTTPExecutor(&executor.HTTPOptions{
-			Debug:         r.options.Debug,
-			Template:      template,
-			HTTPRequest:   value,
-			Writer:        writer,
-			Timeout:       r.options.Timeout,
-			Retries:       r.options.Retries,
-			ProxyURL:      r.options.ProxyURL,
-			ProxySocksURL: r.options.ProxySocksURL,
-			CustomHeaders: r.options.CustomHeaders,
-			JSON:          r.options.JSON,
-		})
-	}
-	if err != nil {
-		gologger.Warningf("Could not create http client: %s\n", err)
-		return false
-	}
-
-	limiter := make(chan struct{}, r.options.Threads)
-	wg := &sync.WaitGroup{}
-
-	scanner := bufio.NewScanner(reader)
-	for scanner.Scan() {
-		text := scanner.Text()
-		if text == "" {
+	var unclusteredRequests int64
+	for _, template := range store.Templates() {
+		// workflows will dynamically adjust the totals while running, as
+		// it can't be know in advance which requests will be called
+		if len(template.Workflows) > 0 {
 			continue
 		}
-		limiter <- struct{}{}
-		wg.Add(1)
-
-		go func(URL string) {
-			var err error
-
-			if httpExecutor != nil {
-				err = httpExecutor.ExecuteHTTP(URL)
-			}
-			if dnsExecutor != nil {
-				err = dnsExecutor.ExecuteDNS(URL)
-			}
-			if err != nil {
-				gologger.Warningf("Could not execute step: %s\n", err)
-			}
-			<-limiter
-			wg.Done()
-		}(text)
+		unclusteredRequests += int64(template.TotalRequests) * r.inputCount
 	}
-	close(limiter)
-	wg.Wait()
 
-	// See if we got any results from the executors
-	var results bool
-	if httpExecutor != nil {
-		results = httpExecutor.GotResults()
-	}
-	if dnsExecutor != nil {
-		if !results {
-			results = dnsExecutor.GotResults()
+	if r.options.VerboseVerbose {
+		for _, template := range store.Templates() {
+			r.logAvailableTemplate(template.Path)
+		}
+		for _, template := range store.Workflows() {
+			r.logAvailableTemplate(template.Path)
 		}
 	}
-	return results
+	templatesMap := make(map[string]*templates.Template)
+	for _, v := range store.Templates() {
+		templatesMap[v.ID] = v
+	}
+	originalTemplatesCount := len(store.Templates())
+	clusterCount := 0
+	clusters := clusterer.Cluster(templatesMap)
+	for _, cluster := range clusters {
+		if len(cluster) > 1 && !r.options.OfflineHTTP {
+			executerOpts := protocols.ExecuterOptions{
+				Output:       r.output,
+				Options:      r.options,
+				Progress:     r.progress,
+				Catalog:      r.catalog,
+				RateLimiter:  r.ratelimiter,
+				IssuesClient: r.issuesClient,
+				Browser:      r.browser,
+				ProjectFile:  r.projectFile,
+				Interactsh:   r.interactsh,
+			}
+			clusterID := fmt.Sprintf("cluster-%s", xid.New().String())
+
+			finalTemplates = append(finalTemplates, &templates.Template{
+				ID:            clusterID,
+				RequestsHTTP:  cluster[0].RequestsHTTP,
+				Executer:      clusterer.NewExecuter(cluster, &executerOpts),
+				TotalRequests: len(cluster[0].RequestsHTTP),
+			})
+			clusterCount += len(cluster)
+		} else {
+			finalTemplates = append(finalTemplates, cluster...)
+		}
+	}
+	finalTemplates = append(finalTemplates, store.Workflows()...)
+
+	var totalRequests int64
+	for _, t := range finalTemplates {
+		if len(t.Workflows) > 0 {
+			continue
+		}
+		totalRequests += int64(t.TotalRequests) * r.inputCount
+	}
+	if totalRequests < unclusteredRequests {
+		gologger.Info().Msgf("Templates clustered: %d (Reduced %d HTTP Requests)", clusterCount, unclusteredRequests-totalRequests)
+	}
+	workflowCount := len(store.Workflows())
+	templateCount := originalTemplatesCount + workflowCount
+
+	// 0 matches means no templates were found in directory
+	if templateCount == 0 {
+		return errors.New("no templates were found")
+	}
+
+	results := &atomic.Bool{}
+	wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
+
+	// tracks global progress and captures stdout/stderr until p.Wait finishes
+	r.progress.Init(r.inputCount, templateCount, totalRequests)
+
+	for _, t := range finalTemplates {
+		wgtemplates.Add()
+		go func(template *templates.Template) {
+			defer wgtemplates.Done()
+
+			if len(template.Workflows) > 0 {
+				results.CAS(false, r.processWorkflowWithList(template))
+			} else {
+				results.CAS(false, r.processTemplateWithList(template))
+			}
+		}(t)
+	}
+	wgtemplates.Wait()
+
+	if r.interactsh != nil {
+		matched := r.interactsh.Close()
+		if matched {
+			results.CAS(false, true)
+		}
+	}
+	r.progress.Stop()
+
+	if r.issuesClient != nil {
+		r.issuesClient.Close()
+	}
+	if !results.Load() {
+		gologger.Info().Msgf("No results found. Better luck next time!")
+	}
+	if r.browser != nil {
+		r.browser.Close()
+	}
+	return nil
 }
 
-// ProcessWorkflowWithList coming from stdin or list of targets
-func (r *Runner) ProcessWorkflowWithList(workflow *workflows.Workflow) {
-	var file *os.File
-	var err error
-	// Handle a list of hosts as argument
-	if r.options.Targets != "" {
-		file, err = os.Open(r.options.Targets)
-	} else if r.options.Stdin {
-		file, err = os.Open(r.tempFile)
-	}
+// readNewTemplatesFile reads newly added templates from directory if it exists
+func (r *Runner) readNewTemplatesFile() ([]string, error) {
+	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	file, err := os.Open(additionsFile)
 	if err != nil {
-		gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err)
+		return nil, err
 	}
 	defer file.Close()
 
+	templatesList := []string{}
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
 		text := scanner.Text()
 		if text == "" {
 			continue
 		}
-		if err := r.ProcessWorkflow(workflow, text); err != nil {
-			gologger.Warningf("Could not run workflow for %s: %s\n", text, err)
-		}
+		templatesList = append(templatesList, text)
 	}
+	return templatesList, nil
 }
 
-// ProcessWorkflow towards an URL
-func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error {
-	script := tengo.NewScript([]byte(workflow.Logic))
-
-	for name, value := range workflow.Variables {
-		var writer *bufio.Writer
-		if r.output != nil {
-			writer = bufio.NewWriter(r.output)
-			defer writer.Flush()
-		}
-
-		// Check if the template is an absolute path or relative path.
-		// If the path is absolute, use it. Otherwise,
-		if r.isRelative(value) {
-			newPath, err := r.resolvePath(value)
-			if err != nil {
-				return err
-			}
-			value = newPath
-		}
-
-		// Single yaml provided
-		var templatesList []*workflows.Template
-		if strings.HasSuffix(value, ".yaml") {
-			t, err := templates.Parse(value)
-			if err != nil {
-				return err
-			}
-			template := &workflows.Template{}
-			if len(t.RequestsHTTP) > 0 {
-				template.HTTPOptions = &executor.HTTPOptions{
-					Debug:         r.options.Debug,
-					Writer:        writer,
-					Template:      t,
-					Timeout:       r.options.Timeout,
-					Retries:       r.options.Retries,
-					ProxyURL:      r.options.ProxyURL,
-					ProxySocksURL: r.options.ProxySocksURL,
-					CustomHeaders: r.options.CustomHeaders,
-				}
-			} else if len(t.RequestsDNS) > 0 {
-				template.DNSOptions = &executor.DNSOptions{
-					Debug:    r.options.Debug,
-					Template: t,
-					Writer:   writer,
-				}
-			}
-			if template.DNSOptions != nil || template.HTTPOptions != nil {
-				templatesList = append(templatesList, template)
-			}
-		} else {
-			matches := []string{}
-
-			err := godirwalk.Walk(value, &godirwalk.Options{
-				Callback: func(path string, d *godirwalk.Dirent) error {
-					if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
-						matches = append(matches, path)
-					}
-					return nil
-				},
-				ErrorCallback: func(path string, err error) godirwalk.ErrorAction {
-					return godirwalk.SkipNode
-				},
-				Unsorted: true,
-			})
-			if err != nil {
-				return err
-			}
-			// 0 matches means no templates were found in directory
-			if len(matches) == 0 {
-				return errors.New("no match found in the directory")
-			}
-
-			for _, match := range matches {
-				t, err := templates.Parse(match)
-				if err != nil {
-					return err
-				}
-				template := &workflows.Template{}
-				if len(t.RequestsHTTP) > 0 {
-					template.HTTPOptions = &executor.HTTPOptions{
-						Debug:         r.options.Debug,
-						Writer:        writer,
-						Template:      t,
-						Timeout:       r.options.Timeout,
-						Retries:       r.options.Retries,
-						ProxyURL:      r.options.ProxyURL,
-						ProxySocksURL: r.options.ProxySocksURL,
-						CustomHeaders: r.options.CustomHeaders,
-					}
-				} else if len(t.RequestsDNS) > 0 {
-					template.DNSOptions = &executor.DNSOptions{
-						Debug:    r.options.Debug,
-						Template: t,
-						Writer:   writer,
-					}
-				}
-				if template.DNSOptions != nil || template.HTTPOptions != nil {
-					templatesList = append(templatesList, template)
-				}
-			}
-		}
-
-		script.Add(name, &workflows.NucleiVar{Templates: templatesList, URL: URL})
-	}
-
-	_, err := script.RunContext(context.Background())
+// readNewTemplatesFile reads newly added templates from directory if it exists
+func (r *Runner) countNewTemplates() int {
+	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	file, err := os.Open(additionsFile)
 	if err != nil {
-		gologger.Errorf("Could not execute workflow '%s': %s\n", workflow.ID, err)
-		return err
+		return 0
 	}
-	return nil
-}
+	defer file.Close()
 
-func (r *Runner) parse(file string) (interface{}, error) {
-	// check if it's a template
-	template, errTemplate := templates.Parse(file)
-	if errTemplate == nil {
-		return template, nil
-	}
-
-	// check if it's a workflow
-	workflow, errWorkflow := workflows.Parse(file)
-	if errWorkflow == nil {
-		return workflow, nil
-	}
-
-	if errTemplate != nil {
-		return nil, errTemplate
-	}
-	if errWorkflow != nil {
-		return nil, errWorkflow
-	}
-	return nil, errors.New("unknown error occured")
+	count := 0
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text == "" {
+			continue
+		}
+		count++
+	}
+	return count
 }

v2/internal/runner/templates.go

@@ -0,0 +1,123 @@
+package runner
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/karrick/godirwalk"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"gopkg.in/yaml.v2"
+)
+
+// parseTemplateFile returns the parsed template file
+func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+
+	template := &templates.Template{}
+	err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
+	if err != nil {
+		return nil, err
+	}
+	return template, nil
+}
+
+func (r *Runner) templateLogMsg(id, name, author, severity string) string {
+	// Display the message for the template
+	message := fmt.Sprintf("[%s] %s (%s)",
+		r.colorizer.BrightBlue(id).String(),
+		r.colorizer.Bold(name).String(),
+		r.colorizer.BrightYellow(appendAtSignToAuthors(author)).String())
+	if severity != "" {
+		message += " [" + r.severityColors.Data[severity] + "]"
+	}
+	return message
+}
+
+// appendAtSignToAuthors appends @ before each author and returns final string
+func appendAtSignToAuthors(author string) string {
+	authors := strings.Split(author, ",")
+	if len(authors) == 0 {
+		return "@none"
+	}
+	if len(authors) == 1 {
+		if !strings.HasPrefix(authors[0], "@") {
+			return fmt.Sprintf("@%s", authors[0])
+		}
+		return authors[0]
+	}
+	values := make([]string, 0, len(authors))
+	for _, k := range authors {
+		if !strings.HasPrefix(authors[0], "@") {
+			values = append(values, fmt.Sprintf("@%s", k))
+		} else {
+			values = append(values, k)
+		}
+	}
+	return strings.Join(values, ",")
+}
+
+func (r *Runner) logAvailableTemplate(tplPath string) {
+	t, err := r.parseTemplateFile(tplPath)
+	if err != nil {
+		gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
+	} else {
+		gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
+	}
+}
+
+// ListAvailableTemplates prints available templates to stdout
+func (r *Runner) listAvailableTemplates() {
+	if r.templatesConfig == nil {
+		return
+	}
+
+	if _, err := os.Stat(r.templatesConfig.TemplatesDirectory); os.IsNotExist(err) {
+		gologger.Error().Msgf("%s does not exists", r.templatesConfig.TemplatesDirectory)
+		return
+	}
+
+	gologger.Print().Msgf(
+		"\nListing available v.%s nuclei templates for %s",
+		r.templatesConfig.CurrentVersion,
+		r.templatesConfig.TemplatesDirectory,
+	)
+	err := directoryWalker(
+		r.templatesConfig.TemplatesDirectory,
+		func(path string, d *godirwalk.Dirent) error {
+			if d.IsDir() && path != r.templatesConfig.TemplatesDirectory {
+				gologger.Print().Msgf("\n%s:\n\n", r.colorizer.Bold(r.colorizer.BgBrightBlue(d.Name())).String())
+			} else if strings.HasSuffix(path, ".yaml") {
+				r.logAvailableTemplate(path)
+			}
+			return nil
+		},
+	)
+	// directory couldn't be walked
+	if err != nil {
+		gologger.Error().Msgf("Could not find templates in directory '%s': %s\n", r.templatesConfig.TemplatesDirectory, err)
+	}
+}
+
+func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error {
+	return godirwalk.Walk(fsPath, &godirwalk.Options{
+		Callback: callback,
+		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
+			return godirwalk.SkipNode
+		},
+		Unsorted: true,
+	})
+}

v2/internal/runner/update.go

@@ -2,9 +2,12 @@ package runner
 
 import (
 	"archive/zip"
+	"bufio"
 	"bytes"
 	"context"
-	"errors"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -13,66 +16,36 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+	"runtime"
+	"strconv"
 	"strings"
+	"time"
 
+	"github.com/apex/log"
 	"github.com/blang/semver"
-	"github.com/google/go-github/v32/github"
-	jsoniter "github.com/json-iterator/go"
+	"github.com/google/go-github/github"
+	"github.com/olekukonko/tablewriter"
+	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+
+	"github.com/tj/go-update"
+	"github.com/tj/go-update/progress"
+	githubUpdateStore "github.com/tj/go-update/stores/github"
 )
 
-// nucleiConfig contains some configuration options for nuclei
-type nucleiConfig struct {
-	TemplatesDirectory string `json:"templates-directory,omitempty"`
-	CurrentVersion     string `json:"current-version,omitempty"`
-}
+const (
+	userName = "projectdiscovery"
+	repoName = "nuclei-templates"
+)
+
+const nucleiIgnoreFile = ".nuclei-ignore"
 
 // nucleiConfigFilename is the filename of nuclei configuration file.
-const nucleiConfigFilename = ".nuclei-config.json"
+const nucleiConfigFilename = ".templates-config.json"
 
 var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
 
-// readConfiguration reads the nuclei configuration file from disk.
-func (r *Runner) readConfiguration() (*nucleiConfig, error) {
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return nil, err
-	}
-
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
-	file, err := os.Open(templatesConfigFile)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-
-	config := &nucleiConfig{}
-	if err = jsoniter.NewDecoder(file).Decode(config); err != nil {
-		return nil, err
-	}
-	return config, nil
-}
-
-// readConfiguration reads the nuclei configuration file from disk.
-func (r *Runner) writeConfiguration(config *nucleiConfig) error {
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return err
-	}
-
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
-	file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	if err = jsoniter.NewEncoder(file).Encode(config); err != nil {
-		return err
-	}
-	return nil
-}
-
 // updateTemplates checks if the default list of nuclei-templates
 // exist in the users home directory, if not the latest revision
 // is downloaded from github.
@@ -84,46 +57,113 @@ func (r *Runner) updateTemplates() error {
 	if err != nil {
 		return err
 	}
+	configDir := path.Join(home, "/.config", "/nuclei")
+	_ = os.MkdirAll(configDir, os.ModePerm)
 
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
-	if _, err := os.Stat(templatesConfigFile); !os.IsNotExist(err) {
-		config, err := r.readConfiguration()
+	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
+	if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
+		configuration, readErr := config.ReadConfiguration()
 		if err != nil {
-			return err
+			return readErr
 		}
-		r.templatesConfig = config
+		r.templatesConfig = configuration
 	}
 
-	if r.templatesConfig == nil || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
-		if !r.options.UpdateTemplates {
-			gologger.Labelf("nuclei-templates are not installed, use update-templates flag.\n")
-			return nil
+	ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
+	if r.templatesConfig == nil {
+		currentConfig := &config.Config{
+			TemplatesDirectory: path.Join(home, "nuclei-templates"),
+			IgnoreURL:          ignoreURL,
+			NucleiVersion:      config.Version,
 		}
+		if writeErr := config.WriteConfiguration(currentConfig, false, false); writeErr != nil {
+			return errors.Wrap(writeErr, "could not write template configuration")
+		}
+		r.templatesConfig = currentConfig
+	}
+
+	if r.options.NoUpdateTemplates {
+		return nil
+	}
+	// Check if last checked for nuclei-ignore is more than 1 hours.
+	// and if true, run the check.
+	//
+	// Also at the same time fetch latest version from github to do outdated nuclei
+	// and templates check.
+	checkedIgnore := false
+	if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
+		r.fetchLatestVersionsFromGithub()
+
+		if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" {
+			ignoreURL = r.templatesConfig.IgnoreURL
+		}
+		gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
+
+		checkedIgnore = true
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
+		if reqErr == nil {
+			resp, httpGet := http.DefaultClient.Do(req)
+			if httpGet != nil {
+				if resp != nil && resp.Body != nil {
+					resp.Body.Close()
+				}
+				gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
+			} else {
+				data, _ := ioutil.ReadAll(resp.Body)
+				resp.Body.Close()
+
+				if len(data) > 0 {
+					_ = ioutil.WriteFile(path.Join(configDir, nucleiIgnoreFile), data, 0644)
+				}
+				if r.templatesConfig != nil {
+					err = config.WriteConfiguration(r.templatesConfig, false, true)
+					if err != nil {
+						gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
+					}
+				}
+			}
+		}
+		cancel()
+	}
+
+	ctx := context.Background()
+	if r.templatesConfig.CurrentVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
+		gologger.Info().Msgf("nuclei-templates are not installed, installing...\n")
 
 		// Use custom location if user has given a template directory
-		if r.options.TemplatesDirectory != "" {
-			home = r.options.TemplatesDirectory
+		r.templatesConfig = &config.Config{
+			TemplatesDirectory: path.Join(home, "nuclei-templates"),
+		}
+		if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") {
+			r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
 		}
-		r.templatesConfig = &nucleiConfig{TemplatesDirectory: path.Join(home, "nuclei-templates")}
-		os.RemoveAll(r.templatesConfig.TemplatesDirectory)
 
 		// Download the repository and also write the revision to a HEAD file.
-		version, asset, err := r.getLatestReleaseFromGithub()
-		if err != nil {
-			return err
+		version, asset, getErr := r.getLatestReleaseFromGithub()
+		if getErr != nil {
+			return getErr
 		}
+		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
 
-		gologger.Verbosef("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory)
-
-		if err = r.downloadReleaseAndUnzip(asset.GetZipballURL()); err != nil {
+		r.fetchLatestVersionsFromGithub() // also fetch latest versions
+		_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
+		if err != nil {
 			return err
 		}
 		r.templatesConfig.CurrentVersion = version.String()
-		if err = r.writeConfiguration(r.templatesConfig); err != nil {
+
+		err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
+		if err != nil {
 			return err
 		}
+		gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). GoodLuck!\n", version.String())
+		return nil
+	}
 
-		gologger.Infof("Successfully downloaded nuclei-templates (v%s). Enjoy!\n", version.String())
+	// Check if last checked is more than 24 hours and we don't have updateTemplates flag.
+	// If not, return since we don't want to do anything now.
+	if time.Since(r.templatesConfig.LastChecked) < 24*time.Hour && !r.options.UpdateTemplates {
 		return nil
 	}
 
@@ -136,6 +176,7 @@ func (r *Runner) updateTemplates() error {
 	if indices[0] > 0 {
 		verText = verText[indices[0]:]
 	}
+
 	oldVersion, err := semver.Make(verText)
 	if err != nil {
 		return err
@@ -147,39 +188,33 @@ func (r *Runner) updateTemplates() error {
 	}
 
 	if version.EQ(oldVersion) {
-		gologger.Labelf("Latest version of nuclei-templates installed: v%s\n", oldVersion.String())
-		return nil
+		return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore)
 	}
+
 	if version.GT(oldVersion) {
-		if !r.options.UpdateTemplates {
-			gologger.Labelf("You're using outdated nuclei-templates. Latest v%s\n", version.String())
-			return nil
-		}
+		gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
+		gologger.Info().Msgf("Downloading latest release...")
+
 		if r.options.TemplatesDirectory != "" {
-			home = r.options.TemplatesDirectory
-			r.templatesConfig.TemplatesDirectory = path.Join(home, "nuclei-templates")
+			r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
 		}
 		r.templatesConfig.CurrentVersion = version.String()
-		os.RemoveAll(r.templatesConfig.TemplatesDirectory)
 
-		gologger.Verbosef("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory)
-
-		if err = r.downloadReleaseAndUnzip(asset.GetZipballURL()); err != nil {
+		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
+		r.fetchLatestVersionsFromGithub()
+		_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
+		if err != nil {
 			return err
 		}
-		if err = r.writeConfiguration(r.templatesConfig); err != nil {
+		err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
+		if err != nil {
 			return err
 		}
-		gologger.Infof("Successfully updated nuclei-templates (v%s). Enjoy!\n", version.String())
+		gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", version.String())
 	}
 	return nil
 }
 
-const (
-	userName = "projectdiscovery"
-	repoName = "nuclei-templates"
-)
-
 // getLatestReleaseFromGithub returns the latest release from github
 func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.RepositoryRelease, error) {
 	client := github.NewClient(nil)
@@ -201,6 +236,7 @@ func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.Repositor
 		if indices[0] > 0 {
 			verText = verText[indices[0]:]
 		}
+
 		ver, err := semver.Make(verText)
 		if err != nil {
 			return semver.Version{}, nil, err
@@ -218,36 +254,88 @@ func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.Repositor
 }
 
 // downloadReleaseAndUnzip downloads and unzips the release in a directory
-func (r *Runner) downloadReleaseAndUnzip(downloadURL string) error {
-	req, err := http.NewRequest("GET", downloadURL, nil)
+func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadURL string) (*templateUpdateResults, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
-		return fmt.Errorf("Failed to create HTTP request to %s: %s", downloadURL, err)
+		return nil, fmt.Errorf("failed to create HTTP request to %s: %s", downloadURL, err)
 	}
 
 	res, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return fmt.Errorf("Failed to download a release file from %s: %s", downloadURL, err)
+		return nil, fmt.Errorf("failed to download a release file from %s: %s", downloadURL, err)
 	}
 	defer res.Body.Close()
-
-	if res.StatusCode != 200 {
-		return fmt.Errorf("Failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode)
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode)
 	}
 
 	buf, err := ioutil.ReadAll(res.Body)
 	if err != nil {
-		return fmt.Errorf("Failed to create buffer for zip file: %s", err)
+		return nil, fmt.Errorf("failed to create buffer for zip file: %s", err)
 	}
 
 	reader := bytes.NewReader(buf)
 	z, err := zip.NewReader(reader, reader.Size())
 	if err != nil {
-		return fmt.Errorf("Failed to uncompress zip file: %s", err)
+		return nil, fmt.Errorf("failed to uncompress zip file: %s", err)
 	}
 
 	// Create the template folder if it doesn't exists
-	os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm)
+	err = os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create template base folder: %s", err)
+	}
 
+	results, err := r.compareAndWriteTemplates(z)
+	if err != nil {
+		return nil, fmt.Errorf("failed to write templates: %s", err)
+	}
+
+	if r.options.Verbose {
+		r.printUpdateChangelog(results, version)
+	}
+	checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
+	err = writeTemplatesChecksum(checksumFile, results.checksums)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not write checksum")
+	}
+
+	// Write the additions to a cached file for new runs.
+	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	buffer := &bytes.Buffer{}
+	for _, addition := range results.additions {
+		buffer.WriteString(addition)
+		buffer.WriteString("\n")
+	}
+	err = ioutil.WriteFile(additionsFile, buffer.Bytes(), os.ModePerm)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not write new additions file")
+	}
+	return results, err
+}
+
+type templateUpdateResults struct {
+	additions     []string
+	deletions     []string
+	modifications []string
+	totalCount    int
+	checksums     map[string]string
+}
+
+// compareAndWriteTemplates compares and returns the stats of a template
+// update operations.
+func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults, error) {
+	results := &templateUpdateResults{
+		checksums: make(map[string]string),
+	}
+
+	// We use file-checksums that are md5 hashes to store the list of files->hashes
+	// that have been downloaded previously.
+	// If the path isn't found in new update after being read from the previous checksum,
+	// it is removed. This allows us fine-grained control over the download process
+	// as well as solves a long problem with nuclei-template updates.
+	checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
+	previousChecksum, _ := readPreviousTemplatesChecksum(checksumFile)
 	for _, file := range z.File {
 		directory, name := filepath.Split(file.Name)
 		if name == "" {
@@ -256,52 +344,252 @@ func (r *Runner) downloadReleaseAndUnzip(downloadURL string) error {
 		paths := strings.Split(directory, "/")
 		finalPath := strings.Join(paths[1:], "/")
 
+		if strings.HasPrefix(name, ".") || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") {
+			continue
+		}
+		results.totalCount++
 		templateDirectory := path.Join(r.templatesConfig.TemplatesDirectory, finalPath)
-		os.MkdirAll(templateDirectory, os.ModePerm)
+		err := os.MkdirAll(templateDirectory, os.ModePerm)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err)
+		}
+
+		templatePath := path.Join(templateDirectory, name)
 
-		f, err := os.Create(path.Join(templateDirectory, name))
+		isAddition := false
+		if _, statErr := os.Stat(templatePath); os.IsNotExist(statErr) {
+			isAddition = true
+		}
+		f, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777)
 		if err != nil {
-			return fmt.Errorf("Could not create uncompressed file: %s", err)
+			f.Close()
+			return nil, fmt.Errorf("could not create uncompressed file: %s", err)
 		}
-		defer f.Close()
 
 		reader, err := file.Open()
 		if err != nil {
-			return fmt.Errorf("Could not open archive to extract file: %s", err)
+			f.Close()
+			return nil, fmt.Errorf("could not open archive to extract file: %s", err)
 		}
-		io.Copy(f, reader)
+		hasher := md5.New()
+
+		// Save file and also read into hasher for md5
+		_, err = io.Copy(f, io.TeeReader(reader, hasher))
+		if err != nil {
+			f.Close()
+			return nil, fmt.Errorf("could not write template file: %s", err)
+		}
+		f.Close()
+
+		oldChecksum, checksumOK := previousChecksum[templatePath]
+
+		checksum := hex.EncodeToString(hasher.Sum(nil))
+		if isAddition {
+			results.additions = append(results.additions, path.Join(finalPath, name))
+		} else if checksumOK && oldChecksum[0] != checksum {
+			results.modifications = append(results.modifications, path.Join(finalPath, name))
+		}
+		results.checksums[templatePath] = checksum
+	}
+
+	// If we don't find a previous file in new download and it hasn't been
+	// changed on the disk, delete it.
+	for k, v := range previousChecksum {
+		_, ok := results.checksums[k]
+		if !ok && v[0] == v[1] {
+			os.Remove(k)
+			results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(k, r.templatesConfig.TemplatesDirectory), "/"))
+		}
+	}
+	return results, nil
+}
+
+// readPreviousTemplatesChecksum reads the previous checksum file from the disk.
+//
+// It reads two checksums, the first checksum is what we expect and the second is
+// the actual checksum of the file on disk currently.
+func readPreviousTemplatesChecksum(file string) (map[string][2]string, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+
+	checksum := make(map[string][2]string)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text == "" {
+			continue
+		}
+		parts := strings.Split(text, ",")
+		if len(parts) < 2 {
+			continue
+		}
+		values := [2]string{parts[1]}
+
+		f, err := os.Open(parts[0])
+		if err != nil {
+			return nil, err
+		}
+
+		hasher := md5.New()
+		if _, err := io.Copy(hasher, f); err != nil {
+			return nil, err
+		}
+		f.Close()
+
+		values[1] = hex.EncodeToString(hasher.Sum(nil))
+		checksum[parts[0]] = values
+	}
+	return checksum, nil
+}
+
+// writeTemplatesChecksum writes the nuclei-templates checksum data to disk.
+func writeTemplatesChecksum(file string, checksum map[string]string) error {
+	f, err := os.Create(file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	builder := &strings.Builder{}
+	for k, v := range checksum {
+		builder.WriteString(k)
+		builder.WriteString(",")
+		builder.WriteString(v)
+		builder.WriteString("\n")
+
+		if _, checksumErr := f.WriteString(builder.String()); checksumErr != nil {
+			return err
+		}
+		builder.Reset()
 	}
 	return nil
 }
 
-// isRelative checks if a given path is a relative path
-func (r *Runner) isRelative(path string) bool {
-	if strings.HasPrefix(path, "/") || strings.Contains(path, ":\\") {
-		return false
+func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version string) {
+	if len(results.additions) > 0 && r.options.Verbose {
+		gologger.Print().Msgf("\nNewly added templates: \n\n")
+
+		for _, addition := range results.additions {
+			gologger.Print().Msgf("%s", addition)
+		}
 	}
-	return true
+
+	gologger.Print().Msgf("\nNuclei Templates v%s Changelog\n", version)
+	data := [][]string{
+		{strconv.Itoa(results.totalCount), strconv.Itoa(len(results.additions)), strconv.Itoa(len(results.deletions))},
+	}
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Total", "Added", "Removed"})
+	for _, v := range data {
+		table.Append(v)
+	}
+	table.Render()
 }
 
-// resolvePath gets the absolute path to the template by either
-// looking in the current directory or checking the nuclei templates directory.
-//
-// Current directory is given preference over the nuclei-templates directory.
-func (r *Runner) resolvePath(templateName string) (string, error) {
-	curDirectory, err := os.Getwd()
+// fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github
+func (r *Runner) fetchLatestVersionsFromGithub() {
+	nucleiLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei")
+	if err != nil {
+		gologger.Warning().Msgf("Could not fetch latest nuclei release: %s", err)
+	}
+	templatesLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei-templates")
+	if err != nil {
+		gologger.Warning().Msgf("Could not fetch latest nuclei-templates release: %s", err)
+	}
+	if r.templatesConfig != nil {
+		r.templatesConfig.NucleiLatestVersion = nucleiLatest
+		r.templatesConfig.NucleiTemplatesLatestVersion = templatesLatest
+	}
+}
+
+type githubTagData struct {
+	Name string
+}
+
+// githubFetchLatestTagRepo fetches latest tag from github
+// This function was half written by github copilot AI :D.
+func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	url := fmt.Sprintf("https://api.github.com/repos/%s/tags", repo)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
 		return "", err
 	}
-	templatePath := path.Join(curDirectory, templateName)
-	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
-		gologger.Infof("Found template in current directory: %s\n", templatePath)
-		return templatePath, nil
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
 	}
-	if r.templatesConfig != nil {
-		templatePath := path.Join(r.templatesConfig.TemplatesDirectory, templateName)
-		if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
-			gologger.Infof("Found template in nuclei-templates directory: %s\n", templatePath)
-			return templatePath, nil
-		}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
 	}
-	return "", fmt.Errorf("no such path found: %s", templateName)
+
+	var tags []githubTagData
+	err = json.Unmarshal(body, &tags)
+	if err != nil {
+		return "", err
+	}
+	if len(tags) == 0 {
+		return "", fmt.Errorf("no tags found for %s", repo)
+	}
+	return strings.TrimPrefix(tags[0].Name, "v"), nil
+}
+
+// updateNucleiVersionToLatest implements nuclei auto-updation using Github Releases.
+func updateNucleiVersionToLatest(verbose bool) error {
+	if verbose {
+		log.SetLevel(log.DebugLevel)
+	}
+	var command string
+	switch runtime.GOOS {
+	case "windows":
+		command = "nuclei.exe"
+	default:
+		command = "nuclei"
+	}
+	m := &update.Manager{
+		Command: command,
+		Store: &githubUpdateStore.Store{
+			Owner:   "projectdiscovery",
+			Repo:    "nuclei",
+			Version: config.Version,
+		},
+	}
+	releases, err := m.LatestReleases()
+	if err != nil {
+		return errors.Wrap(err, "could not fetch latest release")
+	}
+	if len(releases) == 0 {
+		gologger.Info().Msgf("No new updates found for nuclei engine!")
+		return nil
+	}
+
+	latest := releases[0]
+	var currentOS string
+	switch runtime.GOOS {
+	case "darwin":
+		currentOS = "macOS"
+	default:
+		currentOS = runtime.GOOS
+	}
+	final := latest.FindZip(currentOS, runtime.GOARCH)
+	if final == nil {
+		return fmt.Errorf("no compatible binary found for %s/%s", currentOS, runtime.GOARCH)
+	}
+	tarball, err := final.DownloadProxy(progress.Reader)
+	if err != nil {
+		return errors.Wrap(err, "could not download latest release")
+	}
+	if err := m.Install(tarball); err != nil {
+		return errors.Wrap(err, "could not install latest release")
+	}
+	gologger.Info().Msgf("Successfully updated to Nuclei %s\n", latest.Version)
+	return nil
 }

v2/internal/runner/update_test.go

@@ -0,0 +1,156 @@
+package runner
+
+import (
+	"archive/zip"
+	"context"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
+	gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{})
+
+	baseTemplates, err := ioutil.TempDir("", "old-temp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(baseTemplates)
+
+	err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create write base file")
+
+	err = zipFromDirectory("base.zip", baseTemplates)
+	require.Nil(t, err, "could not create zip from directory")
+	defer os.Remove("base.zip")
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "base.zip")
+	}))
+	defer ts.Close()
+
+	templatesDirectory, err := ioutil.TempDir("", "template-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(templatesDirectory)
+
+	r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}, options: testutils.DefaultOptions}
+	results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
+	require.Nil(t, err, "could not download release and unzip")
+	require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")
+
+	newTempDir, err := ioutil.TempDir("", "new-tmp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(newTempDir)
+
+	err = ioutil.WriteFile(path.Join(newTempDir, "base.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create base file")
+	err = ioutil.WriteFile(path.Join(newTempDir, "new.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create new file")
+
+	err = zipFromDirectory("new.zip", newTempDir)
+	require.Nil(t, err, "could not create new zip from directory")
+	defer os.Remove("new.zip")
+
+	ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "new.zip")
+	}))
+	defer ts2.Close()
+
+	results, err = r.downloadReleaseAndUnzip(context.Background(), "1.0.1", ts2.URL)
+	require.Nil(t, err, "could not download release and unzip")
+
+	require.Equal(t, "new.yaml", results.additions[0], "could not get correct new addition")
+}
+
+func TestDownloadReleaseAndUnzipDeletion(t *testing.T) {
+	gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{})
+
+	baseTemplates, err := ioutil.TempDir("", "old-temp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(baseTemplates)
+
+	err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create write base file")
+
+	err = zipFromDirectory("base.zip", baseTemplates)
+	require.Nil(t, err, "could not create zip from directory")
+	defer os.Remove("base.zip")
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "base.zip")
+	}))
+	defer ts.Close()
+
+	templatesDirectory, err := ioutil.TempDir("", "template-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(templatesDirectory)
+
+	r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}, options: testutils.DefaultOptions}
+
+	results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
+	require.Nil(t, err, "could not download release and unzip")
+	require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")
+
+	newTempDir, err := ioutil.TempDir("", "new-tmp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(newTempDir)
+
+	err = zipFromDirectory("new.zip", newTempDir)
+	require.Nil(t, err, "could not create new zip from directory")
+	defer os.Remove("new.zip")
+
+	ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "new.zip")
+	}))
+	defer ts2.Close()
+
+	results, err = r.downloadReleaseAndUnzip(context.Background(), "1.0.1", ts2.URL)
+	require.Nil(t, err, "could not download release and unzip")
+
+	require.Equal(t, "base.yaml", results.deletions[0], "could not get correct new deletions")
+}
+
+func zipFromDirectory(zipPath, directory string) error {
+	file, err := os.Create(zipPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	w := zip.NewWriter(file)
+	defer w.Close()
+
+	walker := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		file, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		f, err := w.Create(strings.TrimPrefix(path, directory))
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(f, file)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+	return filepath.Walk(directory, walker)
+}

v2/internal/runner/validate.go

@@ -1,57 +0,0 @@
-package runner
-
-import (
-	"errors"
-	"net/url"
-
-	"github.com/projectdiscovery/gologger"
-)
-
-// validateOptions validates the configuration options passed
-func (options *Options) validateOptions() error {
-	// Both verbose and silent flags were used
-	if options.Verbose && options.Silent {
-		return errors.New("both verbose and silent mode specified")
-	}
-
-	// Check if a list of templates was provided and it exists
-	if options.Templates == "" && !options.UpdateTemplates {
-		return errors.New("no template/templates provided")
-	}
-
-	if options.Targets == "" && !options.Stdin && options.Target == "" && !options.UpdateTemplates {
-		return errors.New("no target input provided")
-	}
-
-	// Validate proxy options if provided
-	if options.ProxyURL != "" && !isValidProxyURL(options.ProxyURL) {
-		return errors.New("invalid http proxy format (It should be http://username:password@host:port)")
-	}
-	if options.ProxySocksURL != "" && !isValidProxyURL(options.ProxySocksURL) {
-		return errors.New("invalid socks proxy format (It should be socks5://username:password@host:port)")
-	}
-
-	return nil
-}
-
-func isValidProxyURL(URL string) bool {
-	if _, err := url.Parse(URL); err != nil {
-		return false
-	}
-
-	return true
-}
-
-// configureOutput configures the output on the screen
-func (options *Options) configureOutput() {
-	// If the user desires verbose output, show verbose output
-	if options.Verbose {
-		gologger.MaxLevel = gologger.Verbose
-	}
-	if options.NoColor {
-		gologger.UseColors = false
-	}
-	if options.Silent {
-		gologger.MaxLevel = gologger.Silent
-	}
-}

v2/internal/testutils/integration.go

@@ -0,0 +1,96 @@
+package testutils
+
+import (
+	"net"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+// RunNucleiAndGetResults returns a list of results for a template
+func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
+	cmd := exec.Command("./nuclei", "-t", template, "-target", url, "-silent")
+	if debug {
+		cmd = exec.Command("./nuclei", "-t", template, "-target", url, "-debug")
+		cmd.Stderr = os.Stderr
+	}
+	cmd.Args = append(cmd.Args, extra...)
+
+	data, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+	parts := []string{}
+	items := strings.Split(string(data), "\n")
+	for _, i := range items {
+		if i != "" {
+			parts = append(parts, i)
+		}
+	}
+	return parts, nil
+}
+
+// RunNucleiWorkflowAndGetResults returns a list of results for a workflow
+func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
+	cmd := exec.Command("./nuclei", "-w", template, "-target", url, "-silent")
+	if debug {
+		cmd = exec.Command("./nuclei", "-w", template, "-target", url, "-debug")
+		cmd.Stderr = os.Stderr
+	}
+	cmd.Args = append(cmd.Args, extra...)
+
+	data, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+	parts := []string{}
+	items := strings.Split(string(data), "\n")
+	for _, i := range items {
+		if i != "" {
+			parts = append(parts, i)
+		}
+	}
+	return parts, nil
+}
+
+// TestCase is a single integration test case
+type TestCase interface {
+	// Execute executes a test case and returns any errors if occurred
+	Execute(filePath string) error
+}
+
+// TCPServer creates a new tcp server that returns a response
+type TCPServer struct {
+	URL      string
+	listener net.Listener
+}
+
+// NewTCPServer creates a new TCP server from a handler
+func NewTCPServer(handler func(conn net.Conn)) *TCPServer {
+	server := &TCPServer{}
+
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		panic(err)
+	}
+	server.URL = l.Addr().String()
+	server.listener = l
+
+	go func() {
+		for {
+			// Listen for an incoming connection.
+			conn, err := l.Accept()
+			if err != nil {
+				continue
+			}
+			// Handle connections in a new goroutine.
+			go handler(conn)
+		}
+	}()
+	return server
+}
+
+// Close closes the TCP server
+func (s *TCPServer) Close() {
+	s.listener.Close()
+}

v2/internal/testutils/testutils.go

@@ -0,0 +1,125 @@
+package testutils
+
+import (
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"go.uber.org/ratelimit"
+)
+
+// Init initializes the protocols and their configurations
+func Init(options *types.Options) {
+	_ = protocolinit.Init(options)
+}
+
+// DefaultOptions is the default options structure for nuclei during mocking.
+var DefaultOptions = &types.Options{
+	Metrics:            false,
+	Debug:              false,
+	DebugRequests:      false,
+	DebugResponse:      false,
+	Silent:             false,
+	Version:            false,
+	Verbose:            false,
+	NoColor:            true,
+	UpdateTemplates:    false,
+	JSON:               false,
+	JSONRequests:       false,
+	EnableProgressBar:  false,
+	TemplatesVersion:   false,
+	TemplateList:       false,
+	Stdin:              false,
+	StopAtFirstMatch:   false,
+	NoMeta:             false,
+	Project:            false,
+	MetricsPort:        0,
+	BulkSize:           25,
+	TemplateThreads:    10,
+	Timeout:            5,
+	Retries:            1,
+	RateLimit:          150,
+	ProjectPath:        "",
+	Severity:           []string{},
+	Target:             "",
+	Targets:            "",
+	Output:             "",
+	ProxyURL:           "",
+	ProxySocksURL:      "",
+	TemplatesDirectory: "",
+	TraceLogFile:       "",
+	Templates:          []string{},
+	ExcludedTemplates:  []string{},
+	CustomHeaders:      []string{},
+}
+
+// MockOutputWriter is a mocked output writer.
+type MockOutputWriter struct {
+	aurora          aurora.Aurora
+	RequestCallback func(templateID, url, requestType string, err error)
+	WriteCallback   func(o *output.ResultEvent)
+}
+
+// NewMockOutputWriter creates a new mock output writer
+func NewMockOutputWriter() *MockOutputWriter {
+	return &MockOutputWriter{aurora: aurora.NewAurora(false)}
+}
+
+// Close closes the output writer interface
+func (m *MockOutputWriter) Close() {}
+
+// Colorizer returns the colorizer instance for writer
+func (m *MockOutputWriter) Colorizer() aurora.Aurora {
+	return m.aurora
+}
+
+// Write writes the event to file and/or screen.
+func (m *MockOutputWriter) Write(result *output.ResultEvent) error {
+	if m.WriteCallback != nil {
+		m.WriteCallback(result)
+	}
+	return nil
+}
+
+// Request writes a log the requests trace log
+func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) {
+	if m.RequestCallback != nil {
+		m.RequestCallback(templateID, url, requestType, err)
+	}
+}
+
+// TemplateInfo contains info for a mock executed template.
+type TemplateInfo struct {
+	ID   string
+	Info map[string]interface{}
+	Path string
+}
+
+// NewMockExecuterOptions creates a new mock executeroptions struct
+func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecuterOptions {
+	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
+	executerOpts := &protocols.ExecuterOptions{
+		TemplateID:   info.ID,
+		TemplateInfo: info.Info,
+		TemplatePath: info.Path,
+		Output:       NewMockOutputWriter(),
+		Options:      options,
+		Progress:     progressImpl,
+		ProjectFile:  nil,
+		IssuesClient: nil,
+		Browser:      nil,
+		Catalog:      catalog.New(options.TemplatesDirectory),
+		RateLimiter:  ratelimit.New(options.RateLimit),
+	}
+	return executerOpts
+}
+
+// NoopWriter is a NooP gologger writer.
+type NoopWriter struct{}
+
+// Write writes the data to an output writer.
+func (n *NoopWriter) Write(data []byte, level levels.Level) {}

v2/pkg/catalog/catalogue.go

@@ -0,0 +1,12 @@
+package catalog
+
+// Catalog is a template catalog helper implementation
+type Catalog struct {
+	templatesDirectory string
+}
+
+// New creates a new Catalog structure using provided input items
+func New(directory string) *Catalog {
+	catalog := &Catalog{templatesDirectory: directory}
+	return catalog
+}

v2/pkg/catalog/config/config.go

@@ -0,0 +1,138 @@
+package config
+
+import (
+	"os"
+	"path"
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+	"gopkg.in/yaml.v2"
+)
+
+// Config contains the internal nuclei engine configuration
+type Config struct {
+	TemplatesDirectory string    `json:"templates-directory,omitempty"`
+	CurrentVersion     string    `json:"current-version,omitempty"`
+	LastChecked        time.Time `json:"last-checked,omitempty"`
+	IgnoreURL          string    `json:"ignore-url,omitempty"`
+	NucleiVersion      string    `json:"nuclei-version,omitempty"`
+	LastCheckedIgnore  time.Time `json:"last-checked-ignore,omitempty"`
+
+	NucleiLatestVersion          string `json:"nuclei-latest-version"`
+	NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"`
+}
+
+// nucleiConfigFilename is the filename of nuclei configuration file.
+const nucleiConfigFilename = ".templates-config.json"
+
+// Version is the current version of nuclei
+const Version = `2.4.2`
+
+func getConfigDetails() (string, error) {
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		return "", errors.Wrap(err, "could not get home directory")
+	}
+	configDir := path.Join(homeDir, "/.config", "/nuclei")
+	_ = os.MkdirAll(configDir, os.ModePerm)
+	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
+	return templatesConfigFile, nil
+}
+
+// ReadConfiguration reads the nuclei configuration file from disk.
+func ReadConfiguration() (*Config, error) {
+	templatesConfigFile, err := getConfigDetails()
+	if err != nil {
+		return nil, err
+	}
+
+	file, err := os.Open(templatesConfigFile)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	config := &Config{}
+	err = jsoniter.NewDecoder(file).Decode(config)
+	if err != nil {
+		return nil, err
+	}
+	return config, nil
+}
+
+// WriteConfiguration writes the updated nuclei configuration to disk
+func WriteConfiguration(config *Config, checked, checkedIgnore bool) error {
+	if config.IgnoreURL == "" {
+		config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
+	}
+	if checked {
+		config.LastChecked = time.Now()
+	}
+	if checkedIgnore {
+		config.LastCheckedIgnore = time.Now()
+	}
+	config.NucleiVersion = Version
+
+	templatesConfigFile, err := getConfigDetails()
+	if err != nil {
+		return err
+	}
+	file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	err = jsoniter.NewEncoder(file).Encode(config)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+const nucleiIgnoreFile = ".nuclei-ignore"
+
+// IgnoreFile is an internal nuclei template blocking configuration file
+type IgnoreFile struct {
+	Tags  []string `yaml:"tags"`
+	Files []string `yaml:"files"`
+}
+
+// ReadIgnoreFile reads the nuclei ignore file returning blocked tags and paths
+func ReadIgnoreFile() IgnoreFile {
+	file, err := os.Open(getIgnoreFilePath())
+	if err != nil {
+		gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err)
+		return IgnoreFile{}
+	}
+	defer file.Close()
+
+	ignore := IgnoreFile{}
+	if err := yaml.NewDecoder(file).Decode(&ignore); err != nil {
+		gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err)
+		return IgnoreFile{}
+	}
+	return ignore
+}
+
+// getIgnoreFilePath returns the ignore file path for the runner
+func getIgnoreFilePath() string {
+	var defIgnoreFilePath string
+
+	home, err := os.UserHomeDir()
+	if err == nil {
+		configDir := path.Join(home, "/.config", "/nuclei")
+		_ = os.MkdirAll(configDir, os.ModePerm)
+
+		defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile)
+		return defIgnoreFilePath
+	}
+	cwd, err := os.Getwd()
+	if err != nil {
+		return defIgnoreFilePath
+	}
+	cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile)
+	return cwdIgnoreFilePath
+}

v2/pkg/catalog/find.go

@@ -0,0 +1,148 @@
+package catalog
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/karrick/godirwalk"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+)
+
+// GetTemplatesPath returns a list of absolute paths for the provided template list.
+func (c *Catalog) GetTemplatesPath(definitions []string) []string {
+	// keeps track of processed dirs and files
+	processed := make(map[string]bool)
+	allTemplates := []string{}
+
+	for _, t := range definitions {
+		paths, err := c.GetTemplatePath(t)
+		if err != nil {
+			gologger.Error().Msgf("Could not find template '%s': %s\n", t, err)
+		}
+		for _, path := range paths {
+			if _, ok := processed[path]; !ok {
+				processed[path] = true
+				allTemplates = append(allTemplates, path)
+			}
+		}
+	}
+	return allTemplates
+}
+
+// GetTemplatePath parses the specified input template path and returns a compiled
+// list of finished absolute paths to the templates evaluating any glob patterns
+// or folders provided as in.
+func (c *Catalog) GetTemplatePath(target string) ([]string, error) {
+	processed := make(map[string]struct{})
+
+	absPath, err := c.convertPathToAbsolute(target)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not find template file")
+	}
+
+	// Template input includes a wildcard
+	if strings.Contains(absPath, "*") {
+		matches, findErr := c.findGlobPathMatches(absPath, processed)
+		if findErr != nil {
+			return nil, errors.Wrap(findErr, "could not find glob matches")
+		}
+		if len(matches) == 0 {
+			return nil, errors.Errorf("no templates found for path")
+		}
+		return matches, nil
+	}
+
+	// Template input is either a file or a directory
+	match, file, err := c.findFileMatches(absPath, processed)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not find file")
+	}
+	if file {
+		if match != "" {
+			return []string{match}, nil
+		}
+		return nil, nil
+	}
+
+	// Recursively walk down the Templates directory and run all
+	// the template file checks
+	matches, err := c.findDirectoryMatches(absPath, processed)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not find directory matches")
+	}
+	if len(matches) == 0 {
+		return nil, errors.Errorf("no templates found in path")
+	}
+	return matches, nil
+}
+
+// convertPathToAbsolute resolves the paths provided to absolute paths
+// before doing any operations on them regardless of them being blob, folders, files, etc.
+func (c *Catalog) convertPathToAbsolute(t string) (string, error) {
+	if strings.Contains(t, "*") {
+		file := path.Base(t)
+		absPath, err := c.ResolvePath(path.Dir(t), "")
+		if err != nil {
+			return "", err
+		}
+		return path.Join(absPath, file), nil
+	}
+	return c.ResolvePath(t, "")
+}
+
+// findGlobPathMatches returns the matched files from a glob path
+func (c *Catalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) {
+	matches, err := filepath.Glob(absPath)
+	if err != nil {
+		return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
+	}
+	results := make([]string, 0, len(matches))
+	for _, match := range matches {
+		if _, ok := processed[match]; !ok {
+			processed[match] = struct{}{}
+			results = append(results, match)
+		}
+	}
+	return results, nil
+}
+
+// findFileMatches finds if a path is an absolute file. If the path
+// is a file, it returns true otherwise false with no errors.
+func (c *Catalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) {
+	info, err := os.Stat(absPath)
+	if err != nil {
+		return "", false, err
+	}
+	if !info.Mode().IsRegular() {
+		return "", false, nil
+	}
+	if _, ok := processed[absPath]; !ok {
+		processed[absPath] = struct{}{}
+		return absPath, true, nil
+	}
+	return "", true, nil
+}
+
+// findDirectoryMatches finds matches for templates from a directory
+func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) {
+	var results []string
+	err := godirwalk.Walk(absPath, &godirwalk.Options{
+		Unsorted: true,
+		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
+			return godirwalk.SkipNode
+		},
+		Callback: func(path string, d *godirwalk.Dirent) error {
+			if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
+				if _, ok := processed[path]; !ok {
+					results = append(results, path)
+					processed[path] = struct{}{}
+				}
+			}
+			return nil
+		},
+	})
+	return results, err
+}

v2/pkg/catalog/loader/filter/path_filter.go

@@ -0,0 +1,45 @@
+package filter
+
+import "github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+
+// PathFilter is a path based template filter
+type PathFilter struct {
+	excludedTemplates          []string
+	alwaysIncludedTemplatesMap map[string]struct{}
+}
+
+// PathFilterConfig contains configuration options for Path based templates Filter
+type PathFilterConfig struct {
+	IncludedTemplates []string
+	ExcludedTemplates []string
+}
+
+// NewPathFilter creates a new path filter from provided config
+func NewPathFilter(config *PathFilterConfig, catalogClient *catalog.Catalog) *PathFilter {
+	filter := &PathFilter{
+		excludedTemplates:          catalogClient.GetTemplatesPath(config.ExcludedTemplates),
+		alwaysIncludedTemplatesMap: make(map[string]struct{}),
+	}
+
+	alwaysIncludeTemplates := catalogClient.GetTemplatesPath(config.IncludedTemplates)
+	for _, tpl := range alwaysIncludeTemplates {
+		filter.alwaysIncludedTemplatesMap[tpl] = struct{}{}
+	}
+	return filter
+}
+
+// Match performs match for path filter on templates and returns final list
+func (p *PathFilter) Match(templates []string) map[string]struct{} {
+	templatesMap := make(map[string]struct{})
+	for _, tpl := range templates {
+		templatesMap[tpl] = struct{}{}
+	}
+	for _, template := range p.excludedTemplates {
+		if _, ok := p.alwaysIncludedTemplatesMap[template]; ok {
+			continue
+		} else {
+			delete(templatesMap, template)
+		}
+	}
+	return templatesMap
+}

v2/pkg/catalog/loader/filter/tag_filter.go

@@ -0,0 +1,178 @@
+package filter
+
+import (
+	"errors"
+	"strings"
+)
+
+// TagFilter is used to filter nuclei templates for tag based execution
+type TagFilter struct {
+	allowedTags map[string]struct{}
+	severities  map[string]struct{}
+	authors     map[string]struct{}
+	block       map[string]struct{}
+	matchAllows map[string]struct{}
+}
+
+// ErrExcluded is returned for execluded templates
+var ErrExcluded = errors.New("the template was excluded")
+
+// Match takes a tag and whether the template was matched from user
+// input and returns true or false using a tag filter.
+//
+// If the tag was specified in deny list, it will not return true
+// unless it is explicitly specified by user in includeTags which is the
+// matchAllows section.
+//
+// It returns true if the tag is specified, or false.
+func (t *TagFilter) Match(tag, author, severity string) (bool, error) {
+	matchedAny := false
+	if len(t.allowedTags) > 0 {
+		_, ok := t.allowedTags[tag]
+		if ok {
+			matchedAny = true
+		}
+	}
+	_, ok := t.block[tag]
+	if ok {
+		if _, allowOk := t.matchAllows[tag]; allowOk {
+			return true, nil
+		}
+		return false, ErrExcluded
+	}
+	if len(t.authors) > 0 {
+		_, ok = t.authors[author]
+		if !ok {
+			return false, nil
+		}
+		matchedAny = true
+	}
+	if len(t.severities) > 0 {
+		_, ok = t.severities[severity]
+		if !ok {
+			return false, nil
+		}
+		matchedAny = true
+	}
+	if len(t.allowedTags) == 0 && len(t.authors) == 0 && len(t.severities) == 0 {
+		return true, nil
+	}
+	return matchedAny, nil
+}
+
+// MatchWithAllowedTags takes an addition list of allowed tags
+// and returns true if the match was successful.
+func (t *TagFilter) MatchWithAllowedTags(allowed []string, tag, author, severity string) (bool, error) {
+	matchedAny := false
+
+	allowedMap := make(map[string]struct{})
+	for _, tag := range allowed {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := allowedMap[val]; !ok {
+				allowedMap[val] = struct{}{}
+			}
+		}
+	}
+	if len(allowedMap) > 0 {
+		_, ok := allowedMap[tag]
+		if ok {
+			matchedAny = true
+		}
+	}
+	_, ok := t.block[tag]
+	if ok && !matchedAny {
+		if _, allowOk := t.matchAllows[tag]; allowOk {
+			return true, nil
+		}
+		return false, ErrExcluded
+	}
+	if len(t.authors) > 0 {
+		_, ok = t.authors[author]
+		if !ok {
+			return false, nil
+		}
+		matchedAny = true
+	}
+	if len(t.severities) > 0 {
+		_, ok = t.severities[severity]
+		if !ok {
+			return false, nil
+		}
+		matchedAny = true
+	}
+	if len(allowedMap) == 0 && len(t.authors) == 0 && len(t.severities) == 0 {
+		return true, nil
+	}
+	return matchedAny, nil
+}
+
+type Config struct {
+	Tags        []string
+	ExcludeTags []string
+	Authors     []string
+	Severities  []string
+	IncludeTags []string
+}
+
+// New returns a tag filter for nuclei tag based execution
+//
+// It takes into account Tags, Severities, Authors, IncludeTags, ExcludeTags.
+func New(config *Config) *TagFilter {
+	filter := &TagFilter{
+		allowedTags: make(map[string]struct{}),
+		authors:     make(map[string]struct{}),
+		severities:  make(map[string]struct{}),
+		block:       make(map[string]struct{}),
+		matchAllows: make(map[string]struct{}),
+	}
+	for _, tag := range config.ExcludeTags {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.block[val]; !ok {
+				filter.block[val] = struct{}{}
+			}
+		}
+	}
+	for _, tag := range config.Severities {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.severities[val]; !ok {
+				filter.severities[val] = struct{}{}
+			}
+		}
+	}
+	for _, tag := range config.Authors {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.authors[val]; !ok {
+				filter.authors[val] = struct{}{}
+			}
+		}
+	}
+	for _, tag := range config.Tags {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.allowedTags[val]; !ok {
+				filter.allowedTags[val] = struct{}{}
+			}
+			delete(filter.block, val)
+		}
+	}
+	for _, tag := range config.IncludeTags {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.matchAllows[val]; !ok {
+				filter.matchAllows[val] = struct{}{}
+			}
+			delete(filter.block, val)
+		}
+	}
+	return filter
+}
+
+func splitCommaTrim(value string) []string {
+	if !strings.Contains(value, ",") {
+		return []string{strings.ToLower(value)}
+	}
+	splitted := strings.Split(value, ",")
+	final := make([]string, len(splitted))
+	for i, value := range splitted {
+		final[i] = strings.ToLower(strings.TrimSpace(value))
+	}
+	return final
+}

v2/pkg/catalog/loader/filter/tag_filter_test.go

@@ -0,0 +1,84 @@
+package filter
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestTagBasedFilter(t *testing.T) {
+	config := &Config{
+		Tags: []string{"cves", "2021", "jira"},
+	}
+	filter := New(config)
+
+	t.Run("true", func(t *testing.T) {
+		matched, _ := filter.Match("jira", "pdteam", "low")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("false", func(t *testing.T) {
+		matched, _ := filter.Match("consul", "pdteam", "low")
+		require.False(t, matched, "could not get correct match")
+	})
+	t.Run("not-match-excludes", func(t *testing.T) {
+		config := &Config{
+			ExcludeTags: []string{"dos"},
+		}
+		filter := New(config)
+		matched, err := filter.Match("dos", "pdteam", "low")
+		require.False(t, matched, "could not get correct match")
+		require.Equal(t, ErrExcluded, err, "could not get correct error")
+	})
+	t.Run("match-includes", func(t *testing.T) {
+		config := &Config{
+			Tags:        []string{"cves", "fuzz"},
+			ExcludeTags: []string{"dos", "fuzz"},
+			IncludeTags: []string{"fuzz"},
+		}
+		filter := New(config)
+		matched, err := filter.Match("fuzz", "pdteam", "low")
+		require.Nil(t, err, "could not get match")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("match-author", func(t *testing.T) {
+		config := &Config{
+			Authors: []string{"pdteam"},
+		}
+		filter := New(config)
+		matched, _ := filter.Match("fuzz", "pdteam", "low")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("match-severity", func(t *testing.T) {
+		config := &Config{
+			Severities: []string{"high"},
+		}
+		filter := New(config)
+		matched, _ := filter.Match("fuzz", "pdteam", "high")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("match-exclude-with-tags", func(t *testing.T) {
+		config := &Config{
+			Tags:        []string{"tag"},
+			ExcludeTags: []string{"another"},
+		}
+		filter := New(config)
+		matched, _ := filter.Match("another", "pdteam", "high")
+		require.False(t, matched, "could not get correct match")
+	})
+	t.Run("match-conditions", func(t *testing.T) {
+		config := &Config{
+			Authors:    []string{"pdteam"},
+			Tags:       []string{"jira"},
+			Severities: []string{"high"},
+		}
+		filter := New(config)
+		matched, _ := filter.Match("jira", "pdteam", "high")
+		require.True(t, matched, "could not get correct match")
+		matched, _ = filter.Match("jira", "pdteam", "low")
+		require.False(t, matched, "could not get correct match")
+		matched, _ = filter.Match("jira", "random", "low")
+		require.False(t, matched, "could not get correct match")
+		matched, _ = filter.Match("consul", "random", "low")
+		require.False(t, matched, "could not get correct match")
+	})
+}

v2/pkg/catalog/loader/load/load.go

@@ -0,0 +1,100 @@
+package load
+
+import (
+	"bytes"
+	"errors"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"gopkg.in/yaml.v2"
+)
+
+// Load loads a template by parsing metadata and running
+// all tag and path based filters on the template.
+func Load(templatePath string, workflow bool, customTags []string, tagFilter *filter.TagFilter) (bool, error) {
+	f, err := os.Open(templatePath)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		return false, err
+	}
+
+	template := make(map[string]interface{})
+	err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
+	if err != nil {
+		return false, err
+	}
+
+	info, ok := template["info"]
+	if !ok {
+		return false, errors.New("no template info field provided")
+	}
+	infoMap, ok := info.(map[interface{}]interface{})
+	if !ok {
+		return false, errors.New("could not get info")
+	}
+
+	if _, nameOk := infoMap["name"]; !nameOk {
+		return false, errors.New("no template name field provided")
+	}
+	author, ok := infoMap["author"]
+	if !ok {
+		return false, errors.New("no template author field provided")
+	}
+	severity, ok := infoMap["severity"]
+	if !ok {
+		severity = ""
+	}
+
+	templateTags, ok := infoMap["tags"]
+	if !ok {
+		templateTags = ""
+	}
+	tagStr := types.ToString(templateTags)
+
+	tags := strings.Split(tagStr, ",")
+	severityStr := strings.ToLower(types.ToString(severity))
+	authors := strings.Split(types.ToString(author), ",")
+
+	matched := false
+
+	_, workflowsFound := template["workflows"]
+	if !workflowsFound && workflow {
+		return false, nil
+	}
+	if workflow {
+		return true, nil
+	}
+	for _, tag := range tags {
+		for _, author := range authors {
+			var match bool
+			var err error
+
+			if len(customTags) == 0 {
+				match, err = tagFilter.Match(strings.ToLower(strings.TrimSpace(tag)), strings.ToLower(strings.TrimSpace(author)), severityStr)
+			} else {
+				match, err = tagFilter.MatchWithAllowedTags(customTags, strings.ToLower(strings.TrimSpace(tag)), strings.ToLower(strings.TrimSpace(author)), severityStr)
+			}
+			if err == filter.ErrExcluded {
+				return false, filter.ErrExcluded
+			}
+			if !matched && match {
+				matched = true
+			}
+		}
+	}
+	if !matched {
+		return false, nil
+	}
+	if workflowsFound && !workflow {
+		return false, nil
+	}
+	return true, nil
+}

v2/pkg/catalog/loader/loader.go

@@ -0,0 +1,202 @@
+package loader
+
+import (
+	"strings"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/load"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+)
+
+// Config contains the configuration options for the loader
+type Config struct {
+	Templates        []string
+	Workflows        []string
+	ExcludeTemplates []string
+	IncludeTemplates []string
+
+	Tags        []string
+	ExcludeTags []string
+	Authors     []string
+	Severities  []string
+	IncludeTags []string
+
+	Catalog            *catalog.Catalog
+	ExecutorOptions    protocols.ExecuterOptions
+	TemplatesDirectory string
+}
+
+// Store is a storage for loaded nuclei templates
+type Store struct {
+	tagFilter      *filter.TagFilter
+	pathFilter     *filter.PathFilter
+	config         *Config
+	finalTemplates []string
+
+	templates []*templates.Template
+	workflows []*templates.Template
+
+	preprocessor templates.Preprocessor
+}
+
+// New creates a new template store based on provided configuration
+func New(config *Config) (*Store, error) {
+	// Create a tag filter based on provided configuration
+	store := &Store{
+		config: config,
+		tagFilter: filter.New(&filter.Config{
+			Tags:        config.Tags,
+			ExcludeTags: config.ExcludeTags,
+			Authors:     config.Authors,
+			Severities:  config.Severities,
+			IncludeTags: config.IncludeTags,
+		}),
+		pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{
+			IncludedTemplates: config.IncludeTemplates,
+			ExcludedTemplates: config.ExcludeTemplates,
+		}, config.Catalog),
+	}
+
+	// Handle a case with no templates or workflows, where we use base directory
+	if len(config.Templates) == 0 && len(config.Workflows) == 0 {
+		config.Templates = append(config.Templates, config.TemplatesDirectory)
+	}
+	store.finalTemplates = append(store.finalTemplates, config.Templates...)
+	return store, nil
+}
+
+// Templates returns all the templates in the store
+func (s *Store) Templates() []*templates.Template {
+	return s.templates
+}
+
+// Workflows returns all the workflows in the store
+func (s *Store) Workflows() []*templates.Template {
+	return s.workflows
+}
+
+// RegisterPreprocessor allows a custom preprocessor to be passed to the store to run against templates
+func (s *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
+	s.preprocessor = preprocessor
+}
+
+// Load loads all the templates from a store, performs filtering and returns
+// the complete compiled templates for a nuclei execution configuration.
+func (s *Store) Load() {
+	s.templates = s.LoadTemplates(s.finalTemplates)
+	s.workflows = s.LoadWorkflows(s.config.Workflows)
+}
+
+// ValidateTemplates takes a list of templates and validates them
+// erroring out on discovering any faulty templates.
+func (s *Store) ValidateTemplates(templatesList, workflowsList []string) bool {
+	includedTemplates := s.config.Catalog.GetTemplatesPath(templatesList)
+	includedWorkflows := s.config.Catalog.GetTemplatesPath(workflowsList)
+	templatesMap := s.pathFilter.Match(includedTemplates)
+	workflowsMap := s.pathFilter.Match(includedWorkflows)
+
+	notErrored := true
+	for k := range templatesMap {
+		_, err := s.loadTemplate(k, false)
+		if err != nil {
+			if strings.Contains(err.Error(), "cannot create template executer") {
+				continue
+			}
+			if err == filter.ErrExcluded {
+				continue
+			}
+			notErrored = false
+			gologger.Error().Msgf("Error occurred loading template %s: %s\n", k, err)
+			continue
+		}
+		_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
+		if err != nil {
+			if strings.Contains(err.Error(), "cannot create template executer") {
+				continue
+			}
+			if err == filter.ErrExcluded {
+				continue
+			}
+			notErrored = false
+			gologger.Error().Msgf("Error occurred parsing template %s: %s\n", k, err)
+		}
+	}
+	for k := range workflowsMap {
+		_, err := s.loadTemplate(k, true)
+		if err != nil {
+			if strings.Contains(err.Error(), "cannot create template executer") {
+				continue
+			}
+			if err == filter.ErrExcluded {
+				continue
+			}
+			notErrored = false
+			gologger.Error().Msgf("Error occurred loading workflow %s: %s\n", k, err)
+		}
+		_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
+		if err != nil {
+			if strings.Contains(err.Error(), "cannot create template executer") {
+				continue
+			}
+			if err == filter.ErrExcluded {
+				continue
+			}
+			notErrored = false
+			gologger.Error().Msgf("Error occurred parsing workflow %s: %s\n", k, err)
+		}
+	}
+	return notErrored
+}
+
+// LoadTemplates takes a list of templates and returns paths for them
+func (s *Store) LoadTemplates(templatesList []string) []*templates.Template {
+	includedTemplates := s.config.Catalog.GetTemplatesPath(templatesList)
+	templatesMap := s.pathFilter.Match(includedTemplates)
+
+	loadedTemplates := make([]*templates.Template, 0, len(templatesMap))
+	for k := range templatesMap {
+		loaded, err := s.loadTemplate(k, false)
+		if err != nil {
+			gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
+		}
+		if loaded {
+			parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
+			if err != nil {
+				gologger.Warning().Msgf("Could not parse template %s: %s\n", k, err)
+			} else if parsed != nil {
+				loadedTemplates = append(loadedTemplates, parsed)
+			}
+		}
+	}
+	return loadedTemplates
+}
+
+// LoadWorkflows takes a list of workflows and returns paths for them
+func (s *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
+	includedWorkflows := s.config.Catalog.GetTemplatesPath(s.config.Workflows)
+	workflowsMap := s.pathFilter.Match(includedWorkflows)
+
+	loadedWorkflows := make([]*templates.Template, 0, len(workflowsMap))
+	for k := range workflowsMap {
+		loaded, err := s.loadTemplate(k, true)
+		if err != nil {
+			gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err)
+		}
+		if loaded {
+			parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
+			if err != nil {
+				gologger.Warning().Msgf("Could not parse workflow %s: %s\n", k, err)
+			} else if parsed != nil {
+				loadedWorkflows = append(loadedWorkflows, parsed)
+			}
+		}
+	}
+	return loadedWorkflows
+}
+
+func (s *Store) loadTemplate(templatePath string, workflow bool) (bool, error) {
+	return load.Load(templatePath, workflow, nil, s.tagFilter)
+}

v2/pkg/catalog/loader/loader_test.go

@@ -0,0 +1,40 @@
+package loader
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadTemplates(t *testing.T) {
+	store, err := New(&Config{
+		Templates: []string{"cves/CVE-2021-21315.yaml"},
+	})
+	require.Nil(t, err, "could not load templates")
+	require.Equal(t, []string{"cves/CVE-2021-21315.yaml"}, store.finalTemplates, "could not get correct templates")
+
+	templatesDirectory := "/test"
+	t.Run("blank", func(t *testing.T) {
+		store, err := New(&Config{
+			TemplatesDirectory: templatesDirectory,
+		})
+		require.Nil(t, err, "could not load templates")
+		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
+	})
+	t.Run("only-tags", func(t *testing.T) {
+		store, err := New(&Config{
+			Tags:               []string{"cves"},
+			TemplatesDirectory: templatesDirectory,
+		})
+		require.Nil(t, err, "could not load templates")
+		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
+	})
+	t.Run("tags-with-path", func(t *testing.T) {
+		store, err := New(&Config{
+			Tags:               []string{"cves"},
+			TemplatesDirectory: templatesDirectory,
+		})
+		require.Nil(t, err, "could not load templates")
+		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
+	})
+}

v2/pkg/catalog/path.go

@@ -0,0 +1,45 @@
+package catalog
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+// ResolvePath resolves the path to an absolute one in various ways.
+//
+// It checks if the filename is an absolute path, looks in the current directory
+// or checking the nuclei templates directory. If a second path is given,
+// it also tries to find paths relative to that second path.
+func (c *Catalog) ResolvePath(templateName, second string) (string, error) {
+	if strings.HasPrefix(templateName, "/") || strings.Contains(templateName, ":\\") {
+		return templateName, nil
+	}
+
+	if second != "" {
+		secondBasePath := path.Join(filepath.Dir(second), templateName)
+		if _, err := os.Stat(secondBasePath); !os.IsNotExist(err) {
+			return secondBasePath, nil
+		}
+	}
+
+	curDirectory, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+
+	templatePath := path.Join(curDirectory, templateName)
+	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
+		return templatePath, nil
+	}
+
+	if c.templatesDirectory != "" {
+		templatePath := path.Join(c.templatesDirectory, templateName)
+		if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
+			return templatePath, nil
+		}
+	}
+	return "", fmt.Errorf("no such path found: %s", templateName)
+}

v2/pkg/executor/dns_utils.go

@@ -1,38 +0,0 @@
-package executor
-
-import (
-	"net/url"
-
-	"github.com/asaskevich/govalidator"
-)
-
-// isURL tests a string to determine if it is a well-structured url or not.
-func isURL(toTest string) bool {
-	_, err := url.ParseRequestURI(toTest)
-	if err != nil {
-		return false
-	}
-
-	u, err := url.Parse(toTest)
-	if err != nil || u.Scheme == "" || u.Host == "" {
-		return false
-	}
-	return true
-}
-
-// extractDomain extracts the domain name of a URL
-func extractDomain(URL string) string {
-	u, err := url.Parse(URL)
-	if err != nil {
-		return ""
-	}
-	hostname := u.Hostname()
-	return hostname
-}
-
-// isDNS tests a string to determine if it is a well-structured dns or not
-// even if it's oneliner, we leave it wrapped in a function call for
-// future improvements
-func isDNS(toTest string) bool {
-	return govalidator.IsDNSName(toTest)
-}

v2/pkg/executor/executer_http.go

@@ -1,282 +0,0 @@
-package executor
-
-import (
-	"bufio"
-	"crypto/tls"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/http/httputil"
-	"net/url"
-	"os"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/retryablehttp-go"
-	"golang.org/x/net/proxy"
-)
-
-// HTTPExecutor is client for performing HTTP requests
-// for a template.
-type HTTPExecutor struct {
-	debug         bool
-	results       uint32
-	jsonOutput    bool
-	httpClient    *retryablehttp.Client
-	template      *templates.Template
-	httpRequest   *requests.HTTPRequest
-	writer        *bufio.Writer
-	outputMutex   *sync.Mutex
-	customHeaders requests.CustomHeaders
-}
-
-// HTTPOptions contains configuration options for the HTTP executor.
-type HTTPOptions struct {
-	Template      *templates.Template
-	HTTPRequest   *requests.HTTPRequest
-	Writer        *bufio.Writer
-	Timeout       int
-	Retries       int
-	ProxyURL      string
-	ProxySocksURL string
-	Debug         bool
-	JSON          bool
-	CustomHeaders requests.CustomHeaders
-}
-
-// NewHTTPExecutor creates a new HTTP executor from a template
-// and a HTTP request query.
-func NewHTTPExecutor(options *HTTPOptions) (*HTTPExecutor, error) {
-	var proxyURL *url.URL
-	var err error
-
-	if options.ProxyURL != "" {
-		proxyURL, err = url.Parse(options.ProxyURL)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	// Create the HTTP Client
-	client := makeHTTPClient(proxyURL, options)
-	client.CheckRetry = retryablehttp.HostSprayRetryPolicy()
-
-	executer := &HTTPExecutor{
-		debug:         options.Debug,
-		jsonOutput:    options.JSON,
-		results:       0,
-		httpClient:    client,
-		template:      options.Template,
-		httpRequest:   options.HTTPRequest,
-		outputMutex:   &sync.Mutex{},
-		writer:        options.Writer,
-		customHeaders: options.CustomHeaders,
-	}
-	return executer, nil
-}
-
-// GotResults returns true if there were any results for the executor
-func (e *HTTPExecutor) GotResults() bool {
-	if atomic.LoadUint32(&e.results) == 0 {
-		return false
-	}
-	return true
-}
-
-// ExecuteHTTP executes the HTTP request on a URL
-func (e *HTTPExecutor) ExecuteHTTP(URL string) error {
-	// Compile each request for the template based on the URL
-	compiledRequest, err := e.httpRequest.MakeHTTPRequest(URL)
-	if err != nil {
-		return errors.Wrap(err, "could not make http request")
-	}
-
-	// Send the request to the target servers
-mainLoop:
-	for compiledRequest := range compiledRequest {
-		if compiledRequest.Error != nil {
-			return errors.Wrap(err, "could not make http request")
-		}
-		e.setCustomHeaders(compiledRequest)
-		req := compiledRequest.Request
-
-		if e.debug {
-			gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID)
-			dumpedRequest, err := httputil.DumpRequest(req.Request, true)
-			if err != nil {
-				return errors.Wrap(err, "could not dump http request")
-			}
-			fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
-		}
-
-		resp, err := e.httpClient.Do(req)
-		if err != nil {
-			if resp != nil {
-				resp.Body.Close()
-			}
-			gologger.Warningf("Could not do request: %s\n", err)
-			continue
-		}
-
-		if e.debug {
-			gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID)
-			dumpedResponse, err := httputil.DumpResponse(resp, true)
-			if err != nil {
-				return errors.Wrap(err, "could not dump http response")
-			}
-			fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
-		}
-
-		data, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			io.Copy(ioutil.Discard, resp.Body)
-			resp.Body.Close()
-			return errors.Wrap(err, "could not read http body")
-		}
-		resp.Body.Close()
-
-		// net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request
-		// so in case we have to manually do it
-		data, err = requests.HandleDecompression(compiledRequest.Request, data)
-		if err != nil {
-			return errors.Wrap(err, "could not decompress http body")
-		}
-
-		// Convert response body from []byte to string with zero copy
-		body := unsafeToString(data)
-
-		var headers string
-		matcherCondition := e.httpRequest.GetMatchersCondition()
-		for _, matcher := range e.httpRequest.Matchers {
-			headers = headersToString(resp.Header)
-			// Check if the matcher matched
-			if !matcher.Match(resp, body, headers) {
-				// If the condition is AND we haven't matched, try next request.
-				if matcherCondition == matchers.ANDCondition {
-					continue mainLoop
-				}
-			} else {
-				// If the matcher has matched, and its an OR
-				// write the first output then move to next matcher.
-				if matcherCondition == matchers.ORCondition && len(e.httpRequest.Extractors) == 0 {
-					e.writeOutputHTTP(compiledRequest, matcher, nil)
-					atomic.CompareAndSwapUint32(&e.results, 0, 1)
-				}
-			}
-		}
-
-		// All matchers have successfully completed so now start with the
-		// next task which is extraction of input from matchers.
-		var extractorResults []string
-		for _, extractor := range e.httpRequest.Extractors {
-			headers = headersToString(resp.Header)
-			for match := range extractor.Extract(body, headers) {
-				extractorResults = append(extractorResults, match)
-			}
-		}
-
-		// Write a final string of output if matcher type is
-		// AND or if we have extractors for the mechanism too.
-		if len(e.httpRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
-			e.writeOutputHTTP(compiledRequest, nil, extractorResults)
-			atomic.CompareAndSwapUint32(&e.results, 0, 1)
-		}
-	}
-
-	gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL)
-
-	return nil
-}
-
-// Close closes the http executor for a template.
-func (e *HTTPExecutor) Close() {
-	e.outputMutex.Lock()
-	e.writer.Flush()
-	e.outputMutex.Unlock()
-}
-
-// makeHTTPClient creates a http client
-func makeHTTPClient(proxyURL *url.URL, options *HTTPOptions) *retryablehttp.Client {
-	retryablehttpOptions := retryablehttp.DefaultOptionsSpraying
-	retryablehttpOptions.RetryWaitMax = 10 * time.Second
-	retryablehttpOptions.RetryMax = options.Retries
-	followRedirects := options.HTTPRequest.Redirects
-	maxRedirects := options.HTTPRequest.MaxRedirects
-
-	transport := &http.Transport{
-		MaxIdleConnsPerHost: -1,
-		TLSClientConfig: &tls.Config{
-			Renegotiation:      tls.RenegotiateOnceAsClient,
-			InsecureSkipVerify: true,
-		},
-		DisableKeepAlives: true,
-	}
-
-	// Attempts to overwrite the dial function with the socks proxied version
-	if options.ProxySocksURL != "" {
-		var proxyAuth *proxy.Auth
-		socksURL, err := url.Parse(options.ProxySocksURL)
-		if err == nil {
-			proxyAuth = &proxy.Auth{}
-			proxyAuth.User = socksURL.User.Username()
-			proxyAuth.Password, _ = socksURL.User.Password()
-		}
-		dialer, err := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct)
-		if err == nil {
-			transport.Dial = dialer.Dial
-		}
-	}
-
-	if proxyURL != nil {
-		transport.Proxy = http.ProxyURL(proxyURL)
-	}
-	return retryablehttp.NewWithHTTPClient(&http.Client{
-		Transport:     transport,
-		Timeout:       time.Duration(options.Timeout) * time.Second,
-		CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects),
-	}, retryablehttpOptions)
-}
-
-type checkRedirectFunc func(_ *http.Request, requests []*http.Request) error
-
-func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirectFunc {
-	return func(_ *http.Request, requests []*http.Request) error {
-		if !followRedirects {
-			return http.ErrUseLastResponse
-		}
-		if maxRedirects == 0 {
-			if len(requests) > 10 {
-				return http.ErrUseLastResponse
-			}
-			return nil
-		}
-		if len(requests) > maxRedirects {
-			return http.ErrUseLastResponse
-		}
-		return nil
-	}
-}
-
-func (e *HTTPExecutor) setCustomHeaders(r *requests.CompiledHTTP) {
-	for _, customHeader := range e.customHeaders {
-		// This should be pre-computed somewhere and done only once
-		tokens := strings.Split(customHeader, ":")
-		// if it's an invalid header skip it
-		if len(tokens) < 2 {
-			continue
-		}
-
-		headerName, headerValue := tokens[0], strings.Join(tokens[1:], "")
-		headerName = strings.TrimSpace(headerName)
-		headerValue = strings.TrimSpace(headerValue)
-		r.Request.Header.Set(headerName, headerValue)
-	}
-}

v2/pkg/executor/executor_dns.go

@@ -1,149 +0,0 @@
-package executor
-
-import (
-	"bufio"
-	"fmt"
-	"os"
-	"sync"
-	"sync/atomic"
-
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	retryabledns "github.com/projectdiscovery/retryabledns"
-)
-
-// DNSExecutor is a client for performing a DNS request
-// for a template.
-type DNSExecutor struct {
-	debug       bool
-	jsonOutput  bool
-	results     uint32
-	dnsClient   *retryabledns.Client
-	template    *templates.Template
-	dnsRequest  *requests.DNSRequest
-	writer      *bufio.Writer
-	outputMutex *sync.Mutex
-}
-
-// DefaultResolvers contains the list of resolvers known to be trusted.
-var DefaultResolvers = []string{
-	"1.1.1.1:53", // Cloudflare
-	"1.0.0.1:53", // Cloudflare
-	"8.8.8.8:53", // Google
-	"8.8.4.4:53", // Google
-}
-
-// DNSOptions contains configuration options for the DNS executor.
-type DNSOptions struct {
-	Debug      bool
-	JSON       bool
-	Template   *templates.Template
-	DNSRequest *requests.DNSRequest
-	Writer     *bufio.Writer
-}
-
-// NewDNSExecutor creates a new DNS executor from a template
-// and a DNS request query.
-func NewDNSExecutor(options *DNSOptions) *DNSExecutor {
-	dnsClient := retryabledns.New(DefaultResolvers, options.DNSRequest.Retries)
-
-	executer := &DNSExecutor{
-		debug:       options.Debug,
-		jsonOutput:  options.JSON,
-		results:     0,
-		dnsClient:   dnsClient,
-		template:    options.Template,
-		dnsRequest:  options.DNSRequest,
-		writer:      options.Writer,
-		outputMutex: &sync.Mutex{},
-	}
-	return executer
-}
-
-// GotResults returns true if there were any results for the executor
-func (e *DNSExecutor) GotResults() bool {
-	if atomic.LoadUint32(&e.results) == 0 {
-		return false
-	}
-	return true
-}
-
-// ExecuteDNS executes the DNS request on a URL
-func (e *DNSExecutor) ExecuteDNS(URL string) error {
-	// Parse the URL and return domain if URL.
-	var domain string
-	if isURL(URL) {
-		domain = extractDomain(URL)
-	} else {
-		domain = URL
-	}
-
-	// Compile each request for the template based on the URL
-	compiledRequest, err := e.dnsRequest.MakeDNSRequest(domain)
-	if err != nil {
-		return errors.Wrap(err, "could not make dns request")
-	}
-
-	if e.debug {
-		gologger.Infof("Dumped DNS request for %s (%s)\n\n", URL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s\n", compiledRequest.String())
-	}
-
-	// Send the request to the target servers
-	resp, err := e.dnsClient.Do(compiledRequest)
-	if err != nil {
-		return errors.Wrap(err, "could not send dns request")
-	}
-
-	gologger.Verbosef("Sent DNS request to %s\n", "dns-request", URL)
-
-	if e.debug {
-		gologger.Infof("Dumped DNS response for %s (%s)\n\n", URL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s\n", resp.String())
-	}
-
-	matcherCondition := e.dnsRequest.GetMatchersCondition()
-	for _, matcher := range e.dnsRequest.Matchers {
-		// Check if the matcher matched
-		if !matcher.MatchDNS(resp) {
-			// If the condition is AND we haven't matched, return.
-			if matcherCondition == matchers.ANDCondition {
-				return nil
-			}
-		} else {
-			// If the matcher has matched, and its an OR
-			// write the first output then move to next matcher.
-			if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 {
-				e.writeOutputDNS(domain, matcher, nil)
-				atomic.CompareAndSwapUint32(&e.results, 0, 1)
-			}
-		}
-	}
-
-	// All matchers have successfully completed so now start with the
-	// next task which is extraction of input from matchers.
-	var extractorResults []string
-	for _, extractor := range e.dnsRequest.Extractors {
-		for match := range extractor.ExtractDNS(resp.String()) {
-			extractorResults = append(extractorResults, match)
-		}
-	}
-
-	// Write a final string of output if matcher type is
-	// AND or if we have extractors for the mechanism too.
-	if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
-		e.writeOutputDNS(domain, nil, extractorResults)
-		atomic.CompareAndSwapUint32(&e.results, 0, 1)
-	}
-	return nil
-}
-
-// Close closes the dns executor for a template.
-func (e *DNSExecutor) Close() {
-	e.outputMutex.Lock()
-	e.writer.Flush()
-	e.outputMutex.Unlock()
-}

v2/pkg/executor/http_utils.go

@@ -1,41 +0,0 @@
-package executor
-
-import (
-	"net/http"
-	"strings"
-	"unsafe"
-)
-
-type jsonOutput struct {
-	Template         string   `json:"template"`
-	Type             string   `json:"type"`
-	Matched          string   `json:"matched"`
-	MatcherName      string   `json:"matcher_name,omitempty"`
-	ExtractedResults []string `json:"extracted_results,omitempty"`
-	Severity         string   `json:"severity"`
-	Author           string   `json:"author"`
-}
-
-// unsafeToString converts byte slice to string with zero allocations
-func unsafeToString(bs []byte) string {
-	return *(*string)(unsafe.Pointer(&bs))
-}
-
-// headersToString converts http headers to string
-func headersToString(headers http.Header) string {
-	builder := &strings.Builder{}
-
-	for header, values := range headers {
-		builder.WriteString(header)
-		builder.WriteString(": ")
-
-		for i, value := range values {
-			builder.WriteString(value)
-			if i != len(values)-1 {
-				builder.WriteRune(',')
-			}
-		}
-		builder.WriteRune('\n')
-	}
-	return builder.String()
-}

v2/pkg/executor/output_dns.go

@@ -1,76 +0,0 @@
-package executor
-
-import (
-	"strings"
-
-	jsoniter "github.com/json-iterator/go"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-)
-
-// writeOutputDNS writes dns output to streams
-func (e *DNSExecutor) writeOutputDNS(domain string, matcher *matchers.Matcher, extractorResults []string) {
-	if e.jsonOutput {
-		output := jsonOutput{
-			Template: e.template.ID,
-			Type:     "dns",
-			Matched:  domain,
-			Severity: e.template.Info.Severity,
-			Author:   e.template.Info.Author,
-		}
-		if matcher != nil && len(matcher.Name) > 0 {
-			output.MatcherName = matcher.Name
-		}
-		if len(extractorResults) > 0 {
-			output.ExtractedResults = extractorResults
-		}
-		data, err := jsoniter.Marshal(output)
-		if err != nil {
-			gologger.Warningf("Could not marshal json output: %s\n", err)
-		}
-
-		gologger.Silentf("%s", string(data))
-
-		if e.writer != nil {
-			e.outputMutex.Lock()
-			e.writer.Write(data)
-			e.writer.WriteRune('\n')
-			e.outputMutex.Unlock()
-		}
-		return
-	}
-
-	builder := &strings.Builder{}
-	builder.WriteRune('[')
-	builder.WriteString(e.template.ID)
-	if matcher != nil && len(matcher.Name) > 0 {
-		builder.WriteString(":")
-		builder.WriteString(matcher.Name)
-	}
-	builder.WriteString("] [dns] ")
-
-	builder.WriteString(domain)
-
-	// If any extractors, write the results
-	if len(extractorResults) > 0 {
-		builder.WriteString(" [")
-		for i, result := range extractorResults {
-			builder.WriteString(result)
-			if i != len(extractorResults)-1 {
-				builder.WriteRune(',')
-			}
-		}
-		builder.WriteString("]")
-	}
-	builder.WriteRune('\n')
-
-	// Write output to screen as well as any output file
-	message := builder.String()
-	gologger.Silentf("%s", message)
-
-	if e.writer != nil {
-		e.outputMutex.Lock()
-		e.writer.WriteString(message)
-		e.outputMutex.Unlock()
-	}
-}

v2/pkg/executor/output_http.go

@@ -1,94 +0,0 @@
-package executor
-
-import (
-	"strings"
-
-	jsoniter "github.com/json-iterator/go"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-)
-
-// writeOutputHTTP writes http output to streams
-func (e *HTTPExecutor) writeOutputHTTP(req *requests.CompiledHTTP, matcher *matchers.Matcher, extractorResults []string) {
-	URL := req.Request.URL.String()
-
-	if e.jsonOutput {
-		output := jsonOutput{
-			Template: e.template.ID,
-			Type:     "http",
-			Matched:  URL,
-			Severity: e.template.Info.Severity,
-			Author:   e.template.Info.Author,
-		}
-		if matcher != nil && len(matcher.Name) > 0 {
-			output.MatcherName = matcher.Name
-		}
-		if len(extractorResults) > 0 {
-			output.ExtractedResults = extractorResults
-		}
-		data, err := jsoniter.Marshal(output)
-		if err != nil {
-			gologger.Warningf("Could not marshal json output: %s\n", err)
-		}
-
-		gologger.Silentf("%s", string(data))
-
-		if e.writer != nil {
-			e.outputMutex.Lock()
-			e.writer.Write(data)
-			e.writer.WriteRune('\n')
-			e.outputMutex.Unlock()
-		}
-		return
-	}
-
-	builder := &strings.Builder{}
-
-	builder.WriteRune('[')
-	builder.WriteString(e.template.ID)
-	if matcher != nil && len(matcher.Name) > 0 {
-		builder.WriteString(":")
-		builder.WriteString(matcher.Name)
-	}
-	builder.WriteString("] [http] ")
-
-	// Escape the URL by replacing all % with %%
-	escapedURL := strings.Replace(URL, "%", "%%", -1)
-	builder.WriteString(escapedURL)
-
-	// If any extractors, write the results
-	if len(extractorResults) > 0 {
-		builder.WriteString(" [")
-		for i, result := range extractorResults {
-			builder.WriteString(result)
-			if i != len(extractorResults)-1 {
-				builder.WriteRune(',')
-			}
-		}
-		builder.WriteString("]")
-	}
-
-	// write meta if any
-	if len(req.Meta) > 0 {
-		builder.WriteString(" [")
-		var metas []string
-		for name, value := range req.Meta {
-			metas = append(metas, name+"="+value.(string))
-		}
-		builder.WriteString(strings.Join(metas, ","))
-		builder.WriteString("]")
-	}
-
-	builder.WriteRune('\n')
-
-	// Write output to screen as well as any output file
-	message := builder.String()
-	gologger.Silentf("%s", message)
-
-	if e.writer != nil {
-		e.outputMutex.Lock()
-		e.writer.WriteString(message)
-		e.outputMutex.Unlock()
-	}
-}

v2/pkg/extractors/extract.go

@@ -1,36 +0,0 @@
-package extractors
-
-// Extract extracts response from the parts of request using a regex
-func (e *Extractor) Extract(body, headers string) map[string]struct{} {
-	// Match the parts as required for regex check
-	if e.part == BodyPart {
-		return e.extractRegex(body)
-	} else if e.part == HeaderPart {
-		return e.extractRegex(headers)
-	} else {
-		matches := e.extractRegex(headers)
-		if len(matches) > 0 {
-			return matches
-		}
-		return e.extractRegex(body)
-	}
-}
-
-// ExtractDNS extracts response from dns message using a regex
-func (e *Extractor) ExtractDNS(msg string) map[string]struct{} {
-	// Match the parts as required for regex check
-	return e.extractRegex(msg)
-}
-
-// extractRegex extracts text from a corpus and returns it
-func (e *Extractor) extractRegex(corpus string) map[string]struct{} {
-	results := make(map[string]struct{})
-
-	for _, regex := range e.regexCompiled {
-		matches := regex.FindAllString(corpus, -1)
-		for _, match := range matches {
-			results[match] = struct{}{}
-		}
-	}
-	return results
-}

v2/pkg/generators/attack.go

@@ -1,20 +0,0 @@
-package generators
-
-// Type is type of attack
-type Type int
-
-const (
-	// Sniper attack - each variable replaced with values at a time
-	Sniper Type = iota + 1
-	// PitchFork attack - Each variable replaced with positional value in multiple wordlists
-	PitchFork
-	// ClusterBomb attack - Generate all possible combinations of values
-	ClusterBomb
-)
-
-// AttackTypes is an table for conversion of attack type from string.
-var AttackTypes = map[string]Type{
-	"sniper":      Sniper,
-	"pitchfork":   PitchFork,
-	"clusterbomb": ClusterBomb,
-}

v2/pkg/generators/clusterbomb.go

@@ -1,50 +0,0 @@
-package generators
-
-// ClusterbombGenerator Attack - Generate all possible combinations from an input map with all values listed
-// as slices of the same size
-func ClusterbombGenerator(payloads map[string][]string) (out chan map[string]interface{}) {
-	out = make(chan map[string]interface{})
-
-	// generator
-	go func() {
-		defer close(out)
-		var order []string
-		var parts [][]string
-		for name, wordlist := range payloads {
-			order = append(order, name)
-			parts = append(parts, wordlist)
-		}
-
-		var n = 1
-		for _, ar := range parts {
-			n *= len(ar)
-		}
-
-		var at = make([]int, len(parts))
-	loop:
-		for {
-			// increment position counters
-			for i := len(parts) - 1; i >= 0; i-- {
-				if at[i] > 0 && at[i] >= len(parts[i]) {
-					if i == 0 || (i == 1 && at[i-1] == len(parts[0])-1) {
-						break loop
-					}
-					at[i] = 0
-					at[i-1]++
-				}
-			}
-			// construct permutation
-			item := make(map[string]interface{})
-			for i, ar := range parts {
-				var p = at[i]
-				if p >= 0 && p < len(ar) {
-					item[order[i]] = ar[p]
-				}
-			}
-			out <- item
-			at[len(parts)-1]++
-		}
-	}()
-
-	return out
-}

v2/pkg/generators/dsl.go

@@ -1,111 +0,0 @@
-package generators
-
-import (
-	"crypto/md5"
-	"crypto/sha1"
-	"crypto/sha256"
-	"encoding/base64"
-	"encoding/hex"
-	"html"
-	"net/url"
-	"regexp"
-	"strings"
-
-	"github.com/Knetic/govaluate"
-)
-
-// HelperFunctions contains the dsl functions
-func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
-	functions = make(map[string]govaluate.ExpressionFunction)
-	// strings
-	functions["len"] = func(args ...interface{}) (interface{}, error) {
-		length := len(args[0].(string))
-		return (float64)(length), nil
-	}
-	functions["toupper"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ToUpper(args[0].(string)), nil
-	}
-	functions["tolower"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ToLower(args[0].(string)), nil
-	}
-	functions["replace"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Replace(args[0].(string), args[1].(string), args[2].(string), -1), nil
-	}
-	functions["trim"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Trim(args[0].(string), args[2].(string)), nil
-	}
-	functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimLeft(args[0].(string), args[1].(string)), nil
-	}
-	functions["trimright"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimRight(args[0].(string), args[1].(string)), nil
-	}
-	functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimSpace(args[0].(string)), nil
-	}
-	functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimPrefix(args[0].(string), args[1].(string)), nil
-	}
-	functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimSuffix(args[0].(string), args[1].(string)), nil
-	}
-	functions["reverse"] = func(args ...interface{}) (interface{}, error) {
-		return reverseString(args[0].(string)), nil
-	}
-	// encoding
-	functions["base64"] = func(args ...interface{}) (interface{}, error) {
-		sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
-		return sEnc, nil
-	}
-	functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
-		sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
-		return sEnc, nil
-	}
-	functions["url_encode"] = func(args ...interface{}) (interface{}, error) {
-		return url.PathEscape(args[0].(string)), nil
-	}
-	functions["url_decode"] = func(args ...interface{}) (interface{}, error) {
-		return url.PathUnescape(args[0].(string))
-	}
-	functions["hex_encode"] = func(args ...interface{}) (interface{}, error) {
-		return hex.EncodeToString([]byte(args[0].(string))), nil
-	}
-	functions["hex_decode"] = func(args ...interface{}) (interface{}, error) {
-		hx, _ := hex.DecodeString(args[0].(string))
-		return string(hx), nil
-	}
-	functions["html_escape"] = func(args ...interface{}) (interface{}, error) {
-		return html.EscapeString(args[0].(string)), nil
-	}
-	functions["html_unescape"] = func(args ...interface{}) (interface{}, error) {
-		return html.UnescapeString(args[0].(string)), nil
-	}
-	// hashing
-	functions["md5"] = func(args ...interface{}) (interface{}, error) {
-		hash := md5.Sum([]byte(args[0].(string)))
-		return hex.EncodeToString(hash[:]), nil
-	}
-	functions["sha256"] = func(args ...interface{}) (interface{}, error) {
-		h := sha256.New()
-		h.Write([]byte(args[0].(string)))
-		return hex.EncodeToString(h.Sum(nil)), nil
-	}
-	functions["sha1"] = func(args ...interface{}) (interface{}, error) {
-		h := sha1.New()
-		h.Write([]byte(args[0].(string)))
-		return hex.EncodeToString(h.Sum(nil)), nil
-	}
-	// search
-	functions["contains"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Contains(args[0].(string), args[1].(string)), nil
-	}
-	functions["regex"] = func(args ...interface{}) (interface{}, error) {
-		compiled, err := regexp.Compile(args[0].(string))
-		if err != nil {
-			return nil, err
-		}
-		return compiled.MatchString(args[1].(string)), nil
-	}
-
-	return
-}