Merge pull request #886 from projectdiscovery/dev

2.4.2 Bugfix Release

.github/dependabot.yml

@@ -0,0 +1,37 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    target-branch: "dev"
+    commit-message:
+      prefix: "chore"
+      include: "scope"
+
+  # Maintain dependencies for go modules
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    target-branch: "dev"
+    commit-message:
+      prefix: "chore"
+      include: "scope"
+
+  # Maintain dependencies for docker
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    target-branch: "dev"
+    commit-message:
+      prefix: "chore"
+      include: "scope"

.github/workflows/build-test.yml

@@ -0,0 +1,31 @@
+name: 🔨 Build Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+
+jobs:  
+  build:
+    name: Test Builds
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.15
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Test
+        run: go test ./...
+        working-directory: v2/
+
+      - name: Integration Tests
+        run: bash run.sh
+        working-directory: integration_tests/
+
+      - name: Build
+        run: go build .
+        working-directory: v2/cmd/nuclei/

.github/workflows/build.yaml

@@ -1,50 +0,0 @@
-name: Build
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-
-jobs:
-  lint:
-    name: golangci-lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v1
-        with:
-          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.31
-          args: --timeout 5m
-          working-directory: v2/
-
-          # Optional: working directory, useful for monorepos
-          # working-directory: somedir
-
-          # Optional: golangci-lint command line arguments.
-          # args: --issues-exit-code=0
-
-          # Optional: show only new issues if it's a pull request. The default value is `false`.
-          # only-new-issues: true
-
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.14
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Test
-        run: go test .
-        working-directory: v2/cmd/nuclei/
-
-      - name: Build
-        run: go build .
-        working-directory: v2/cmd/nuclei/

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,38 @@
+name: 🚨 CodeQL Analysis
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - dev
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/dockerhub-push.yml

@@ -1,17 +1,34 @@
-# dockerhub-push pushes docker build to dockerhub automatically
-# on the creation of a new release
-name: Publish to Dockerhub on creation of a new release
-on: 
+name: 🌥 Docker Push
+
+on:
   release:
     types: [published]
+  workflow_dispatch:
+
 jobs:
-  build:
+  docker:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@master
-    - name: Publish to Dockerhub Registry
-      uses: elgohr/Publish-Docker-Github-Action@master
-      with:
-        name: projectdiscovery/nuclei
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64,linux/arm
+          push: true
+          tags: projectdiscovery/nuclei:latest

.github/workflows/lint-test.yml

@@ -0,0 +1,19 @@
+name: 🙏🏻 Lint Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  lint:
+    name: Lint Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          args: --timeout 5m
+          working-directory: v2/

.github/workflows/release-binary.yml

@@ -1,8 +1,9 @@
-name: Release
+name: 🎉 Release Binary
 on:
   create:
     tags:
       - v*
+  workflow_dispatch:
 
 jobs: 
   release: 
@@ -17,7 +18,7 @@ jobs:
         name: "Set up Go"
         uses: actions/setup-go@v2
         with: 
-          go-version: 1.14
+          go-version: 1.16
       - 
         env: 
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
@@ -25,4 +26,5 @@ jobs:
         uses: goreleaser/goreleaser-action@v2
         with: 
           args: "release --rm-dist"
-          version: latest
+          version: latest
+          workdir: v2/

.gitignore

@@ -1,3 +1,10 @@
 cmd/nuclei/nuclei*
 v2/cmd/nuclei/nuclei
 .idea
+integration_tests/integration-test
+integration_tests/nuclei
+v2/cmd/integration-test/integration-test
+bin
+v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class
+v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class
+v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser

.golangci.yml

@@ -6,9 +6,9 @@ linters-settings:
   # funlen:
   #   lines: 100
   #   statements: 50
-  goconst:
-    min-len: 2
-    min-occurrences: 2
+  #goconst:
+  #  min-len: 2
+  #  min-occurrences: 2
   gocritic:
     enabled-tags:
       - diagnostic
@@ -56,18 +56,17 @@ linters:
   # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
   disable-all: true
   enable:
-    - bodyclose
+    #- bodyclose
     - deadcode
     - dogsled
     - errcheck
     - exhaustive
     - gochecknoinits
-    - goconst
+    #- goconst
     - gocritic
     - gofmt
     - goimports
-    - golint
-    - gomnd
+    #- gomnd
     - goprintffuncname
     - gosimple
     - govet
@@ -89,6 +88,7 @@ linters:
     - unused
     - varcheck
     - whitespace
+    - revive
 
   # don't enable:
   # - depguard
@@ -105,11 +105,4 @@ linters:
   # - nestif
   # - prealloc
   # - testpackage
-  # - wsl
-
-# golangci.com configuration
-# https://github.com/golangci/golangci/wiki/Configuration
-service:
-  golangci-lint-version: 1.31.x # use the fixed version to not introduce new linters unexpectedly
-  prepare:
-    - echo "here I can run custom commands, but no preparation needed for this repo"
+  # - wsl

.goreleaser.yml

@@ -1,21 +0,0 @@
-builds:
-    - binary: nuclei
-      main: v2/cmd/nuclei/main.go
-      goos:
-        - linux
-        - windows
-        - darwin
-      goarch:
-        - amd64
-        - 386
-        - arm
-        - arm64
-  
-archives:
-    - id: tgz
-      format: tar.gz
-      replacements:
-          darwin: macOS
-      format_overrides:
-          - goos: windows
-            format: zip

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at contact@projectdiscovery.io. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

Dockerfile

@@ -1,19 +1,7 @@
-FROM golang:alpine as builder
+FROM golang:1.16.6-alpine as build-env
+RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
 
-RUN mkdir -p /app
-WORKDIR /app
-COPY ./v2/go.mod .
-RUN go mod download
-
-COPY . .
-RUN cd ./v2/cmd/nuclei && go build -o nuclei .
-
-FROM alpine
-
-RUN mkdir /app
-RUN adduser --home /app --shell /bin/sh --disabled-password appuser
-COPY --from=builder /app/v2/cmd/nuclei/nuclei /app
-USER appuser
-
-WORKDIR /app
-ENTRYPOINT ["/app/nuclei"]
+FROM alpine:latest
+RUN apk add --no-cache bind-tools ca-certificates
+COPY --from=build-env /go/bin/nuclei /usr/local/bin/nuclei
+ENTRYPOINT ["nuclei"]

LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Exposed Atoms
+Copyright (c) 2021 ProjectDiscovery, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,58 +1,70 @@
 <h1 align="center">
-  <img src="static/nuclei-logo.png" alt="nuclei" width="200px"></a>
   <br>
+  <a href="https://nuclei.projectdiscovery.io"><img src="static/nuclei-logo.png" width="200px" alt="Nuclei"></a>
 </h1>
 
-[![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT)
-[![Go Report Card](https://goreportcard.com/badge/github.com/projectdiscovery/nuclei)](https://goreportcard.com/report/github.com/projectdiscovery/nuclei)
-[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectdiscovery/nuclei/issues)
-[![GitHub Release](https://img.shields.io/github/release/projectdiscovery/nuclei)](https://github.com/projectdiscovery/nuclei/releases)
-[![Follow on Twitter](https://img.shields.io/twitter/follow/pdnuclei.svg?logo=twitter)](https://twitter.com/pdnuclei)
-[![Docker Images](https://img.shields.io/docker/pulls/projectdiscovery/nuclei.svg)](https://hub.docker.com/r/projectdiscovery/nuclei)
-[![Chat on Discord](https://img.shields.io/discord/695645237418131507.svg?logo=discord)](https://discord.gg/KECAGdH)
+<h4 align="center">Fast and customisable vulnerability scanner based on simple YAML based DSL.</h4>
+
 
 <p align="center">
-<a href="https://nuclei.projectdiscovery.io/templating-guide/" target="_blank"><img src="static/read-the-docs-button.png" height="42px"/></center></a>  <a href="https://github.com/projectdiscovery/nuclei-templates" target="_blank"><img src="static/download-templates-button.png" height="42px"/></a>
+<a href="https://goreportcard.com/report/github.com/projectdiscovery/nuclei"><img src="https://goreportcard.com/badge/github.com/projectdiscovery/nuclei"></a>
+<a href="https://github.com/projectdiscovery/nuclei/issues"><img src="https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat"></a>
+<a href="https://github.com/projectdiscovery/nuclei/releases"><img src="https://img.shields.io/github/release/projectdiscovery/nuclei"></a>
+<a href="https://twitter.com/pdnuclei"><img src="https://img.shields.io/twitter/follow/pdnuclei.svg?logo=twitter"></a>
+<a href="https://discord.gg/projectdiscovery"><img src="https://img.shields.io/discord/695645237418131507.svg?logo=discord"></a>
+<a href="https://github.com/projectdiscovery/nuclei/actions/workflows/build-test.yml"><img src="https://github.com/projectdiscovery/nuclei/actions/workflows/build-test.yml/badge.svg?branch=master"></a>
+</p>
+      
+<p align="center">
+  <a href="#how-it-works">How</a> •
+  <a href="#install-nuclei">Install</a> •
+  <a href="#for-security-engineers">For Security Engineers</a> •
+  <a href="#for-developers-and-organisations">For Developers</a> •
+  <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">Documentation</a> •
+  <a href="#credits">Credits</a> •
+  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">FAQs</a> •
+  <a href="https://discord.gg/projectdiscovery">Join Discord</a>
 </p>
 
+---
 
-Nuclei is a fast tool for configurable targeted scanning based on templates offering massive extensibility and ease of use.
+Nuclei is used to send requests across targets based on a template leading to zero false positives and providing fast scanning on large number of hosts. Nuclei offers scanning for a variety of protocols including TCP, DNS, HTTP, File, etc. With powerful and flexible templating, all kinds of security checks can be modelled with Nuclei.
 
-Nuclei is used to send requests across targets based on a template leading to zero false positives and providing effective scanning for known paths. Main use cases for nuclei are during initial reconnaissance phase to quickly check for low hanging fruits or CVEs across targets that are known and easily detectable. It uses [retryablehttp-go library](https://github.com/projectdiscovery/retryablehttp-go) designed to handle various errors and retries in case of blocking by WAFs, this is also one of our core modules from custom-queries.
-
-We have also [open-sourced a template repository](https://github.com/projectdiscovery/nuclei-templates) to maintain various type of templates, we hope that you will contribute there too. Templates are provided in hopes that these will be useful and will allow everyone to build their own templates for the scanner. Checkout the templating guide at [**nuclei.projectdiscovery.io**](https://nuclei.projectdiscovery.io/templating-guide/) for a primer on nuclei templates.
-
-## Resources
-
--   [Features](#features)
--   [Usage](#usage)
--   [Installation Instructions](#installation-instructions)
-    -   [From Binary](#from-binary)
-    -   [From Source](#from-source)
-    -   [From Github](#from-github)
--   [Nuclei templates](#nuclei-templates)
--   [Running nuclei](#running-nuclei)
-    -   [Running with a single template.](#running-with-single-template)
-    -   [Running with multiple templates.](#running-with-multiple-templates)
-    -   [Running with subfinder](#running-with-subfinder)
-    -   [Running in Docker](#running-in-docker-container)
--   [Template exclusion](#template-exclusion)
--   [Thanks](#thanks)
+We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-templates) that houses various type of vulnerability templates contributed by **more than 100** security researchers and engineers. It is preloaded with ready to use templates using `-update-templates` flag.
 
 
-## Features
 
-<h1 align="left">
-  <img src="static/nuclei-run.png" alt="nuclei" width="700px"></a>
-  <br>
-</h1>
+## How it works
 
--   Simple and modular code base making it easy to contribute.
--   Fast And fully configurable using a template based engine.
--   Handles edge cases doing retries, backoffs etc for handling WAFs.
--   Smart matching functionality for zero false positive scanning.
 
-## Usage
+<h3 align="center">
+  <img src="static/nuclei-flow.jpg" alt="nuclei-flow" width="700px"></a>
+</h3>
+
+
+# Install Nuclei
+
+```sh
+GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+```
+
+**More installation [methods can be found here](https://nuclei.projectdiscovery.io/nuclei/get-started/).**
+
+<table>
+<tr>
+<td>  
+
+### Nuclei Templates
+
+Nuclei has had built-in support for automatic update/download templates since version [v2.4.0](https://github.com/projectdiscovery/nuclei/releases/tag/v2.4.0). [**Nuclei-Templates**](https://github.com/projectdiscovery/nuclei-templates) project provides a community-contributed list of ready-to-use templates that is constantly updated.
+
+You may still use the `update-templates` flag to update the nuclei templates at any time; automatic updates happen every 24 hours. You can write your own checks for your individual workflow and needs following Nuclei's [templating guide](https://nuclei.projectdiscovery.io/templating-guide/).
+
+</td>
+</tr>
+</table>
+
+### Usage
 
 ```sh
 nuclei -h
@@ -60,189 +72,215 @@ nuclei -h
 
 This will display help for the tool. Here are all the switches it supports.
 
-|        Flag       |                      Description                      |                     Example                     |
-| :---------------: | :---------------------------------------------------: | :---------------------------------------------: |
-|         -c        |       Number of concurrent requests (default 10)      |                  nuclei -c 100                  |
-|         -l        |             List of urls to run templates             |                nuclei -l urls.txt               |
-|      -target      |             Target to scan using templates            |        nuclei -target hxxps://example.com       |
-|         -t        |    Templates input file/files to check across hosts   |             nuclei -t git-core.yaml             |
-|         -t        |    Templates input file/files to check across hosts   |         nuclei -t nuclei-templates/cves/        |
-|        -nC        |               Don't Use colors in output              |                    nuclei -nC                   |
-|       -json       |         Prints and write output in json format        |                   nuclei -json                  |
-|   -json-requests  |  Write requests/responses for matches in JSON output  |           nuclei -json -json-requests           |
-|         -o        |         File to save output result (optional)         |               nuclei -o output.txt              |
-|       -pbar       |           Enable the progress bar (optional)          |                   nuclei -pbar                  |
-|      -silent      |           Show only found results in output           |                  nuclei -silent                 |
-|                   |             (except when using with pbar)             |                                                 |
-|      -retries     | Number of times to retry a failed request (default 1) |                nuclei -retries 1                |
-|      -timeout     |       Seconds to wait before timeout (default 5)      |                nuclei -timeout 5                |
-|      -rl          |       Rate-Limit of requests per specified target     |                nuclei -rl 100                   |
-|      -severity    |Run templates based on severity                        |                nuclei -severity critical, low                |
-|      -exclude     |Template input dir/file/files to exclude               |                nuclei -exclude panels -exclude tokens           |
-|       -debug      |         Allow debugging of request/responses.         |                  nuclei -debug                  |
-| -update-templates |         Download and updates nuclei templates         |             nuclei -update-templates            |
-| -update-directory |    Directory for storing nuclei-templates(optional)   |        nuclei -update-directory templates       |
-|        -tl        |                List available templates               |                    nuclei -tl                   |
-|         -v        |       Shows verbose output of all sent requests       |                    nuclei -v                    |
-|      -version     |                 Show version of nuclei                |                 nuclei -version                 |
-|     -proxy-url    |                       Proxy URL                       |     nuclei -proxy-url hxxp://127.0.0.1:8080     |
-|  -proxy-socks-url |                    Socks proxy  URL                   | nuclei -proxy-socks-url socks5://127.0.0.1:8080 |
-|         -H        |                     Custom Header                     |         nuclei -H "x-bug-bounty: hacker"        |
 
-## Installation Instructions
+```yaml
+Nuclei is a fast, template based vulnerability scanner focusing
+on extensive configurability, massive extensibility and ease of use.
 
-### From Binary
+Usage:
+  ./nuclei [flags]
 
-The installation is easy. You can download the pre-built binaries for your platform from the [Releases](https://github.com/projectdiscovery/nuclei/releases/) page. Extract them using tar, move it to your `$PATH`and you're ready to go.
+Flags:
+TARGET:
+   -u, -target string  target URL/host to scan
+   -l, -list string    path to file containing a list of target URLs/hosts to scan (one per line)
+
+TEMPLATES:
+   -tl                      list all available templates
+   -t, -templates string[]  template or template directory paths to include in the scan
+   -w, -workflows string[]  list of workflows to run
+   -nt, -new-templates      run newly added templates only
+   -validate                validate the passed templates to nuclei
+
+FILTERING:
+   -tags string[]                         execute a subset of templates that contain the provided tags
+   -include-tags string[]                 tags from the default deny list that permit executing more intrusive templates
+   -etags, -exclude-tags string[]         exclude templates with the provided tags
+   -include-templates string[]            templates to be executed even if they are excluded either by default or configuration
+   -exclude-templates, -exclude string[]  template or template directory paths to exclude
+   -severity, -impact string[]            execute templates that match the provided severities only
+   -author string[]                       execute templates that are (co-)created by the specified authors
+
+OUTPUT:
+   -o, -output string            output file to write found issues/vulnerabilities
+   -silent                       display findings only
+   -v, -verbose                  show verbose output
+   -vv                           display extra verbose information
+   -nc, -no-color                disable output content coloring (ANSI escape codes)
+   -json                         write output in JSONL(ines) format
+   -irr, -include-rr             include request/response pairs in the JSONL output (for findings only)
+   -nm, -no-meta                 don't display match metadata
+   -rdb, -report-db string       local nuclei reporting database (always use this to persist report data)
+   -me, -markdown-export string  directory to export results in markdown format
+   -se, -sarif-export string     file to export results in SARIF format
+
+CONFIGURATIONS:
+   -config string              path to the nuclei configuration file
+   -rc, -report-config string  nuclei reporting module configuration file
+   -H, -header string[]        custom headers in header:value format
+   -r, -resolvers string       file containing resolver list for nuclei
+   -system-resolvers           use system DNS resolving as error fallback
+   -passive                    enable passive HTTP response processing mode
+
+INTERACTSH:
+   -no-interactsh                     do not use interactsh server for blind interaction polling
+   -interactsh-url string             self-hosted Interactsh Server URL (default "https://interact.sh")
+   -interactions-cache-size int       number of requests to keep in the interactions cache (default 5000)
+   -interactions-eviction int         number of seconds to wait before evicting requests from cache (default 60)
+   -interactions-poll-duration int    number of seconds to wait before each interaction poll request (default 5)
+   -interactions-cooldown-period int  extra time for interaction polling before exiting (default 5)
+
+RATE-LIMIT:
+   -rl, -rate-limit int  maximum number of requests to send per second (default 150)
+   -bs, -bulk-size int   maximum number of hosts to be analyzed in parallel per template (default 25)
+   -c, -concurrency int  maximum number of templates to be executed in parallel (default 10)
+
+OPTIMIZATIONS:
+   -timeout int               time to wait in seconds before timeout (default 5)
+   -retries int               number of times to retry a failed request (default 1)
+   -project                   use a project folder to avoid sending same request multiple times
+   -project-path string       set a specific project path (default "/var/folders/ml/m31ysb5x73l1s3kjlyn5g4180000gn/T/")
+   -spm, -stop-at-first-path  stop processing HTTP requests after the first match (may break template/workflow logic)
+
+HEADLESS:
+   -headless          enable templates that require headless browser support
+   -page-timeout int  seconds to wait for each page in headless mode (default 20)
+   -show-browser      show the browser on the screen when running templates with headless mode
+
+DEBUG:
+   -debug                     show all requests and responses
+   -debug-req                 show all sent requests
+   -debug-resp                show all received responses
+   -proxy, -proxy-url string  URL of the HTTP proxy server
+   -proxy-socks-url string    URL of the SOCKS proxy server
+   -trace-log string          file to write sent requests trace log
+   -version                   show nuclei version
+   -tv, -templates-version    shows the version of the installed nuclei-templates
+
+UPDATE:
+   -update                        update nuclei to the latest released version
+   -ut, -update-templates         update the community templates to latest released version
+   -ud, -update-directory string  overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates")
+
+STATISTICS:
+   -stats                    display statistics about the running scan
+   -stats-json               write statistics data to an output file in JSONL(ines) format
+   -si, -stats-interval int  number of seconds to wait between showing a statistics update (default 5)
+   -metrics                  expose nuclei metrics on a port
+   -metrics-port int         port to expose nuclei metrics on (default 9092)
+```
+
+### Running Nuclei
+
+Scanning target domain with [community-curated](https://github.com/projectdiscovery/nuclei-templates) nuclei templates.
 
 ```sh
-Download latest binary from https://github.com/projectdiscovery/nuclei/releases
-
-▶ tar -xzvf nuclei-linux-amd64.tar.gz
-▶ mv nuclei /usr/local/bin/
-▶ nuclei -version
+nuclei -u https://example.com
 ```
 
-### From Source
-
-nuclei requires **go1.14+** to install successfully. Run the following command to get the repo -
+Scanning target URLs with [community-curated](https://github.com/projectdiscovery/nuclei-templates) nuclei templates.
 
 ```sh
-▶ GO111MODULE=on go get -u -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+nuclei -list urls.txt
 ```
 
-### From Github
+Example of `urls.txt`:
 
-```sh
-▶ git clone https://github.com/projectdiscovery/nuclei.git; cd nuclei/v2/cmd/nuclei/; go build; mv nuclei /usr/local/bin/; nuclei -version
+```yaml
+http://example.com
+http://app.example.com
+http://test.example.com
+http://uat.example.com
 ```
 
-## Nuclei templates
+**More detailed examples of running nuclei can be found [here](https://nuclei.projectdiscovery.io/nuclei/get-started/#running-nuclei).**
 
-You can download or update the nuclei templates using `update-templates` flag.
+# For Security Engineers
 
-```sh
-▶ nuclei -update-templates
-```
+Nuclei offers great number of features that are helpful for security engineers to customise workflow in their organisation. With the varieties of scan capabilities (like DNS, HTTP, TCP), security engineers can easily create their suite of custom checks with Nuclei.
 
-or download it from [nuclei templates](https://github.com/projectdiscovery/nuclei-templates) Github project.
+- Varieties of protocols supported: TCP, DNS, HTTP, File, etc
+- Achieve complex vulnerability steps with workflows and [dynamic requests.](https://blog.projectdiscovery.io/nuclei-unleashed-quickly-write-complex-exploits/)
+- Easy to integrate into CI/CD, designed to be easily integrated into regression cycle to actively check the fix and re-appearance of vulnerability. 
 
-```sh
-▶ git clone https://github.com/projectdiscovery/nuclei-templates.git
-```
+<h1 align="left">
+  <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/"><img src="static/learn-more-button.png" width="170px" alt="Learn More"></a>
+</h1>
 
-**Please refer to nuclei [templating guide](https://nuclei.projectdiscovery.io/templating-guide/) to writing your own custom templates.**
+<table>
+<tr>
+<td>  
 
-## Running nuclei
+**For bugbounty hunters:**
 
-### Running with single template.
+Nuclei allows you to customise your testing approach with your own suite of checks and easily run across your bug bounty programs. Moreover, Nuclei can be easily integrated into any continuous scanning workflow.
 
-This will run `git-core.yaml` template against all the hosts in `urls.txt` and returns the matched results.
+- Designed to be easily integrated into other tool workflow.
+- Can process thousands of hosts in few minutes.
+- Easily automate your custom testing approach with our simple YAML DSL.
 
-```sh
-▶ nuclei -l urls.txt -t files/git-core.yaml -o results.txt
-```
+Please check our other open-source projects that might fit into your bug bounty workflow: [github.com/projectdiscovery](http://github.com/projectdiscovery), we also host daily [refresh of DNS data at Chaos](http://chaos.projectdiscovery.io).
 
-You can also pass the list of urls at standard input (STDIN). This allows for easy integration in automation pipelines.
+</td>
+</tr>
+</table>
 
-```sh
-▶ cat urls.txt | nuclei -t files/git-core.yaml -o results.txt
-```
+<table>
+<tr>
+<td>
+  
+**For pentesters:**
 
-💡 Nuclei accepts list of URLs as input, for example here is how `urls.txt` looks like:- 
+Nuclei immensely improve how you approach security assessment by augmenting the manual repetitve processes. Consultancies are already converting their manual assessment steps with Nuclei, it allows them to run set of their custom assessment approach across thousands of hosts in an automated manner. 
 
-```
-https://test.some-site.com
-http://vuls-testing.com
-https://test.com
-```
-### Running with multiple templates.
+Pen-testers get the full power of our public templates and customization capabilities to speed-up their assessment process, and specifically with the regression cycle where you can easily verify the fix.
 
-This will run the tool against all the urls in `urls.txt` with all the templates in the `cves` and `files` directory and returns the matched results.
+- Easily create your compliance, standards suite (e.g. OWASP Top 10) checklist.
+- With capabilities like [fuzz](https://nuclei.projectdiscovery.io/templating-guide/#advance-fuzzing) and [workflows](https://nuclei.projectdiscovery.io/templating-guide/#workflows), complex manual steps and repetitive assessment can be easily automated with Nuclei.
+- Easy to re-test vulnerability-fix by just re-running the template.
 
-```sh
-▶ nuclei -l urls.txt -t cves/ -t files/ -o results.txt
-```
-
-### Running with subfinder.
-
-```sh
-▶ subfinder -d hackerone.com -silent | httpx -silent | nuclei -t cves/ -o results.txt
-```
-
-### Running in Docker container
-
-You can use the [nuclei dockerhub image](https://hub.docker.com/r/projectdiscovery/nuclei). Simply run -
-
-```sh
-▶ docker pull projectdiscovery/nuclei
-```
-
-After downloading or building the container, run the following:
-
-```sh
-▶ docker run -it projectdiscovery/nuclei
-```
-
-For example, this will run the tool against all the hosts in `urls.txt` and output the results to your host file system:
-
-```sh
-▶ cat urls.txt | docker run -v /path/to/nuclei-templates:/app/nuclei-templates -v /path/to/nuclei/config:/app/.nuclei-config.json -i projectdiscovery/nuclei -t ./files/git-config.yaml > results.txt
-```
-
-Remember to change `/path-to-nuclei-templates` to the real path on your host file system.
-
-### Template Exclusion
-
-[Nuclei-templates](https://github.com/projectdiscovery/nuclei-templates) includes multiple checks including many that are useful for attack surface mapping and not necessarily a security issue, in cases where you only looking to scan few specific templates or directory, here are few options / flags to filter or exclude them from running. 
-
-#### Running templates based on severity
-
-You can run the templates based on the specific severity of the template, single and multiple severity can be used for scan. 
-
-```sh
-nuclei -l urls.txt -t cves/ -severity critical, medium
-```
-
-The above example will run all the templates under `cves` directory with `critical` and `medium` severity. 
-
-```sh
-nuclei -l urls.txt -t panels/ -t technologies -severity info
-```
-
-The above example will run all the templates under `panels` and `technologies` directory with **severity** marked as `info`
-
-#### Running templates with exclusion
-
-We do not suggest running all the nuclei-templates directory at once, in case of doing so, one can make use of `exclude` flag to exclude specific directory or templates to ignore from scanning. 
-
-```sh
-nuclei -l urls.txt -t nuclei-templates -exclude panels/ -exclude technologies -exclude files/wp-xmlrpc.yaml
-```
-
-Note:- both directory and specific templates case be excluded from scan as shared in the above example.
-
-#### Using `.nuclei-ignore` file for template exclusion
-
-Since release of nuclei [v2.1.1](https://github.com/projectdiscovery/nuclei/releases/tag/v2.1.1), we have added support of `.nuclei-ignore` file that works along with `update-templates` flag of nuclei, in **.nuclei-ignore** file, you can define all the template directory or template path that you wanted to exclude from all the nuclei scans, to start using this feature, make sure you installed nuclei templates using `nuclei -update-templates` flag, now you can add/update/remove templates in the file that you wanted to exclude from running. 
-
-```
-nano ~/nuclei-templates/.nuclei-ignore
-```
-
-Default **nuclei-ignore** list can be accessed from [here](https://github.com/projectdiscovery/nuclei-templates/blob/master/.nuclei-ignore), in case you don't want to exclude anything, simply remove the `.nuclei-ignore` file.
-
-* * *
-
-# 📋 Notes
-- Progress bar is experimental feature, might not work in few cases. 
-- Progress bar doesn't work with workflows, numbers are not accurate due to conditional execution.
+</td>
+</tr>
+</table>
 
 
-## Thanks
+# For Developers and Organisations
 
-nuclei is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)** file for more details.
+Nuclei is built with simplicity in mind, with the community backed templates by hundreds of security researchers, it allows you to stay updated with latest security threats using continuous Nuclei scanning on the hosts. It is designed to be easily integrated into regression tests cycle, to verify the fixes and  eliminate vulnerabilities from occuring in future.
 
-Do also check out these similar awesome projects that may fit in your workflow:
+- **CI/CD:** Engineers are already utilising Nuclei within their CI/CD pipeline, it allows them to constantly monitor their staging and production environments with customised templates.
+- **Continuous Regression Cycle:** With Nuclei, you can create your custom template on every new identified vulnerability and put into Nuclei engine to eliminate in the continuous regression cycle.
 
-[Burp Suite](https://portswigger.net/burp), [FFuF](https://github.com/ffuf/ffuf), [Jaeles](https://github.com/jaeles-project/jaeles), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [ChopChop](https://github.com/michelin/ChopChop)
+We have [a discussion thread around this](https://github.com/projectdiscovery/nuclei-templates/discussions/693), there are already some bug bounty programs giving incentives to hackers on writing nuclei templates with every submission, that helps them to eliminate the vulnerability across all their assets, as well as to eliminate future risk in reappearing on productions. If you're interested in implementing it in your organisation, feel free to [reach out to us](mailto:[email protected]). We will be more than happy to help you in the getting started process, or you can also post into the [discussion thread for any help](https://github.com/projectdiscovery/nuclei-templates/discussions/693).
+
+<h3 align="center">
+  <img src="static/regression-with-nuclei.jpg" alt="regression-cycle-with-nuclei" width="1100px"></a>
+</h3>
+
+<h1 align="left">
+  <a href="https://github.com/projectdiscovery/nuclei-action"><img src="static/learn-more-button.png" width="170px" alt="Learn More"></a>
+</h1>
+
+### Resources
+
+- [Community Powered Scanning with Nuclei](https://blog.projectdiscovery.io/community-powered-scanning-with-nuclei/)
+- [Nuclei Unleashed - Quickly write complex exploits](https://blog.projectdiscovery.io/nuclei-unleashed-quickly-write-complex-exploits/)
+- [Nuclei - Fuzz all the things](https://blog.projectdiscovery.io/nuclei-fuzz-all-the-things/)
+- [Nuclei + Interactsh Integration for Automating OOB Testing](https://blog.projectdiscovery.io/nuclei-interactsh-integration/)
+- [Weaponizes nuclei Workflows to Pwn All the Things](https://medium.com/@dwisiswant0/weaponizes-nuclei-workflows-to-pwn-all-the-things-cd01223feb77) by [@dwisiswant0](https://github.com/dwisiswant0)
+- [How to Scan Continuously with Nuclei?](https://medium.com/@dwisiswant0/how-to-scan-continuously-with-nuclei-fcb7e9d8b8b9) by [@dwisiswant0](https://github.com/dwisiswant0)
+- [Hack with Automation !!!](https://dhiyaneshgeek.github.io/web/security/2021/07/19/hack-with-automation/) by [@DhiyaneshGeek](https://github.com/DhiyaneshGeek)
+
+### Credits
+
+Thanks to all the amazing community [contributors for sending PRs](https://github.com/projectdiscovery/nuclei/graphs/contributors). Do also check out the below similar open-source projects that may fit in your workflow:
+
+[FFuF](https://github.com/ffuf/ffuf), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [Jaeles](https://github.com/jaeles-project/jaeles), [ChopChop](https://github.com/michelin/ChopChop)
+
+### License
+
+Nuclei is distributed under [MIT License](https://github.com/projectdiscovery/nuclei/blob/master/LICENSE.md)
+
+<h1 align="left">
+  <a href="https://discord.gg/projectdiscovery"><img src="static/Join-Discord.png" width="380" alt="Join Discord"></a> <a href="https://nuclei.projectdiscovery.io"><img src="static/check-nuclei-documentation.png" width="380" alt="Check Nuclei Documentation"></a>
+</h1>

README_CN.md

@@ -0,0 +1,253 @@
+<h1 align="center">
+  <img src="static/nuclei-logo.png" alt="nuclei" width="200px"></a>
+  <br>
+</h1>
+
+[![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT)
+[![Go Report Card](https://goreportcard.com/badge/github.com/projectdiscovery/nuclei)](https://goreportcard.com/report/github.com/projectdiscovery/nuclei)
+[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectdiscovery/nuclei/issues)
+[![GitHub Release](https://img.shields.io/github/release/projectdiscovery/nuclei)](https://github.com/projectdiscovery/nuclei/releases)
+[![Follow on Twitter](https://img.shields.io/twitter/follow/pdnuclei.svg?logo=twitter)](https://twitter.com/pdnuclei)
+[![Docker Images](https://img.shields.io/docker/pulls/projectdiscovery/nuclei.svg)](https://hub.docker.com/r/projectdiscovery/nuclei)
+[![Chat on Discord](https://img.shields.io/discord/695645237418131507.svg?logo=discord)](https://discord.gg/KECAGdH)
+
+<p align="center">
+<a href="https://nuclei.projectdiscovery.io/templating-guide/" target="_blank"><img src="static/read-the-docs-button.png" height="42px"/></center></a>  <a href="https://github.com/projectdiscovery/nuclei-templates" target="_blank"><img src="static/download-templates-button.png" height="42px"/></a>
+</p>
+
+
+Nuclei是一个基于模板的、可配置攻击目标的扫描快速工具，同时还提供了强大的可扩展性和易用性。
+
+基于模板的nuclei被用来发送请求给目标，有着实现零误报的优点，并且可以对已知的路径进行有效的扫描。nuclei的主要用于在初期的探测阶段快速地对已知的且易于检测的漏洞或者CVE进行扫描。如果存在WAF的话，nuclei使用[retryablehttp-go库](https://github.com/projectdiscovery/retryablehttp-go)来处理各种错误，并且重新尝试攻击，这也是我们自定义功能的核心模块之一。
+
+我们也维护一个具有各个类型的模板的[开源库](https://github.com/projectdiscovery/nuclei-templates)，我们希望你也能贡献一些模板，贡献的这些模板最好是有效的，并且能允许每个人基于你的模板重新构建。查看[**nuclei.projectdiscovery.io**](https://nuclei.projectdiscovery.io/templating-guide/)这个网站去学习制作模板的入门知识。
+
+## 目录
+
+- [目录](#目录)
+- [功能](#功能)
+- [安装](#安装)
+- [Nuclei模板](#nuclei模板)
+- [用法](#用法)
+- [运行Nuclei](#运行nuclei)
+- [排除模板](#排除模板)
+- [致谢](#致谢)
+
+## 功能
+
+<h1 align="left">
+  <img src="static/nuclei-run.png" alt="nuclei" width="700px"></a>
+  <br>
+</h1>
+
+- 有着易于开发的、简单的、模块化的代码库
+- 使用了基于模板的引擎，运行速度极快，可以修改所以配置
+- 可以对特殊情况处理、重试、绕过等，可以绕过WAF
+- 智能匹配，零误报
+
+## 安装
+
+### 二进制文件安装
+
+二进制文件安装很简单，你可以从[Releases](https://github.com/projectdiscovery/nuclei/releases/)页面下载已经构建好的二进制文件压缩包，使用解压工具提取下载的压缩包，并将解压的文件夹移动到$PATH目录，就可以直接使用了。
+
+```sh
+Download latest binary from https://github.com/projectdiscovery/nuclei/releases
+
+▶ tar -xzvf nuclei-linux-amd64.tar.gz
+▶ mv nuclei /usr/local/bin/
+▶ nuclei -version
+```
+
+### 源码安装
+
+nuclei需要**go1.14+**才能成功安装，运行以下命令获取repo
+
+```sh
+▶ GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+```
+
+### GitHub安装
+
+```sh
+▶ git clone https://github.com/projectdiscovery/nuclei.git; cd nuclei/v2/cmd/nuclei/; go build; mv nuclei /usr/local/bin/; nuclei -version
+```
+
+## Nuclei模板
+
+你可以使用`update-templates`来下载和更新nuclei模板，该命令会从unclei的[模板库]()中下载最新版本，这个由社区来维护的库是可以随时使用的。
+
+```sh
+▶ nuclei -update-templates
+```
+
+此外您可以根据自己的工作情况或者需求编写模板，请参阅**nuclei[模板向导](https://nuclei.projectdiscovery.io/templating-guide/)去编写自定义模板**
+
+## 用法
+
+```sh
+nuclei -h
+```
+
+这条命令会显示帮助，以下是nuclei支持的所有命令
+
+|命令|描述|例子|
+|-----|-----|-----|
+|bulk-size|每个模板最大并行的主机数(默认25)|nuclei -bulk-size 25|
+|burp-collaborator-biid|使用burp-collaborator插件|nuclei -burp-collaborator-biid XXXX|
+|c|并行的最大模板数量(默认10)|nuclei -c 10|
+|l|对URL列表进行测试|nuclei -l urls.txt|
+|target|对目标进行测试|nuclei -target hxxps://example.com|
+|t|要检测的模板种类|nuclei -t git-core.yaml -t cves/|
+|no-color|输出不显示颜色|nuclei -no-color|
+|no-meta|不显示匹配的元数据|nuclei -no-meta|
+|json|输出为json格式|nuclei -json|
+|include-rr|json输出格式中包含请求和响应数据|nuclei -json -include-rr|
+|o|输出为文件|nuclei -o output.txt|
+|project|避免发送相同的请求|nuclei -project|
+|stats|使用进度条|nuclei -stats|
+|silent|只输出测试成功的结果|nuclei -silent|
+|retries|失败后的重试次数|nuclei -retries 1|
+|timeout|超时时间(默认为5秒)|nuclei -timeout 5|
+|trace-log|输出日志到log文件|nuclei -trace-log logs|
+|rate-limit|每秒最大请求数(默认150)|nuclei -rate-limit 150|
+|severity|根据严重性选择模板|nuclei  -severity critical,high|
+|stop-at-first-match|第一次匹配不要处理HTTP请求|nuclei -stop-at-frst-match|
+|exclude|排除的模板或文件夹|nuclei -exclude panels -exclude tokens|
+|debug|调试请求或者响应|nuclei -debug|
+|update-templates|下载或者升级模板|nuclei -update-templates|
+|update-directory|选择储存模板的目录(可选)|nuclei -update-directory templates|
+|tl|列出可用的模板|nuclei -tl|
+|templates-version|显示已安装的模板版本|nuclei -templates-version|
+|v|显示发送请求的详细信息|nuclei -v|
+|version|显示nuclei的版本号|nuclei -version|
+|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080|
+|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080|
+|random-agent|使用随机的UA|nuclei -random-agent|
+|H|自定义请求头|nuclei -H “x-bug-bounty:hacker”|
+
+## 运行Nuclei
+
+### 运行单个模板
+
+这将对`urls.txt`中所有的主机运行`git-core.yaml`并返回结果到`results.txt`
+
+```sh
+▶ nuclei -l urls.txt -t files/git-core.yaml -o results.txt
+```
+
+你可以轻松的通过管道使用标准的输入(STDIN)传递URL列表。
+
+```sh
+▶ cat urls.txt | nuclei -t files/git-core.yaml -o results.txt
+```
+
+💡 Nuclei可以接受如下列表的URL作为输入，例如以下URL：
+
+```
+https://test.some-site.com
+http://vuls-testing.com
+https://test.com
+```
+### 运行多个模板
+
+这将会对`urls.txt`中所有的URL运行`cves`和`files`模板检查，并返回输出到`results.txt`
+
+```sh
+▶ nuclei -l urls.txt -t cves/ -t files/ -o results.txt
+```
+
+### 使用subfinder运行
+
+```sh
+▶ subfinder -d hackerone.com -silent | httpx -silent | nuclei -t cves/ -o results.txt
+```
+
+### 在docker中运行
+
+你需要使用[nuclei的docker镜像](https://hub.docker.com/r/projectdiscovery/nuclei)来运行
+
+```sh
+▶ docker pull projectdiscovery/nuclei
+```
+
+下载并构建完成后，运行以下命令：
+
+```sh
+▶ docker run -it projectdiscovery/nuclei
+```
+
+这将会对`urls.txt`中的URL通过docker中的nuclei进行检测，并将结果输出到本机的`results.txt`文件的：
+
+```sh
+▶ cat urls.txt | docker run -v /path/to/nuclei-templates:/app/nuclei-templates -v /path/to/nuclei/config:/app/.nuclei-config.json -i projectdiscovery/nuclei -t /app/nuclei-templates/files/git-config.yaml > results.txt
+```
+记住更改的模板路径到本机
+
+### 速率限制
+
+Nuclei有多种控制速率的方法，包括并行执行多个模板、并行检查多个主机，以及使nuclei限制全局的请求速率，下面就是示例。
+
+- `-c`参数 => 限制并行的模板数
+- `-bulk-size`参数 => 限制并行的主机数
+- `-rate-limit`参数 => 全局速率限制
+
+如果你想快速扫描或者控制扫描，请使用这些标志并输入限制数，`速率限制`只保证控制传出的请求，与其他参数无关。
+
+### 排除模板
+
+[Nuclei模板](https://github.com/projectdiscovery/nuclei-templates)包含多种检查，其中有许多对攻击有用的检查，但并不是都有用的。如果您只希望扫描少数特定的模板或目录，则可以使用如下的参数筛选模板，或将某些模板排除。
+
+#### 排除模板运行
+
+我们不建议同时运行所有的nuclei模板，如果要排除模板，可以使用`exclude`参数来排除特定的目录或模板。
+
+```sh
+nuclei -l urls.txt -t nuclei-templates -exclude panels/ -exclude technologies -exclude files/wp-xmlrpc.yaml
+```
+
+注意：如上述示例中显示的那样，目录和特定模板都将不会扫描
+
+#### 基于严重性运行模板
+
+您可以根据模板的严重性运行模板，扫描时可以选择单个严重性或多个严重性。
+
+```sh
+nuclei -l urls.txt -t cves/ -severity critical,medium
+```
+
+上面的例子将运行`cves`目录下所有`严重`和`中等`的模板。
+
+```sh
+nuclei -l urls.txt -t panels/ -t technologies -severity info
+```
+
+上面的例子将运行`panels`和`technologies`目录下严重性标记为`info`的模板
+
+#### 使用`.nuclei-ignore`文件排除模板
+
+自从nuclei的[v2.1.1版本](https://github.com/projectdiscovery/nuclei/releases/tag/v2.1.1)以来，我们添加了对`.nuclei-ignore`文件的支持，该文件与`update-templates`参数一起使用，在 **.nuclei-ignore** 文件中，您可以定义要从nuclei扫描中排除的所有模板目录或者模板路径，要开始使用此功能，请确保使用`nuclei-update-templates`参数安装nuclei模板，现在可以根据`.nuclei-ignore`的文件来添加、更新、删除模板文件。
+
+```
+nano ~/nuclei-templates/.nuclei-ignore
+```
+
+默认的**nuclei忽略**列表可以访问[这里]((https://github.com/projectdiscovery/nuclei-templates/blob/master/.nuclei-ignore)，如果不想排除任何内容，只需要删除`.nuclei-ignore`文件。
+
+* * *
+
+### 📋 笔记
+
+- 进度条是实验性功能，在某些情况下可能无法使用。
+- 进度条不适用于工作流，因为是条件执行，所以不准确。
+
+
+## 致谢
+
+也要看看这些类似的好项目，或许它们也适合你：
+
+[Burp Suite](https://portswigger.net/burp), [FFuF](https://github.com/ffuf/ffuf), [Jaeles](https://github.com/jaeles-project/jaeles), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [ChopChop](https://github.com/michelin/ChopChop)
+
+--------
+
+Nuclei是由[projectdiscovery](https://projectdiscovery.io)团队用🖤制作的，当然社区也贡献了很多，通过 **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)**文件以获取更多详细信息。

SECURITY.md

@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+DO NOT CREATE AN ISSUE to report a security problem. Instead, please send an email to security@projectdiscovery.io and we will acknowledge it within 3 working days.

integration_tests/dns/basic.yaml

@@ -0,0 +1,17 @@
+id: basic-dns-example
+
+info:
+  name: Test DNS Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: A
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        words:
+          - "1.1.1.1"

integration_tests/http/get-headers.yaml

@@ -0,0 +1,17 @@
+id: basic-get-headers
+
+info:
+  name: Basic GET Headers Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    headers:
+      test: nuclei
+    matchers:
+      - type: word
+        words:
+          - "This is test headers matcher text"

integration_tests/http/get-query-string.yaml

@@ -0,0 +1,15 @@
+id: basic-get-querystring
+
+info:
+  name: Basic GET QueryString Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}?test=nuclei"
+    matchers:
+      - type: word
+        words:
+          - "This is test querystring matcher text"

integration_tests/http/get-redirects.yaml

@@ -0,0 +1,17 @@
+id: basic-get-redirects
+
+info:
+  name: Basic GET Redirects Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    redirects: true
+    max-redirects: 3
+    matchers:
+      - type: word
+        words:
+          - "This is test redirects matcher text"

integration_tests/http/get.yaml

@@ -0,0 +1,15 @@
+id: basic-get
+
+info:
+  name: Basic GET Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "This is test matcher text"

integration_tests/http/post-body.yaml

@@ -0,0 +1,19 @@
+id: basic-post-body
+
+info:
+  name: Basic POST Body Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: POST
+    path:
+      - "{{BaseURL}}"
+    headers: 
+      Content-Type: application/x-www-form-urlencoded
+      Content-Length: 1 # as long as there is a value, nuclei will auto-recalculate it.
+    body: username=test&password=nuclei
+    matchers:
+      - type: word
+        words:
+          - "This is test post-body matcher text"

integration_tests/http/post-json-body.yaml

@@ -0,0 +1,19 @@
+id: basic-post-json-body
+
+info:
+  name: Basic POST JSON Body Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: POST
+    path:
+      - "{{BaseURL}}"
+    headers: 
+      Content-Type: application/json
+      Content-Length: 1
+    body: '{"username":"test","password":"nuclei"}'
+    matchers:
+      - type: word
+        words:
+          - "This is test post-json-body matcher text"

integration_tests/http/post-multipart-body.yaml

@@ -0,0 +1,29 @@
+id: basic-post-multipart-body
+
+info:
+  name: Basic POST Multipart Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: POST
+    path:
+      - "{{BaseURL}}"
+    headers: 
+      Content-Type: multipart/form-data; boundary=d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8
+      Content-Length: 1
+    body: |
+      --d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8
+      Content-Disposition: form-data; name="username"; filename="username"
+      Content-Type: application/octet-stream
+
+      test
+      --d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8
+      Content-Disposition: form-data; name="password"
+
+      nuclei
+      --d64a5c6be2120f494d87b096fff6efe6d3248474d4de2debb1d387b3d8e8--
+    matchers:
+      - type: word
+        words:
+          - "This is test post-multipart matcher text"

integration_tests/http/raw-cookie-reuse.yaml

@@ -0,0 +1,34 @@
+id: cookiereuse-raw-example
+info:
+  name: Test CookieReuse RAW Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        POST / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+
+        testing=parameter
+      - |
+        GET / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+        
+    cookie-reuse: true
+    matchers:
+      - type: word
+        words:
+          - "Test is test-cookie-reuse matcher text"

integration_tests/http/raw-dynamic-extractor.yaml

@@ -0,0 +1,43 @@
+id: dynamic-extractor-raw-example
+
+info:
+  name: Test Dynamic Extractor RAW Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        POST / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+
+        testing=parameter
+      - |
+        GET /?username={{randkey}} HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Connection: close
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
+        Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
+        Accept-Language: en-US,en;q=0.9
+    extractors:
+      - type: regex
+        name: randkey
+        part: body
+        group: 1
+        internal: true
+        regex:
+          - "Token: '([A-Za-z0-9]+)'"
+
+    cookie-reuse: true
+    matchers:
+      - type: word
+        words:
+          - "Test is test-dynamic-extractor-raw matcher text"

integration_tests/http/raw-get-query.yaml

@@ -0,0 +1,18 @@
+id: basic-raw-query-example
+
+info:
+  name: Test RAW GET Query Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        GET ?test=nuclei HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+
+    matchers:
+      - type: word
+        words:
+          - "Test is test raw-get-query-matcher text"

integration_tests/http/raw-get.yaml

@@ -0,0 +1,18 @@
+id: basic-raw-http-example
+
+info:
+  name: Test RAW GET Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        GET / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+
+    matchers:
+      - type: word
+        words:
+          - "Test is test raw-get-matcher text"

integration_tests/http/raw-payload.yaml

@@ -0,0 +1,29 @@
+id: payload-raw-example
+info:
+  name: Test RAW With Payload Template
+  author: pdteam
+  severity: info
+
+requests:
+  - payloads:
+      username:
+        - test
+      password:
+        - nuclei
+        - guest
+    attack: clusterbomb
+    raw:
+      - |
+        POST / HTTP/1.1
+        User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5)
+        Host: {{Hostname}}
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+        another_header: {{base64('§password§')}}
+        Accept: */*
+
+        username=§username§&password={{password}}
+    matchers:
+      - type: word
+        words:
+          - "Test is raw-payload matcher text"

integration_tests/http/raw-post-body.yaml

@@ -0,0 +1,21 @@
+id: basic-raw-http-body-example
+
+info:
+  name: Test RAW POST Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        POST / HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+        Content-Type: application/x-www-form-urlencoded
+        Content-Length: 1
+
+        username=test&password=nuclei
+    matchers:
+      - type: word
+        words:
+          - "Test is test raw-post-body-matcher text"

integration_tests/http/raw-unsafe-request.yaml

@@ -0,0 +1,20 @@
+id: basic-raw-unsafe-request-example
+
+info:
+  name: Test RAW Unsafe Request Template
+  author: pd-team
+  severity: info
+
+requests:
+  - raw:
+      - |
+        GET / HTTP/1.1
+        Host: 
+        Content-Length: 4
+
+    unsafe: true
+    matchers-condition: and
+    matchers:
+      - type: word
+        words:
+          - "This is test raw-unsafe-matcher test"

integration_tests/http/request-condition-new.yaml

@@ -0,0 +1,19 @@
+id: request-condition-new
+
+info:
+  name: request-condition-new
+  author: pd-team
+  severity: info
+
+requests:
+  - method: GET
+    id: first
+    path:
+      - "{{BaseURL}}/200"
+  - method: GET
+    path:
+      - "{{BaseURL}}/400"
+    matchers:
+      - type: dsl
+        dsl:
+          - "first_status_code==200 && status_code==400"

integration_tests/http/request-condition.yaml

@@ -0,0 +1,18 @@
+id: request-condition
+
+info:
+  name: request-condition
+  author: pd-team
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}/200"
+      - "{{BaseURL}}/400"
+ 
+    req-condition: true
+    matchers:
+      - type: dsl
+        dsl:
+          - "status_code_1==200 && status_code_2==400"

integration_tests/network/basic.yaml

@@ -0,0 +1,18 @@
+id: basic-network-request
+
+info:
+  name: Basic Network Request
+  author: pdteam
+  severity: info
+
+network:
+  - host: 
+      - "{{Hostname}}"
+    inputs:
+      - data: "PING\r\n"
+    read-size: 4
+    matchers:
+      - type: word
+        part: data
+        words:
+          - "PONG"

integration_tests/network/hex.yaml

@@ -0,0 +1,22 @@
+id: hex-network-request
+
+info:
+  name: Hex Input Network Request
+  author: pdteam
+  severity: info
+
+network:
+  - host: 
+      - "{{Hostname}}"
+    inputs:
+      - data: "50494e47"
+        type: hex
+      - data: "\r\n"
+
+    read-size: 4
+    matchers:
+      - type: word
+        part: data
+        encoding: hex
+        words:
+          - "504f4e47"

integration_tests/network/multi-step.yaml

@@ -0,0 +1,32 @@
+id: multi-step
+
+info:
+  name: Multi-Step Network Template
+  author: pd-team
+  severity: info
+
+network:
+  - inputs:
+      - data: "FIRST"
+        read: 4
+        name: first
+      - data: "SECOND"
+        read: 4
+        name: second
+    host:
+      - "{{Hostname}}"
+    read-size: 6
+    matchers:
+      - type: word
+        part: first
+        words:
+          - "PING" 
+      - type: word
+        part: second
+        words:
+          - "PONG" 
+      - type: word
+        part: data
+        words:
+          - "NUCLEI" 
+    matchers-condition: and

integration_tests/run.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+cd ../v2/cmd/nuclei
+go build
+mv nuclei ../../../integration_tests/nuclei 
+cd ../integration-test
+go build
+mv integration-test ../../../integration_tests/integration-test 
+cd ../../../integration_tests
+./integration-test
+if [ $? -eq 0 ]
+then
+  exit 0
+else
+  exit 1
+fi

integration_tests/workflow/basic.yaml

@@ -0,0 +1,10 @@
+id: workflow-example
+
+info:
+  name: Test Workflow Template
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-1.yaml
+  - template: workflow/match-2.yaml

integration_tests/workflow/condition-matched.yaml

@@ -0,0 +1,11 @@
+id: condition-matched-workflow
+
+info:
+  name: Condition Matched Workflow
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-1.yaml
+    subtemplates:
+      - template: workflow/match-2.yaml

integration_tests/workflow/condition-unmatched.yaml

@@ -0,0 +1,11 @@
+id: condition-unmatched-workflow
+
+info:
+  name: Condition UnMatched Workflow
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/nomatch-1.yaml
+    subtemplates:
+      - template: workflow/match-2.yaml

integration_tests/workflow/match-1.yaml

@@ -0,0 +1,15 @@
+id: basic-get
+
+info:
+  name: Basic GET Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "This is test matcher text"

integration_tests/workflow/match-2.yaml

@@ -0,0 +1,16 @@
+id: basic-get-another
+
+info:
+  name: Basic Another GET Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        name: test-matcher
+        words:
+          - "This is test matcher text"

integration_tests/workflow/matcher-name.yaml

@@ -0,0 +1,13 @@
+id: matcher-name-workflow
+
+info:
+  name: Matcher Name Workflow
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-2.yaml
+    matchers: 
+      - name: test-matcher
+        subtemplates:
+          - template: workflow/match-1.yaml

integration_tests/workflow/nomatch-1.yaml

@@ -0,0 +1,15 @@
+id: basic-get-nomatch
+
+info:
+  name: Basic GET Request NoMatch
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "Random"

static/regression-cycle.md

@@ -0,0 +1 @@
+

v2/.goreleaser.yml

@@ -0,0 +1,33 @@
+before:
+  hooks:
+    - go mod tidy
+
+builds:
+- env:
+  - CGO_ENABLED=0
+  goos:
+    - windows
+    - linux
+    - darwin
+  goarch:
+    - amd64
+    - 386
+    - arm
+    - arm64
+
+  ignore:
+    - goos: darwin
+      goarch: '386'
+    - goos: windows
+      goarch: 'arm'
+
+  binary: '{{ .ProjectName }}'
+  main: cmd/nuclei/main.go
+
+archives:
+- format: zip
+  replacements:
+      darwin: macOS
+
+checksum:
+  algorithm: sha256

v2/Makefile

@@ -0,0 +1,14 @@
+# Go parameters
+GOCMD=go
+GOBUILD=$(GOCMD) build
+GOMOD=$(GOCMD) mod
+GOTEST=$(GOCMD) test
+GOGET=$(GOCMD) get
+    
+all: build
+build:
+		$(GOBUILD) -v -ldflags="-extldflags=-static" -o "nuclei" cmd/nuclei/main.go
+test: 
+		$(GOTEST) -v ./...
+tidy:
+		$(GOMOD) tidy

v2/cmd/integration-test/dns.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var dnsTestCases = map[string]testutils.TestCase{
+	"dns/basic.yaml": &dnsBasic{},
+}
+
+type dnsBasic struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *dnsBasic) Execute(filePath string) error {
+	var routerErr error
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, "one.one.one.one", debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/integration-test/http.go

@@ -0,0 +1,523 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"strings"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var httpTestcases = map[string]testutils.TestCase{
+	"http/get-headers.yaml":           &httpGetHeaders{},
+	"http/get-query-string.yaml":      &httpGetQueryString{},
+	"http/get-redirects.yaml":         &httpGetRedirects{},
+	"http/get.yaml":                   &httpGet{},
+	"http/post-body.yaml":             &httpPostBody{},
+	"http/post-json-body.yaml":        &httpPostJSONBody{},
+	"http/post-multipart-body.yaml":   &httpPostMultipartBody{},
+	"http/raw-cookie-reuse.yaml":      &httpRawCookieReuse{},
+	"http/raw-dynamic-extractor.yaml": &httpRawDynamicExtractor{},
+	"http/raw-get-query.yaml":         &httpRawGetQuery{},
+	"http/raw-get.yaml":               &httpRawGet{},
+	"http/raw-payload.yaml":           &httpRawPayload{},
+	"http/raw-post-body.yaml":         &httpRawPostBody{},
+	"http/raw-unsafe-request.yaml":    &httpRawUnsafeRequest{},
+	"http/request-condition.yaml":     &httpRequestCondition{},
+	"http/request-condition-new.yaml": &httpRequestCondition{},
+}
+
+func httpDebugRequestDump(r *http.Request) {
+	if debug {
+		if dump, err := httputil.DumpRequest(r, true); err == nil {
+			fmt.Printf("\nRequest dump: \n%s\n\n", string(dump))
+		}
+	}
+}
+
+type httpGetHeaders struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGetHeaders) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.Header.Get("test"), "nuclei") {
+			fmt.Fprintf(w, "This is test headers matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpGetQueryString struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGetQueryString) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") {
+			fmt.Fprintf(w, "This is test querystring matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpGetRedirects struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGetRedirects) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		http.Redirect(w, r, "/redirected", http.StatusFound)
+	}))
+	router.GET("/redirected", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test redirects matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpGet struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpGet) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpPostBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpPostBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("username"), "test") && strings.EqualFold(r.Form.Get("password"), "nuclei") {
+			fmt.Fprintf(w, "This is test post-body matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpPostJSONBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpPostJSONBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+
+		type doc struct {
+			Username string `json:"username"`
+			Password string `json:"password"`
+		}
+		obj := &doc{}
+		if err := json.NewDecoder(r.Body).Decode(obj); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(obj.Username, "test") && strings.EqualFold(obj.Password, "nuclei") {
+			fmt.Fprintf(w, "This is test post-json-body matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpPostMultipartBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpPostMultipartBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseMultipartForm(1 * 1024); err != nil {
+			routerErr = err
+			return
+		}
+		password, ok := r.MultipartForm.Value["password"]
+		if !ok || len(password) != 1 {
+			routerErr = errors.New("no password in request")
+			return
+		}
+		file := r.MultipartForm.File["username"]
+		if len(file) != 1 {
+			routerErr = errors.New("no file in request")
+			return
+		}
+		if strings.EqualFold(password[0], "nuclei") && strings.EqualFold(file[0].Filename, "username") {
+			fmt.Fprintf(w, "This is test post-multipart matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawDynamicExtractor struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawDynamicExtractor) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("testing"), "parameter") {
+			fmt.Fprintf(w, "Token: 'nuclei'")
+		}
+	}))
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.URL.Query().Get("username"), "nuclei") {
+			fmt.Fprintf(w, "Test is test-dynamic-extractor-raw matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawGetQuery struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawGetQuery) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") {
+			fmt.Fprintf(w, "Test is test raw-get-query-matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawGet struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawGet) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+
+		fmt.Fprintf(w, "Test is test raw-get-matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawPayload struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawPayload) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if !(strings.EqualFold(r.Header.Get("another_header"), "bnVjbGVp") || strings.EqualFold(r.Header.Get("another_header"), "Z3Vlc3Q=")) {
+			return
+		}
+		if strings.EqualFold(r.Form.Get("username"), "test") && (strings.EqualFold(r.Form.Get("password"), "nuclei") || strings.EqualFold(r.Form.Get("password"), "guest")) {
+			fmt.Fprintf(w, "Test is raw-payload matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 2 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawPostBody struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawPostBody) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("username"), "test") && strings.EqualFold(r.Form.Get("password"), "nuclei") {
+			fmt.Fprintf(w, "Test is test raw-post-body-matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawCookieReuse struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawCookieReuse) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		if strings.EqualFold(r.Form.Get("testing"), "parameter") {
+			http.SetCookie(w, &http.Cookie{Name: "nuclei", Value: "test"})
+		}
+	}))
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		if err := r.ParseForm(); err != nil {
+			routerErr = err
+			return
+		}
+		cookie, err := r.Cookie("nuclei")
+		if err != nil {
+			routerErr = err
+			return
+		}
+
+		if strings.EqualFold(cookie.Value, "test") {
+			fmt.Fprintf(w, "Test is test-cookie-reuse matcher text")
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRawUnsafeRequest struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRawUnsafeRequest) Execute(filePath string) error {
+	var routerErr error
+
+	ts := testutils.NewTCPServer(func(conn net.Conn) {
+		defer conn.Close()
+		_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 36\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nThis is test raw-unsafe-matcher test"))
+	})
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, "http://"+ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type httpRequestCondition struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpRequestCondition) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.GET("/200", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		w.WriteHeader(200)
+	}))
+	router.GET("/400", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		w.WriteHeader(400)
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/integration-test/integration-test.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var (
+	debug      = os.Getenv("DEBUG") == "true"
+	customTest = os.Getenv("TEST")
+	protocol   = os.Getenv("PROTO")
+)
+
+func main() {
+	success := aurora.Green("[✓]").String()
+	failed := aurora.Red("[✘]").String()
+
+	protocolTests := map[string]map[string]testutils.TestCase{
+		"http":     httpTestcases,
+		"network":  networkTestcases,
+		"dns":      dnsTestCases,
+		"workflow": workflowTestcases,
+	}
+	for proto, tests := range protocolTests {
+		if protocol == "" || protocol == proto {
+			fmt.Printf("Running test cases for \"%s\" protocol\n", aurora.Blue(proto))
+
+			for file, test := range tests {
+				if customTest != "" && !strings.Contains(file, customTest) {
+					continue // only run tests user asked
+				}
+				err := test.Execute(file)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, file, err)
+					os.Exit(1)
+				} else {
+					fmt.Printf("%s Test \"%s\" passed!\n", success, file)
+				}
+			}
+		}
+	}
+}
+
+func errIncorrectResultsCount(results []string) error {
+	return fmt.Errorf("incorrect number of results %s", strings.Join(results, "\n\t"))
+}

v2/cmd/integration-test/network.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"net"
+
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var networkTestcases = map[string]testutils.TestCase{
+	"network/basic.yaml":      &networkBasic{},
+	"network/hex.yaml":        &networkBasic{},
+	"network/multi-step.yaml": &networkMultiStep{},
+}
+
+type networkBasic struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *networkBasic) Execute(filePath string) error {
+	var routerErr error
+
+	ts := testutils.NewTCPServer(func(conn net.Conn) {
+		defer conn.Close()
+
+		data := make([]byte, 4)
+		if _, err := conn.Read(data); err != nil {
+			routerErr = err
+			return
+		}
+		if string(data) == "PING" {
+			_, _ = conn.Write([]byte("PONG"))
+		}
+	})
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type networkMultiStep struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *networkMultiStep) Execute(filePath string) error {
+	var routerErr error
+
+	ts := testutils.NewTCPServer(func(conn net.Conn) {
+		defer conn.Close()
+
+		data := make([]byte, 5)
+		if _, err := conn.Read(data); err != nil {
+			routerErr = err
+			return
+		}
+		if string(data) == "FIRST" {
+			_, _ = conn.Write([]byte("PING"))
+		}
+
+		data = make([]byte, 6)
+		if _, err := conn.Read(data); err != nil {
+			routerErr = err
+			return
+		}
+		if string(data) == "SECOND" {
+			_, _ = conn.Write([]byte("PONG"))
+		}
+		_, _ = conn.Write([]byte("NUCLEI"))
+	})
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/integration-test/workflow.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var workflowTestcases = map[string]testutils.TestCase{
+	"workflow/basic.yaml":               &workflowBasic{},
+	"workflow/condition-matched.yaml":   &workflowConditionMatched{},
+	"workflow/condition-unmatched.yaml": &workflowConditionUnmatch{},
+	"workflow/matcher-name.yaml":        &workflowMatcherName{},
+}
+
+type workflowBasic struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowBasic) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 2 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type workflowConditionMatched struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowConditionMatched) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type workflowConditionUnmatch struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowConditionUnmatch) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 0 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type workflowMatcherName struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *workflowMatcherName) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+		fmt.Fprintf(w, "This is test matcher text")
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/nuclei/issue-tracker-config.yaml

@@ -0,0 +1,49 @@
+#allow-list:
+#  severity: high,critical
+#deny-list:
+#  severity: low
+
+# github contains configuration options for github issue tracker
+#github: 
+#  # base-url is the optional self-hosted github application url
+#  base-url: ""
+#  # username is the username of the github user
+#  username: ""
+#  # owner is the owner name of the repository for issues.
+#  owner: ""
+#  # token is the token for github account.
+#  token: ""
+#  # project-name is the name of the repository.
+#  project-name: ""
+#  # issue-label is the label of the created issue type
+#  issue-label: ""
+
+# gitlab contains configuration options for gitlab issue tracker
+#gitlab: 
+#  # base-url is the optional self-hosted gitlab application url
+#  base-url: ""
+#  # username is the username of the gitlab user
+#  username: ""
+#  # token is the token for gitlab account.
+#  token: ""
+#  # project-id is the ID of the repository.
+#  project-id: ""
+#  # issue-label is the label of the created issue type
+#  issue-label: ""
+
+# jira contains configuration options for jira issue tracker
+#jira:
+#  # Cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
+#  cloud: true
+#  # URL is the jira application url
+#  url: ""
+#  # account-id is the account-id of the jira user or username in case of on-prem Jira
+#  account-id: ""
+#  # email is the email of the user for jira instance
+#  email: ""
+#  # token is the token for jira instance or password in case of on-prem Jira
+#  token: ""
+#  # project-name is the name of the project.
+#  project-name: ""
+#  # issue-type is the name of the created issue type
+#  issue-type: ""

v2/cmd/nuclei/main.go

@@ -1,19 +1,178 @@
 package main
 
 import (
+	"os"
+	"path"
+
+	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/internal/runner"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+var (
+	cfgFile string
+	options = &types.Options{}
 )
 
 func main() {
-	// Parse the command line flags and read config files
-	options := runner.ParseOptions()
+	readConfig()
+
+	runner.ParseOptions(options)
 
 	nucleiRunner, err := runner.New(options)
 	if err != nil {
-		gologger.Fatalf("Could not create runner: %s\n", err)
+		gologger.Fatal().Msgf("Could not create runner: %s\n", err)
+	}
+	if nucleiRunner == nil {
+		return
+	}
+	if err := nucleiRunner.RunEnumeration(); err != nil {
+		gologger.Fatal().Msgf("Could not run nuclei: %s\n", err)
 	}
-
-	nucleiRunner.RunEnumeration()
 	nucleiRunner.Close()
 }
+
+func readConfig() {
+	home, _ := os.UserHomeDir()
+	templatesDirectory := path.Join(home, "nuclei-templates")
+
+	flagSet := goflags.NewFlagSet()
+	flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing
+on extensive configurability, massive extensibility and ease of use.`)
+
+	createGroup(flagSet, "input", "Target",
+		flagSet.StringVarP(&options.Target, "target", "u", "", "target URL/host to scan"),
+		flagSet.StringVarP(&options.Targets, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
+	)
+
+	createGroup(flagSet, "templates", "Templates",
+		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
+
+		flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"),
+		flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "list of workflows to run"),
+
+		flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run newly added templates only"),
+		flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
+	)
+
+	createGroup(flagSet, "filters", "Filtering",
+		flagSet.NormalizedStringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"),
+		flagSet.NormalizedStringSliceVar(&options.IncludeTags, "include-tags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list
+		flagSet.NormalizedStringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"),
+
+		flagSet.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "templates to be executed even if they are excluded either by default or configuration"),
+		flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"),
+
+		flagSet.NormalizedStringSliceVarP(&options.Severity, "impact", "severity", []string{}, "execute templates that match the provided severities only"),
+		flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"),
+	)
+
+	createGroup(flagSet, "output", "Output",
+		flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write found issues/vulnerabilities"),
+
+		flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"),
+		flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"),
+		flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display extra verbose information"),
+		flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"),
+
+		flagSet.BoolVar(&options.JSON, "json", false, "write output in JSONL(ines) format"),
+		flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"),
+
+		flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "don't display match metadata"),
+		flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "local nuclei reporting database (always use this to persist report data)"),
+
+		flagSet.StringVarP(&options.DiskExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"),
+		flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"),
+	)
+
+	createGroup(flagSet, "configs", "Configurations",
+		flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"),
+		flagSet.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking
+
+		flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "custom headers in header:value format"),
+
+		flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"),
+		flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"),
+		flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"),
+	)
+
+	createGroup(flagSet, "interactsh", "interactsh",
+		flagSet.BoolVar(&options.NoInteractsh, "no-interactsh", false, "do not use interactsh server for blind interaction polling"),
+		flagSet.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "self-hosted Interactsh Server URL"),
+
+		flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"),
+		flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"),
+		flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"),
+		flagSet.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"),
+	)
+
+	createGroup(flagSet, "rate-limit", "Rate-Limit",
+		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
+		flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
+		flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "maximum number of templates to be executed in parallel"),
+	)
+
+	createGroup(flagSet, "optimization", "Optimizations",
+		flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"),
+		flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"),
+
+		flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"),
+		flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"),
+
+		flagSet.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "stop processing HTTP requests after the first match (may break template/workflow logic)"),
+	)
+
+	createGroup(flagSet, "headless", "Headless",
+		flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support"),
+		flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"),
+		flagSet.BoolVar(&options.ShowBrowser, "show-browser", false, "show the browser on the screen when running templates with headless mode"),
+	)
+
+	createGroup(flagSet, "debug", "Debug",
+		flagSet.BoolVar(&options.Debug, "debug", false, "show all requests and responses"),
+		flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"),
+		flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"),
+
+		/* TODO why the separation? http://proxy:port vs socks5://proxy:port etc
+		   TODO should auto-set the HTTP_PROXY variable for the process? */
+		flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"),
+		flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"),
+
+		flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"),
+
+		flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"),
+		flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"),
+	)
+
+	createGroup(flagSet, "update", "Update",
+		flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei to the latest released version"),
+		flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update the community templates to latest released version"),
+		flagSet.BoolVarP(&options.NoUpdateTemplates, "no-update-templates", "nut", false, "Do not check for nuclei-templates updates"),
+		flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default nuclei-templates directory"),
+	)
+
+	createGroup(flagSet, "stats", "Statistics",
+		flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"),
+		flagSet.BoolVar(&options.StatsJSON, "stats-json", false, "write statistics data to an output file in JSONL(ines) format"),
+		flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"),
+
+		flagSet.BoolVar(&options.Metrics, "metrics", false, "expose nuclei metrics on a port"),
+		flagSet.IntVar(&options.MetricsPort, "metrics-port", 9092, "port to expose nuclei metrics on"),
+	)
+
+	_ = flagSet.Parse()
+
+	if cfgFile != "" {
+		if err := flagSet.MergeConfigFile(cfgFile); err != nil {
+			gologger.Fatal().Msgf("Could not read config: %s\n", err)
+		}
+	}
+}
+
+func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) {
+	flagSet.SetGroup(groupName, description)
+	for _, currentFlag := range flags {
+		currentFlag.Group(groupName)
+	}
+}

v2/go.mod

@@ -1,30 +1,63 @@
 module github.com/projectdiscovery/nuclei/v2
 
-go 1.14
+go 1.15
 
 require (
 	github.com/Knetic/govaluate v3.0.0+incompatible
+	github.com/andygrunwald/go-jira v1.13.0
+	github.com/apex/log v1.9.0
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/d5/tengo/v2 v2.6.2
-	github.com/google/go-github/v32 v32.1.0
+	github.com/c4milo/unpackit v0.1.0 // indirect
+	github.com/corpix/uarand v0.1.1
+	github.com/fatih/structs v1.1.0 // indirect
+	github.com/go-rod/rod v0.91.1
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/google/go-github v17.0.0+incompatible
+	github.com/gosuri/uilive v0.0.4 // indirect
+	github.com/gosuri/uiprogress v0.0.1 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
 	github.com/json-iterator/go v1.1.10
+	github.com/julienschmidt/httprouter v1.3.0
+	github.com/karlseguin/ccache v2.0.3+incompatible
 	github.com/karrick/godirwalk v1.16.1
 	github.com/logrusorgru/aurora v2.0.3+incompatible
-	github.com/miekg/dns v1.1.35
+	github.com/mattn/go-runewidth v0.0.10 // indirect
+	github.com/miekg/dns v1.1.38
+	github.com/olekukonko/tablewriter v0.0.5
+	github.com/owenrumney/go-sarif v1.0.4
 	github.com/pkg/errors v0.9.1
-	github.com/projectdiscovery/clistats v0.0.5
-	github.com/projectdiscovery/collaborator v0.0.1
-	github.com/projectdiscovery/fastdialer v0.0.1
-	github.com/projectdiscovery/gologger v1.0.1
+	github.com/projectdiscovery/clistats v0.0.8
+	github.com/projectdiscovery/fastdialer v0.0.8
+	github.com/projectdiscovery/goflags v0.0.6
+	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/hmap v0.0.1
-	github.com/projectdiscovery/rawhttp v0.0.4
-	github.com/projectdiscovery/retryabledns v1.0.5
-	github.com/projectdiscovery/retryablehttp-go v1.0.1
+	github.com/projectdiscovery/interactsh v0.0.3
+	github.com/projectdiscovery/rawhttp v0.0.7
+	github.com/projectdiscovery/retryabledns v1.0.10
+	github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727
+	github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d
 	github.com/remeh/sizedwaitgroup v1.0.0
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/rs/xid v1.2.1
 	github.com/segmentio/ksuid v1.0.3
+	github.com/shirou/gopsutil/v3 v3.21.5
 	github.com/spaolacci/murmur3 v1.1.0
-	github.com/stretchr/testify v1.5.1
+	github.com/spf13/cast v1.3.1
+	github.com/stretchr/testify v1.7.0
+	github.com/syndtr/goleveldb v1.0.0
+	github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible
+	github.com/trivago/tgo v1.0.7 // indirect
+	github.com/valyala/fasttemplate v1.2.1
+	github.com/xanzy/go-gitlab v0.44.0
+	go.uber.org/atomic v1.7.0
+	go.uber.org/multierr v1.6.0
 	go.uber.org/ratelimit v0.1.0
-	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
-	gopkg.in/yaml.v2 v2.3.0
+	golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect
+	golang.org/x/net v0.0.0-20210521195947-fe42d452be8f
+	golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
+	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	gopkg.in/yaml.v2 v2.4.0
 )

v2/go.sum

@@ -1,116 +1,663 @@
-github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
-github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg=
+github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
+github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE=
+github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I=
+github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
+github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA=
+github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
+github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
+github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
+github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
+github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
-github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc=
-github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
+github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
+github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og=
+github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I=
+github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r7ZRSxEo=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
+github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4=
+github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
+github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
+github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
+github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
+github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
+github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
+github.com/eggsampler/acme/v3 v3.2.1 h1:Lfsrg3M2zt00QRnizOFzdpSfsS9oDvPsGrodXS/w1KI=
+github.com/eggsampler/acme/v3 v3.2.1/go.mod h1:/qh0rKC/Dh7Jj+p4So7DbWmFNzC4dpcpK53r226Fhuo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
+github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-rod/rod v0.91.1 h1:7xIlC/bXCXosZqZUl2x6GVB8tv4yMQ4W/ZVdGVa1qYI=
+github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
+github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
-github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
-github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
+github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY=
+github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI=
+github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw=
+github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
+github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs=
+github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
+github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU=
+github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w=
+github.com/karlseguin/ccache/v2 v2.0.7 h1:y5Pfi4eiyYCOD6LS/Kj+o6Nb4M5Ngpw9qFQs+v44ZYM=
+github.com/karlseguin/ccache/v2 v2.0.7/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ=
+github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA=
+github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
 github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
+github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
+github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs=
 github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw=
+github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/owenrumney/go-sarif v1.0.4 h1:0LFC5eHP6amc/9ajM1jDiE52UfXFcl/oozay+X3KgV4=
+github.com/owenrumney/go-sarif v1.0.4/go.mod h1:DXUGbHwQcCMvqcvZbxh8l/7diHsJVztOKZgmPt88RNI=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/projectdiscovery/clistats v0.0.5 h1:vcvOR9PrFRawO/7FWD6pER9nYVSoSTD2F+/fkRs73a0=
-github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
-github.com/projectdiscovery/collaborator v0.0.1 h1:dbQ5BCL/a3c+BB9cGtrGgiLs23+EfSzoaTzX/pxqiTI=
-github.com/projectdiscovery/collaborator v0.0.1/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
-github.com/projectdiscovery/fastdialer v0.0.1 h1:MgBkJ/zkciFu/PcbAz0DYGiZn2aqv6b39NvfXxfN8qg=
-github.com/projectdiscovery/fastdialer v0.0.1/go.mod h1:d24GUzSb93wOY7lu4gJmXAzfomqAGEcRrInEVrM6zbc=
-github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ=
-github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
+github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20=
+github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
+github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0=
+github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4=
+github.com/projectdiscovery/goflags v0.0.6 h1:4ErduTfSC55cRR3TmUg+TQirBlCuBdBadrluAsy1pew=
+github.com/projectdiscovery/goflags v0.0.6/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
+github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E=
+github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
+github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
 github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
-github.com/projectdiscovery/rawhttp v0.0.4 h1:O5IreNGk83d4xTD9e6SpkKbX0sHTs8K1Q33Bz4eYl2E=
-github.com/projectdiscovery/rawhttp v0.0.4/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
-github.com/projectdiscovery/retryabledns v1.0.5 h1:bQivGy5CuqKlwcxRkgA5ENincqIed/BR2sA6t2gdwuI=
-github.com/projectdiscovery/retryabledns v1.0.5/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
-github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
+github.com/projectdiscovery/interactsh v0.0.3 h1:PUkWk+NzSyd5glMqfORmuqizhsd7c3WdTYBOto/MQIU=
+github.com/projectdiscovery/interactsh v0.0.3/go.mod h1:dWnKO14d2FLP3kLhI9DecEsiAC/aZiJoUBGFjGhDskY=
+github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg=
+github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
+github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
+github.com/projectdiscovery/retryabledns v1.0.10 h1:xJZ2aKoqrNg/OZEw1+4+QIOH40V/WkZDYY1ZZc+uphE=
+github.com/projectdiscovery/retryabledns v1.0.10/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
+github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727 h1:CJHP3CLCc/eqdXQEvZy8KiiqtAk9kEsd1URtPyPAQ1s=
+github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
+github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nlOAex7twmrEqD5i6WLnugF9uO3DQ6jDEKN9gevrTAk=
+github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
+github.com/prologic/smtpd v0.0.0-20210126001904-0893ad18168e h1:ZT3wZ92sp/EHEE/HcFCWCsYS3ROLjHb6EqSX8qYrgXw=
+github.com/prologic/smtpd v0.0.0-20210126001904-0893ad18168e/go.mod h1:GkLsdH1RZj6RDKeI9A05NGZYmEZQ/PbQcZPnZoSZuYI=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY=
 github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shirou/gopsutil/v3 v3.21.5 h1:YUBf0w/KPLk7w1803AYBnH7BmA+1Z/Q5MEZxpREUaB4=
+github.com/shirou/gopsutil/v3 v3.21.5/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw=
+github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
+github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
+github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
+github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc=
+github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
+github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
+github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
+github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible h1:guTq1YxwB8XSILkI9q4IrOmrCOS6Hc1L3hmOhi4Swcs=
+github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible/go.mod h1:waFwwyiAhGey2e+dNoYQ/iLhIcFqhCW7zL/+vDU1WLo=
+github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M=
+github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek=
+github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc=
+github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8=
+github.com/trivago/tgo v1.0.1/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
+github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM=
+github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
+github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
+github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
+github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
+github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
+github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
+github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
+github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
+github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
+github.com/xanzy/go-gitlab v0.44.0 h1:cEiGhqu7EpFGuei2a2etAwB+x6403E5CvpLn35y+GPs=
+github.com/xanzy/go-gitlab v0.44.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
+github.com/ysmood/goob v0.3.0 h1:XZ51cZJ4W3WCoCiUktixzMIQF86W7G5VFL4QQ/Q2uS0=
+github.com/ysmood/goob v0.3.0/go.mod h1:S3lq113Y91y1UBf1wj1pFOxeahvfKkCk6mTWTWbDdWs=
+github.com/ysmood/got v0.9.3 h1:qx51X49jL/WAiqZzPTkPZ0zp5pTmrWJa4zYFTYo0gHI=
+github.com/ysmood/got v0.9.3/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
+github.com/ysmood/gotrace v0.2.0 h1:IkTC6rJREwXSaG8yWK+NFwIJGIsxA1DjC6/gxYyQttE=
+github.com/ysmood/gotrace v0.2.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
+github.com/ysmood/gson v0.6.3 h1:4cU+5oOdsyundXHy00t99H0rLXLthuseD3x6W+xmCiU=
+github.com/ysmood/gson v0.6.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
+github.com/ysmood/leakless v0.6.12 h1:XxtRYl97bJklfv4BZVdyGnd/y42p6w8lu1hUzfCkT4M=
+github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/zclconf/go-cty v1.8.2 h1:u+xZfBKgpycDnTNjPhGiTEYZS5qS/Sb5MqSfm7vzcjg=
+github.com/zclconf/go-cty v1.8.2/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
 go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9 h1:umElSU9WZirRdgu2yFHY0ayQkEnKiOC1TtM3fWXFnoU=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df h1:y7QZzfUiTwWam+xBn29Ulb8CBwVN5UdzmMDavl9Whlw=
+golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U=
+golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 h1:5vD4XjIc0X5+kHZjx4UecYdjA6mJo+XXNoaW0EjU5Os=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba h1:xmhUJGQGbxlod18iJGqVEp9cHIPLl7QiX2aA3to708s=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE=
+golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/corvus-ch/zbase32.v1 v1.0.0 h1:K4u1NprbDNvKPczKfHLbwdOWHTZ0zfv2ow71H1nRnFU=
+gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DSo0nK9U+c0Y=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

v2/internal/bufwriter/bufwriter.go

@@ -1,74 +0,0 @@
-package bufwriter
-
-import (
-	"bufio"
-	"os"
-	"sync"
-)
-
-// Writer is a mutex protected buffered writer
-type Writer struct {
-	file   *os.File
-	writer *bufio.Writer
-	mutex  *sync.Mutex
-}
-
-// New creates a new mutex protected buffered writer for a file
-func New(file string) (*Writer, error) {
-	output, err := os.Create(file)
-	if err != nil {
-		return nil, err
-	}
-	return &Writer{file: output, writer: bufio.NewWriter(output), mutex: &sync.Mutex{}}, nil
-}
-
-// Write writes a byte slice to the underlying file
-//
-// It also writes a newline if the last byte isn't a newline character.
-func (w *Writer) Write(data []byte) error {
-	if len(data) == 0 {
-		return nil
-	}
-	w.mutex.Lock()
-	defer w.mutex.Unlock()
-
-	_, err := w.writer.Write(data)
-	if err != nil {
-		return err
-	}
-	if data[len(data)-1] != '\n' {
-		_, err = w.writer.WriteRune('\n')
-	}
-	return err
-}
-
-// WriteString writes a string to the underlying file
-//
-// It also writes a newline if the last byte isn't a newline character.
-func (w *Writer) WriteString(data string) error {
-	if data == "" {
-		return nil
-	}
-	w.mutex.Lock()
-	defer w.mutex.Unlock()
-
-	_, err := w.writer.WriteString(data)
-	if err != nil {
-		return err
-	}
-	if data[len(data)-1] != '\n' {
-		_, err = w.writer.WriteRune('\n')
-	}
-	return err
-}
-
-// Close closes the underlying writer flushing everything to disk
-func (w *Writer) Close() error {
-	w.mutex.Lock()
-	defer w.mutex.Unlock()
-
-	w.writer.Flush()
-	//nolint:errcheck // we don't care whether sync failed or succeeded.
-	w.file.Sync()
-	return w.file.Close()
-}

v2/internal/colorizer/colorizer.go

@@ -0,0 +1,24 @@
+package colorizer
+
+import "github.com/logrusorgru/aurora"
+
+// Colorizer returns a colorized severity printer
+type Colorizer struct {
+	Data map[string]string
+}
+
+const (
+	fgOrange uint8 = 208
+)
+
+// New returns a new severity based colorizer
+func New(colorizer aurora.Aurora) *Colorizer {
+	severityMap := map[string]string{
+		"info":     colorizer.Blue("info").String(),
+		"low":      colorizer.Green("low").String(),
+		"medium":   colorizer.Yellow("medium").String(),
+		"high":     colorizer.Index(fgOrange, "high").String(),
+		"critical": colorizer.Red("critical").String(),
+	}
+	return &Colorizer{Data: severityMap}
+}

v2/internal/progress/progress.go

@@ -1,146 +0,0 @@
-package progress
-
-import (
-	"fmt"
-	"os"
-	"strings"
-	"time"
-
-	"github.com/projectdiscovery/clistats"
-	"github.com/projectdiscovery/gologger"
-)
-
-// Progress is a progress instance for showing program stats
-type Progress struct {
-	active       bool
-	stats        clistats.StatisticsClient
-	tickDuration time.Duration
-}
-
-// NewProgress creates and returns a new progress tracking object.
-func NewProgress(active bool) *Progress {
-	var tickDuration time.Duration
-	if active {
-		tickDuration = 5 * time.Second
-	} else {
-		tickDuration = -1
-	}
-
-	var progress Progress
-	if active {
-		stats, err := clistats.New()
-		if err != nil {
-			gologger.Warningf("Couldn't create progress engine: %s\n", err)
-		}
-		progress.active = active
-		progress.stats = stats
-		progress.tickDuration = tickDuration
-	}
-
-	return &progress
-}
-
-// Init initializes the progress display mechanism by setting counters, etc.
-func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) {
-	if p.active {
-		p.stats.AddStatic("templates", rulesCount)
-		p.stats.AddStatic("hosts", hostCount)
-		p.stats.AddStatic("startedAt", time.Now())
-		p.stats.AddCounter("requests", uint64(0))
-		p.stats.AddCounter("errors", uint64(0))
-		p.stats.AddCounter("total", uint64(requestCount))
-		if err := p.stats.Start(makePrintCallback(), p.tickDuration); err != nil {
-			gologger.Warningf("Couldn't start statistics: %s\n", err)
-		}
-	}
-}
-
-// AddToTotal adds a value to the total request count
-func (p *Progress) AddToTotal(delta int64) {
-	if p.active {
-		p.stats.IncrementCounter("total", int(delta))
-	}
-}
-
-// Update progress tracking information and increments the request counter by one unit.
-func (p *Progress) Update() {
-	if p.active {
-		p.stats.IncrementCounter("requests", 1)
-	}
-}
-
-// Drop drops the specified number of requests from the progress bar total.
-// This may be the case when uncompleted requests are encountered and shouldn't be part of the total count.
-func (p *Progress) Drop(count int64) {
-	if p.active {
-		// mimic dropping by incrementing the completed requests
-		p.stats.IncrementCounter("errors", int(count))
-	}
-}
-
-const bufferSize = 128
-
-func makePrintCallback() func(stats clistats.StatisticsClient) {
-	builder := &strings.Builder{}
-	builder.Grow(bufferSize)
-
-	return func(stats clistats.StatisticsClient) {
-		builder.WriteRune('[')
-		startedAt, _ := stats.GetStatic("startedAt")
-		duration := time.Since(startedAt.(time.Time))
-		builder.WriteString(fmtDuration(duration))
-		builder.WriteRune(']')
-
-		templates, _ := stats.GetStatic("templates")
-		builder.WriteString(" | Templates: ")
-		builder.WriteString(clistats.String(templates))
-		hosts, _ := stats.GetStatic("hosts")
-		builder.WriteString(" | Hosts: ")
-		builder.WriteString(clistats.String(hosts))
-
-		requests, _ := stats.GetCounter("requests")
-		total, _ := stats.GetCounter("total")
-
-		builder.WriteString(" | RPS: ")
-		builder.WriteString(clistats.String(uint64(float64(requests) / duration.Seconds())))
-
-		errors, _ := stats.GetCounter("errors")
-		builder.WriteString(" | Errors: ")
-		builder.WriteString(clistats.String(errors))
-
-		builder.WriteString(" | Requests: ")
-		builder.WriteString(clistats.String(requests))
-		builder.WriteRune('/')
-		builder.WriteString(clistats.String(total))
-		builder.WriteRune(' ')
-		builder.WriteRune('(')
-		//nolint:gomnd // this is not a magic number
-		builder.WriteString(clistats.String(uint64(float64(requests) / float64(total) * 100.0)))
-		builder.WriteRune('%')
-		builder.WriteRune(')')
-		builder.WriteRune('\n')
-
-		fmt.Fprintf(os.Stderr, "%s", builder.String())
-		builder.Reset()
-	}
-}
-
-// fmtDuration formats the duration for the time elapsed
-func fmtDuration(d time.Duration) string {
-	d = d.Round(time.Second)
-	h := d / time.Hour
-	d -= h * time.Hour
-	m := d / time.Minute
-	d -= m * time.Minute
-	s := d / time.Second
-	return fmt.Sprintf("%d:%02d:%02d", h, m, s)
-}
-
-// Stop stops the progress bar execution
-func (p *Progress) Stop() {
-	if p.active {
-		if err := p.stats.Stop(); err != nil {
-			gologger.Warningf("Couldn't stop statistics: %s\n", err)
-		}
-	}
-}

v2/internal/runner/banner.go

@@ -1,23 +1,25 @@
 package runner
 
-import "github.com/projectdiscovery/gologger"
+import (
+	"fmt"
 
-const banner = `
-                       __     _
-     ____  __  _______/ /__  (_)
-    / __ \/ / / / ___/ / _ \/ /
-   / / / / /_/ / /__/ /  __/ /
-  /_/ /_/\__,_/\___/_/\___/_/   v2.2.0
-`
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+)
 
-// Version is the current version of nuclei
-const Version = `2.2.0`
+var banner = fmt.Sprintf(`
+                     __     _
+   ____  __  _______/ /__  (_)
+  / __ \/ / / / ___/ / _ \/ /
+ / / / / /_/ / /__/ /  __/ /
+/_/ /_/\__,_/\___/_/\___/_/   %s
+`, config.Version)
 
 // showBanner is used to show the banner to the user
 func showBanner() {
-	gologger.Printf("%s\n", banner)
-	gologger.Printf("\t\tprojectdiscovery.io\n\n")
+	gologger.Print().Msgf("%s\n", banner)
+	gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n")
 
-	gologger.Labelf("Use with caution. You are responsible for your actions\n")
-	gologger.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n")
+	gologger.Warning().Msgf("Use with caution. You are responsible for your actions\n")
+	gologger.Warning().Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
 }

v2/internal/runner/config.go

@@ -1,136 +1 @@
 package runner
-
-import (
-	"bufio"
-	"os"
-	"path"
-	"regexp"
-	"strings"
-	"time"
-
-	jsoniter "github.com/json-iterator/go"
-)
-
-// nucleiConfig contains some configuration options for nuclei
-type nucleiConfig struct {
-	TemplatesDirectory string    `json:"templates-directory,omitempty"`
-	CurrentVersion     string    `json:"current-version,omitempty"`
-	LastChecked        time.Time `json:"last-checked,omitempty"`
-
-	// IgnorePaths ignores all the paths listed unless specified manually
-	IgnorePaths []string `json:"ignore-paths,omitempty"`
-}
-
-// nucleiConfigFilename is the filename of nuclei configuration file.
-const nucleiConfigFilename = ".nuclei-config.json"
-
-var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
-
-// readConfiguration reads the nuclei configuration file from disk.
-func readConfiguration() (*nucleiConfig, error) {
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return nil, err
-	}
-
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
-	file, err := os.Open(templatesConfigFile)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-
-	config := &nucleiConfig{}
-	err = jsoniter.NewDecoder(file).Decode(config)
-	if err != nil {
-		return nil, err
-	}
-	return config, nil
-}
-
-// readConfiguration reads the nuclei configuration file from disk.
-func (r *Runner) writeConfiguration(config *nucleiConfig) error {
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return err
-	}
-
-	config.LastChecked = time.Now()
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
-	file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	err = jsoniter.NewEncoder(file).Encode(config)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-const nucleiIgnoreFile = ".nuclei-ignore"
-
-// readNucleiIgnoreFile reads the nuclei ignore file marking it in map
-func (r *Runner) readNucleiIgnoreFile() {
-	file, err := os.Open(r.getIgnoreFilePath())
-	if err != nil {
-		return
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		text := scanner.Text()
-		if text == "" {
-			continue
-		}
-		if strings.HasPrefix(text, "#") {
-			continue
-		}
-		r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, text)
-	}
-}
-
-// checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules.
-func (r *Runner) checkIfInNucleiIgnore(item string) bool {
-	if r.templatesConfig == nil {
-		return false
-	}
-
-	for _, paths := range r.templatesConfig.IgnorePaths {
-		// If we have a path to ignore, check if it's in the item.
-		if paths[len(paths)-1] == '/' {
-			if strings.Contains(item, paths) {
-				return true
-			}
-
-			continue
-		}
-		// Check for file based extension in ignores
-		if strings.HasSuffix(item, paths) {
-			return true
-		}
-	}
-
-	return false
-}
-
-func (r *Runner) getIgnoreFilePath() string {
-	defIgnoreFilePath := path.Join(r.templatesConfig.TemplatesDirectory, nucleiIgnoreFile)
-
-	cwd, err := os.Getwd()
-	if err != nil {
-		return defIgnoreFilePath
-	}
-
-	cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile)
-
-	cwdIfpInfo, err := os.Stat(cwdIgnoreFilePath)
-	if os.IsNotExist(err) || cwdIfpInfo.IsDir() {
-		return defIgnoreFilePath
-	}
-
-	return cwdIgnoreFilePath
-}

v2/internal/runner/options.go

@@ -1,136 +1,67 @@
 package runner
 
 import (
+	"bufio"
 	"errors"
-	"flag"
 	"net/url"
 	"os"
+	"strings"
 
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
+	"github.com/projectdiscovery/gologger/formatter"
+	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
-// Options contains the configuration options for tuning
-// the template requesting process.
-// nolint // false positive, options are allocated once and are necessary as is
-type Options struct {
-	Debug                bool                   // Debug mode allows debugging request/responses for the engine
-	Silent               bool                   // Silent suppresses any extra text and only writes found URLs on screen.
-	Version              bool                   // Version specifies if we should just show version and exit
-	Verbose              bool                   // Verbose flag indicates whether to show verbose output or not
-	NoColor              bool                   // No-Color disables the colored output.
-	UpdateTemplates      bool                   // UpdateTemplates updates the templates installed at startup
-	JSON                 bool                   // JSON writes json output to files
-	JSONRequests         bool                   // write requests/responses for matches in JSON output
-	EnableProgressBar    bool                   // Enable progrss bar
-	TemplatesVersion     bool                   // Show the templates installed version
-	TemplateList         bool                   // List available templates
-	Stdin                bool                   // Stdin specifies whether stdin input was given to the process
-	StopAtFirstMatch     bool                   // Stop processing template at first full match (this may break chained requests)
-	NoMeta               bool                   // Don't display metadata for the matches
-	BulkSize             int                    // Number of targets analyzed in parallel for each template
-	TemplateThreads      int                    // Number of templates executed in parallel
-	Project              bool                   // Nuclei uses project folder to avoid sending same HTTP request multiple times
-	ProjectPath          string                 // Nuclei uses a user defined project folder
-	Timeout              int                    // Timeout is the seconds to wait for a response from the server.
-	Retries              int                    // Retries is the number of times to retry the request
-	RateLimit            int                    // Rate-Limit of requests per specified target
-	Severity             string                 // Filter templates based on their severity and only run the matching ones.
-	Target               string                 // Target is a single URL/Domain to scan usng a template
-	Targets              string                 // Targets specifies the targets to scan using templates.
-	Output               string                 // Output is the file to write found subdomains to.
-	ProxyURL             string                 // ProxyURL is the URL for the proxy server
-	ProxySocksURL        string                 // ProxySocksURL is the URL for the proxy socks server
-	TemplatesDirectory   string                 // TemplatesDirectory is the directory to use for storing templates
-	TraceLogFile         string                 // TraceLogFile specifies a file to write with the trace of all requests
-	Templates            multiStringFlag        // Signature specifies the template/templates to use
-	ExcludedTemplates    multiStringFlag        // Signature specifies the template/templates to exclude
-	CustomHeaders        requests.CustomHeaders // Custom global headers
-	Threads              int                    // Thread controls the number of concurrent requests to make.
-	BurpCollaboratorBiid string                 // Burp Collaborator BIID for polling
-}
-
-type multiStringFlag []string
-
-func (m *multiStringFlag) String() string {
-	return ""
-}
-
-func (m *multiStringFlag) Set(value string) error {
-	*m = append(*m, value)
-	return nil
-}
-
 // ParseOptions parses the command line flags provided by a user
-func ParseOptions() *Options {
-	options := &Options{}
-
-	flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template")
-	flag.Var(&options.Templates, "t", "Template input dir/file/files to run on host. Can be used multiple times. Supports globbing.")
-	flag.Var(&options.ExcludedTemplates, "exclude", "Template input dir/file/files to exclude. Can be used multiple times. Supports globbing.")
-	flag.StringVar(&options.Severity, "severity", "", "Filter templates based on their severity and only run the matching ones. Comma-separated values can be used to specify multiple severities.")
-	flag.StringVar(&options.Targets, "l", "", "List of URLs to run templates on")
-	flag.StringVar(&options.Output, "o", "", "File to write output to (optional)")
-	flag.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server")
-	flag.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the proxy socks server")
-	flag.BoolVar(&options.Silent, "silent", false, "Show only results in output")
-	flag.BoolVar(&options.Version, "version", false, "Show version of nuclei")
-	flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output")
-	flag.BoolVar(&options.NoColor, "no-color", false, "Disable colors in output")
-	flag.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout")
-	flag.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
-	flag.Var(&options.CustomHeaders, "H", "Custom Header.")
-	flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses")
-	flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)")
-	flag.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log")
-	flag.StringVar(&options.TemplatesDirectory, "update-directory", "", "Directory to use for storing nuclei-templates")
-	flag.BoolVar(&options.JSON, "json", false, "Write json output to files")
-	flag.BoolVar(&options.JSONRequests, "include-rr", false, "Write requests/responses for matches in JSON output")
-	flag.BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan")
-	flag.BoolVar(&options.TemplateList, "tl", false, "List available templates")
-	flag.IntVar(&options.RateLimit, "rate-limit", 150, "Rate-Limit (maximum requests/second")
-	flag.BoolVar(&options.StopAtFirstMatch, "stop-at-first-match", false, "Stop processing http requests at first match (this may break template/workflow logic)")
-	flag.IntVar(&options.BulkSize, "bulk-size", 25, "Maximum Number of hosts analyzed in parallel per template")
-	flag.IntVar(&options.TemplateThreads, "c", 10, "Maximum Number of templates executed in parallel")
-	flag.BoolVar(&options.Project, "project", false, "Use a project folder to avoid sending same request multiple times")
-	flag.StringVar(&options.ProjectPath, "project-path", "", "Use a user defined project folder, temporary folder is used if not specified but enabled")
-	flag.BoolVar(&options.NoMeta, "no-meta", false, "Don't display metadata for the matches")
-	flag.BoolVar(&options.TemplatesVersion, "templates-version", false, "Shows the installed nuclei-templates version")
-	flag.StringVar(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "", "Burp Collaborator BIID")
-	flag.Parse()
-
+func ParseOptions(options *types.Options) {
 	// Check if stdin pipe was given
 	options.Stdin = hasStdin()
 
 	// Read the inputs and configure the logging
-	options.configureOutput()
+	configureOutput(options)
 
 	// Show the user the banner
 	showBanner()
 
 	if options.Version {
-		gologger.Infof("Current Version: %s\n", Version)
+		gologger.Info().Msgf("Current Version: %s\n", config.Version)
 		os.Exit(0)
 	}
 	if options.TemplatesVersion {
-		config, err := readConfiguration()
+		configuration, err := config.ReadConfiguration()
 		if err != nil {
-			gologger.Fatalf("Could not read template configuration: %s\n", err)
+			gologger.Fatal().Msgf("Could not read template configuration: %s\n", err)
 		}
-		gologger.Infof("Current nuclei-templates version: %s (%s)\n", config.CurrentVersion, config.TemplatesDirectory)
+		gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", configuration.CurrentVersion, configuration.TemplatesDirectory)
 		os.Exit(0)
 	}
 
 	// Validate the options passed by the user and if any
 	// invalid options have been used, exit.
-	err := options.validateOptions()
-	if err != nil {
-		gologger.Fatalf("Program exiting: %s\n", err)
+	if err := validateOptions(options); err != nil {
+		gologger.Fatal().Msgf("Program exiting: %s\n", err)
 	}
 
-	return options
+	// Auto adjust rate limits when using headless mode if the user
+	// hasn't specified any custom limits.
+	if options.Headless && options.BulkSize == 25 && options.TemplateThreads == 10 {
+		options.BulkSize = 2
+		options.TemplateThreads = 2
+	}
+
+	// Load the resolvers if user asked for them
+	loadResolvers(options)
+
+	err := protocolinit.Init(options)
+	if err != nil {
+		gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err)
+	}
 }
 
+// hasStdin returns true if we have stdin input
 func hasStdin() bool {
 	stat, err := os.Stdin.Stat()
 	if err != nil {
@@ -144,40 +75,22 @@ func hasStdin() bool {
 }
 
 // validateOptions validates the configuration options passed
-func (options *Options) validateOptions() error {
+func validateOptions(options *types.Options) error {
 	// Both verbose and silent flags were used
 	if options.Verbose && options.Silent {
 		return errors.New("both verbose and silent mode specified")
 	}
 
-	if !options.TemplateList {
-		// Check if a list of templates was provided and it exists
-		if len(options.Templates) == 0 && !options.UpdateTemplates {
-			return errors.New("no template/templates provided")
-		}
-
-		if options.Targets == "" && !options.Stdin && options.Target == "" && !options.UpdateTemplates {
-			return errors.New("no target input provided")
-		}
-	}
-
 	// Validate proxy options if provided
-	err := validateProxyURL(
-		options.ProxyURL,
-		"invalid http proxy format (It should be http://username:password@host:port)",
-	)
+	err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)")
 	if err != nil {
 		return err
 	}
 
-	err = validateProxyURL(
-		options.ProxySocksURL,
-		"invalid socks proxy format (It should be socks5://username:password@host:port)",
-	)
+	err = validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)")
 	if err != nil {
 		return err
 	}
-
 	return nil
 }
 
@@ -191,22 +104,48 @@ func validateProxyURL(proxyURL, message string) error {
 
 func isValidURL(urlString string) bool {
 	_, err := url.Parse(urlString)
-
 	return err == nil
 }
 
 // configureOutput configures the output on the screen
-func (options *Options) configureOutput() {
+func configureOutput(options *types.Options) {
 	// If the user desires verbose output, show verbose output
 	if options.Verbose {
-		gologger.MaxLevel = gologger.Verbose
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
+	}
+	if options.Debug {
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
 	}
-
 	if options.NoColor {
-		gologger.UseColors = false
+		gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true))
 	}
-
 	if options.Silent {
-		gologger.MaxLevel = gologger.Silent
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
+	}
+}
+
+// loadResolvers loads resolvers from both user provided flag and file
+func loadResolvers(options *types.Options) {
+	if options.ResolversFile == "" {
+		return
+	}
+
+	file, err := os.Open(options.ResolversFile)
+	if err != nil {
+		gologger.Fatal().Msgf("Could not open resolvers file: %s\n", err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		part := scanner.Text()
+		if part == "" {
+			continue
+		}
+		if strings.Contains(part, ":") {
+			options.InternalResolversList = append(options.InternalResolversList, part)
+		} else {
+			options.InternalResolversList = append(options.InternalResolversList, part+":53")
+		}
 	}
 }

v2/internal/runner/paths.go

@@ -1,48 +0,0 @@
-package runner
-
-import (
-	"fmt"
-	"os"
-	"path"
-	"strings"
-
-	"github.com/projectdiscovery/gologger"
-)
-
-// isRelative checks if a given path is a relative path
-func isRelative(filePath string) bool {
-	if strings.HasPrefix(filePath, "/") || strings.Contains(filePath, ":\\") {
-		return false
-	}
-
-	return true
-}
-
-// resolvePath gets the absolute path to the template by either
-// looking in the current directory or checking the nuclei templates directory.
-//
-// Current directory is given preference over the nuclei-templates directory.
-func (r *Runner) resolvePath(templateName string) (string, error) {
-	curDirectory, err := os.Getwd()
-	if err != nil {
-		return "", err
-	}
-
-	templatePath := path.Join(curDirectory, templateName)
-	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
-		gologger.Debugf("Found template in current directory: %s\n", templatePath)
-
-		return templatePath, nil
-	}
-
-	if r.templatesConfig != nil {
-		templatePath := path.Join(r.templatesConfig.TemplatesDirectory, templateName)
-		if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
-			gologger.Debugf("Found template in nuclei-templates directory: %s\n", templatePath)
-
-			return templatePath, nil
-		}
-	}
-
-	return "", fmt.Errorf("no such path found: %s", templateName)
-}

v2/internal/runner/processor.go

@@ -1,332 +1,50 @@
 package runner
 
 import (
-	"context"
-	"fmt"
-	"net/http/cookiejar"
-	"os"
-	"path"
-	"path/filepath"
-	"strings"
-
-	tengo "github.com/d5/tengo/v2"
-	"github.com/d5/tengo/v2/stdlib"
-	"github.com/karrick/godirwalk"
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/internal/progress"
-	"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
-	"github.com/projectdiscovery/nuclei/v2/pkg/executer"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
 	"github.com/remeh/sizedwaitgroup"
+	"go.uber.org/atomic"
 )
 
-// workflowTemplates contains the initialized workflow templates per template group
-type workflowTemplates struct {
-	Name      string
-	Templates []*workflows.Template
-}
-
-// processTemplateWithList processes a template and runs the enumeration on all the targets
-func (r *Runner) processTemplateWithList(p *progress.Progress, template *templates.Template, request interface{}) bool {
-	var httpExecuter *executer.HTTPExecuter
-	var dnsExecuter *executer.DNSExecuter
-	var err error
-
-	// Create an executer based on the request type.
-	switch value := request.(type) {
-	case *requests.DNSRequest:
-		dnsExecuter = executer.NewDNSExecuter(&executer.DNSOptions{
-			TraceLog:      r.traceLog,
-			Debug:         r.options.Debug,
-			Template:      template,
-			DNSRequest:    value,
-			Writer:        r.output,
-			JSON:          r.options.JSON,
-			JSONRequests:  r.options.JSONRequests,
-			NoMeta:        r.options.NoMeta,
-			ColoredOutput: !r.options.NoColor,
-			Colorizer:     r.colorizer,
-			Decolorizer:   r.decolorizer,
-			RateLimiter:   r.ratelimiter,
-		})
-	case *requests.BulkHTTPRequest:
-		httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{
-			TraceLog:         r.traceLog,
-			Debug:            r.options.Debug,
-			Template:         template,
-			BulkHTTPRequest:  value,
-			Writer:           r.output,
-			Timeout:          r.options.Timeout,
-			Retries:          r.options.Retries,
-			ProxyURL:         r.options.ProxyURL,
-			ProxySocksURL:    r.options.ProxySocksURL,
-			CustomHeaders:    r.options.CustomHeaders,
-			JSON:             r.options.JSON,
-			JSONRequests:     r.options.JSONRequests,
-			NoMeta:           r.options.NoMeta,
-			CookieReuse:      value.CookieReuse,
-			ColoredOutput:    !r.options.NoColor,
-			Colorizer:        &r.colorizer,
-			Decolorizer:      r.decolorizer,
-			StopAtFirstMatch: r.options.StopAtFirstMatch,
-			PF:               r.pf,
-			Dialer:           r.dialer,
-			RateLimiter:      r.ratelimiter,
-		})
-	}
-
-	if err != nil {
-		p.Drop(request.(*requests.BulkHTTPRequest).GetRequestCount())
-		gologger.Warningf("Could not create http client: %s\n", err)
-
-		return false
-	}
-
-	var globalresult atomicboolean.AtomBool
-
+// processTemplateWithList process a template on the URL list
+func (r *Runner) processTemplateWithList(template *templates.Template) bool {
+	results := &atomic.Bool{}
 	wg := sizedwaitgroup.New(r.options.BulkSize)
-
-	r.hm.Scan(func(k, _ []byte) error {
+	r.hostMap.Scan(func(k, _ []byte) error {
 		URL := string(k)
+
 		wg.Add()
 		go func(URL string) {
 			defer wg.Done()
 
-			var result *executer.Result
-
-			if httpExecuter != nil {
-				result = httpExecuter.ExecuteHTTP(p, URL)
-				globalresult.Or(result.GotResults)
-			}
-
-			if dnsExecuter != nil {
-				result = dnsExecuter.ExecuteDNS(p, URL)
-				globalresult.Or(result.GotResults)
-			}
-
-			if result.Error != nil {
-				gologger.Warningf("[%s] Could not execute step: %s\n", r.colorizer.Colorizer.BrightBlue(template.ID), result.Error)
+			match, err := template.Executer.Execute(URL)
+			if err != nil {
+				gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err)
 			}
+			results.CAS(false, match)
 		}(URL)
-
 		return nil
 	})
-
 	wg.Wait()
-
-	// See if we got any results from the executers
-	return globalresult.Get()
+	return results.Load()
 }
 
-// ProcessWorkflowWithList coming from stdin or list of targets
-func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflows.Workflow) bool {
-	result := false
-
-	workflowTemplatesList, err := r.preloadWorkflowTemplates(p, workflow)
-	if err != nil {
-		gologger.Warningf("Could not preload templates for workflow %s: %s\n", workflow.ID, err)
-		return result
-	}
-
-	logicBytes := []byte(workflow.Logic)
-
+// processTemplateWithList process a template on the URL list
+func (r *Runner) processWorkflowWithList(template *templates.Template) bool {
+	results := &atomic.Bool{}
 	wg := sizedwaitgroup.New(r.options.BulkSize)
 
-	r.hm.Scan(func(k, _ []byte) error {
-		targetURL := string(k)
+	r.hostMap.Scan(func(k, _ []byte) error {
+		URL := string(k)
 		wg.Add()
-
-		go func(targetURL string) {
+		go func(URL string) {
 			defer wg.Done()
-
-			script := tengo.NewScript(logicBytes)
-			script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
-
-			variables := make(map[string]*workflows.NucleiVar)
-
-			for _, workflowTemplate := range *workflowTemplatesList {
-				name := workflowTemplate.Name
-				variable := &workflows.NucleiVar{Templates: workflowTemplate.Templates, URL: targetURL}
-				err := script.Add(name, variable)
-				if err != nil {
-					gologger.Errorf("Could not initialize script for workflow '%s': %s\n", workflow.ID, err)
-					continue
-				}
-				variables[name] = variable
-			}
-
-			_, err := script.RunContext(context.Background())
-			if err != nil {
-				gologger.Errorf("Could not execute workflow '%s': %s\n", workflow.ID, err)
-			}
-
-			for _, variable := range variables {
-				result = !variable.IsFalsy()
-				if result {
-					break
-				}
-			}
-		}(targetURL)
+			match := template.CompiledWorkflow.RunWorkflow(URL)
+			results.CAS(false, match)
+		}(URL)
 		return nil
 	})
-
 	wg.Wait()
-
-	return result
-}
-
-func (r *Runner) preloadWorkflowTemplates(p *progress.Progress, workflow *workflows.Workflow) (*[]workflowTemplates, error) {
-	var jar *cookiejar.Jar
-
-	if workflow.CookieReuse {
-		var err error
-		jar, err = cookiejar.New(nil)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// Single yaml provided
-	var wflTemplatesList []workflowTemplates
-
-	for name, value := range workflow.Variables {
-		// Check if the template is an absolute path or relative path.
-		// If the path is absolute, use it. Otherwise,
-		if isRelative(value) {
-			newPath, err := r.resolvePath(value)
-			if err != nil {
-				newPath, err = resolvePathWithBaseFolder(filepath.Dir(workflow.GetPath()), value)
-				if err != nil {
-					return nil, err
-				}
-			}
-
-			value = newPath
-		}
-
-		var wtlst []*workflows.Template
-
-		if strings.HasSuffix(value, ".yaml") {
-			t, err := templates.Parse(value)
-			if err != nil {
-				return nil, err
-			}
-
-			template := &workflows.Template{Progress: p}
-			if len(t.BulkRequestsHTTP) > 0 {
-				template.HTTPOptions = &executer.HTTPOptions{
-					TraceLog:         r.traceLog,
-					Debug:            r.options.Debug,
-					Writer:           r.output,
-					Template:         t,
-					Timeout:          r.options.Timeout,
-					Retries:          r.options.Retries,
-					ProxyURL:         r.options.ProxyURL,
-					ProxySocksURL:    r.options.ProxySocksURL,
-					CustomHeaders:    r.options.CustomHeaders,
-					JSON:             r.options.JSON,
-					JSONRequests:     r.options.JSONRequests,
-					CookieJar:        jar,
-					ColoredOutput:    !r.options.NoColor,
-					Colorizer:        &r.colorizer,
-					Decolorizer:      r.decolorizer,
-					PF:               r.pf,
-					RateLimiter:      r.ratelimiter,
-					NoMeta:           r.options.NoMeta,
-					StopAtFirstMatch: r.options.StopAtFirstMatch,
-					Dialer:           r.dialer,
-				}
-			} else if len(t.RequestsDNS) > 0 {
-				template.DNSOptions = &executer.DNSOptions{
-					TraceLog:      r.traceLog,
-					Debug:         r.options.Debug,
-					Template:      t,
-					Writer:        r.output,
-					JSON:          r.options.JSON,
-					JSONRequests:  r.options.JSONRequests,
-					ColoredOutput: !r.options.NoColor,
-					Colorizer:     r.colorizer,
-					Decolorizer:   r.decolorizer,
-					NoMeta:        r.options.NoMeta,
-					RateLimiter:   r.ratelimiter,
-				}
-			}
-
-			if template.DNSOptions != nil || template.HTTPOptions != nil {
-				wtlst = append(wtlst, template)
-			}
-		} else {
-			matches := []string{}
-
-			err := godirwalk.Walk(value, &godirwalk.Options{
-				Callback: func(path string, d *godirwalk.Dirent) error {
-					if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
-						matches = append(matches, path)
-					}
-
-					return nil
-				},
-				ErrorCallback: func(path string, err error) godirwalk.ErrorAction {
-					return godirwalk.SkipNode
-				},
-				Unsorted: true,
-			})
-
-			if err != nil {
-				return nil, err
-			}
-
-			// 0 matches means no templates were found in directory
-			if len(matches) == 0 {
-				return nil, fmt.Errorf("no match found in the directory %s", value)
-			}
-
-			for _, match := range matches {
-				t, err := templates.Parse(match)
-				if err != nil {
-					return nil, err
-				}
-				template := &workflows.Template{Progress: p}
-				if len(t.BulkRequestsHTTP) > 0 {
-					template.HTTPOptions = &executer.HTTPOptions{
-						Debug:         r.options.Debug,
-						Writer:        r.output,
-						Template:      t,
-						Timeout:       r.options.Timeout,
-						Retries:       r.options.Retries,
-						ProxyURL:      r.options.ProxyURL,
-						ProxySocksURL: r.options.ProxySocksURL,
-						CustomHeaders: r.options.CustomHeaders,
-						CookieJar:     jar,
-						TraceLog:      r.traceLog,
-					}
-				} else if len(t.RequestsDNS) > 0 {
-					template.DNSOptions = &executer.DNSOptions{
-						Debug:    r.options.Debug,
-						Template: t,
-						Writer:   r.output,
-						TraceLog: r.traceLog,
-					}
-				}
-				if template.DNSOptions != nil || template.HTTPOptions != nil {
-					wtlst = append(wtlst, template)
-				}
-			}
-		}
-		wflTemplatesList = append(wflTemplatesList, workflowTemplates{Name: name, Templates: wtlst})
-	}
-
-	return &wflTemplatesList, nil
-}
-
-func resolvePathWithBaseFolder(baseFolder, templateName string) (string, error) {
-	templatePath := path.Join(baseFolder, templateName)
-	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
-		gologger.Debugf("Found template in current directory: %s\n", templatePath)
-		return templatePath, nil
-	}
-
-	return "", fmt.Errorf("no such path found: %s", templateName)
+	return results.Load()
 }

v2/internal/runner/runner.go

@@ -2,103 +2,137 @@ package runner
 
 import (
 	"bufio"
+	"fmt"
 	"os"
-	"regexp"
+	"path"
 	"strings"
+	"time"
 
 	"github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
-	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/hmap/store/hybrid"
-	"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
-	"github.com/projectdiscovery/nuclei/v2/internal/progress"
-	"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
-	"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
-	"github.com/projectdiscovery/nuclei/v2/pkg/collaborator"
-	"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
+	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/remeh/sizedwaitgroup"
+	"github.com/rs/xid"
+	"go.uber.org/atomic"
 	"go.uber.org/ratelimit"
+	"gopkg.in/yaml.v2"
 )
 
 // Runner is a client for running the enumeration process.
 type Runner struct {
-	inputCount int64
-
-	traceLog tracelog.Log
-
-	// output is the output file to write if any
-	output *bufwriter.Writer
-
-	templatesConfig *nucleiConfig
-	// options contains configuration options for runner
-	options *Options
-
-	pf *projectfile.ProjectFile
-
-	// progress tracking
-	progress *progress.Progress
-
-	// output coloring
-	colorizer   colorizer.NucleiColorizer
-	decolorizer *regexp.Regexp
-
-	// rate limiter
-	ratelimiter ratelimit.Limiter
-
-	// input deduplication
-	hm     *hybrid.HybridMap
-	dialer *fastdialer.Dialer
+	hostMap         *hybrid.HybridMap
+	output          output.Writer
+	interactsh      *interactsh.Client
+	inputCount      int64
+	templatesConfig *config.Config
+	options         *types.Options
+	projectFile     *projectfile.ProjectFile
+	catalog         *catalog.Catalog
+	progress        progress.Progress
+	colorizer       aurora.Aurora
+	issuesClient    *reporting.Client
+	severityColors  *colorizer.Colorizer
+	browser         *engine.Browser
+	ratelimiter     ratelimit.Limiter
 }
 
 // New creates a new client for running enumeration process.
-func New(options *Options) (*Runner, error) {
+func New(options *types.Options) (*Runner, error) {
 	runner := &Runner{
-		traceLog: &tracelog.NoopLogger{},
-		options:  options,
+		options: options,
 	}
-	if options.TraceLogFile != "" {
-		fileLog, err := tracelog.NewFileLogger(options.TraceLogFile)
-		if err != nil {
-			return nil, errors.Wrap(err, "could not create file trace logger")
+	if options.UpdateNuclei {
+		if err := updateNucleiVersionToLatest(runner.options.Verbose); err != nil {
+			return nil, err
 		}
-		runner.traceLog = fileLog
+		return nil, nil
+	}
+	if err := runner.updateTemplates(); err != nil {
+		gologger.Warning().Msgf("Could not update templates: %s\n", err)
+	}
+	if options.Headless {
+		browser, err := engine.New(options)
+		if err != nil {
+			return nil, err
+		}
+		runner.browser = browser
 	}
 
-	if err := runner.updateTemplates(); err != nil {
-		gologger.Labelf("Could not update templates: %s\n", err)
+	runner.catalog = catalog.New(runner.options.TemplatesDirectory)
+	var reportingOptions *reporting.Options
+	if options.ReportingConfig != "" {
+		file, err := os.Open(options.ReportingConfig)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not open reporting config file")
+		}
+
+		reportingOptions = &reporting.Options{}
+		if parseErr := yaml.NewDecoder(file).Decode(reportingOptions); parseErr != nil {
+			file.Close()
+			return nil, errors.Wrap(parseErr, "could not parse reporting config file")
+		}
+		file.Close()
+	}
+	if options.DiskExportDirectory != "" {
+		if reportingOptions != nil {
+			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
+		} else {
+			reportingOptions = &reporting.Options{}
+			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
+		}
+	}
+	if options.SarifExport != "" {
+		if reportingOptions != nil {
+			reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}
+		} else {
+			reportingOptions = &reporting.Options{}
+			reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}
+		}
+	}
+	if reportingOptions != nil {
+		client, err := reporting.New(reportingOptions, options.ReportingDB)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create issue reporting client")
+		}
+		runner.issuesClient = client
 	}
 
 	// output coloring
 	useColor := !options.NoColor
-	runner.colorizer = *colorizer.NewNucleiColorizer(aurora.NewAurora(useColor))
-
-	if useColor {
-		// compile a decolorization regex to cleanup file output messages
-		runner.decolorizer = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
-	}
+	runner.colorizer = aurora.NewAurora(useColor)
+	runner.severityColors = colorizer.New(runner.colorizer)
 
 	if options.TemplateList {
 		runner.listAvailableTemplates()
 		os.Exit(0)
 	}
 
-	if (len(options.Templates) == 0 || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
+	if (len(options.Templates) == 0 || !options.NewTemplates || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
 		os.Exit(0)
 	}
-	// Read nucleiignore file if given a templateconfig
-	if runner.templatesConfig != nil {
-		runner.readNucleiIgnoreFile()
-	}
-
-	if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil {
-		gologger.Fatalf("Could not create temporary input file: %s\n", err)
-	} else {
-		runner.hm = hm
+	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create temporary input file")
 	}
+	runner.hostMap = hm
 
 	runner.inputCount = 0
 	dupeCount := 0
@@ -107,7 +141,7 @@ func New(options *Options) (*Runner, error) {
 	if options.Target != "" {
 		runner.inputCount++
 		// nolint:errcheck // ignoring error
-		runner.hm.Set(options.Target, nil)
+		runner.hostMap.Set(options.Target, nil)
 	}
 
 	// Handle stdin
@@ -115,85 +149,91 @@ func New(options *Options) (*Runner, error) {
 		scanner := bufio.NewScanner(os.Stdin)
 		for scanner.Scan() {
 			url := strings.TrimSpace(scanner.Text())
-			// skip empty lines
 			if url == "" {
 				continue
 			}
-
-			// skip dupes
-			if _, ok := runner.hm.Get(url); ok {
+			if _, ok := runner.hostMap.Get(url); ok {
 				dupeCount++
 				continue
 			}
-
 			runner.inputCount++
 			// nolint:errcheck // ignoring error
-			runner.hm.Set(url, nil)
+			runner.hostMap.Set(url, nil)
 		}
 	}
 
 	// Handle taget file
 	if options.Targets != "" {
-		input, err := os.Open(options.Targets)
-		if err != nil {
-			gologger.Fatalf("Could not open targets file '%s': %s\n", options.Targets, err)
+		input, inputErr := os.Open(options.Targets)
+		if inputErr != nil {
+			return nil, errors.Wrap(inputErr, "could not open targets file")
 		}
 		scanner := bufio.NewScanner(input)
 		for scanner.Scan() {
 			url := strings.TrimSpace(scanner.Text())
-			// skip empty lines
 			if url == "" {
 				continue
 			}
-
-			// skip dupes
-			if _, ok := runner.hm.Get(url); ok {
+			if _, ok := runner.hostMap.Get(url); ok {
 				dupeCount++
 				continue
 			}
-
 			runner.inputCount++
 			// nolint:errcheck // ignoring error
-			runner.hm.Set(url, nil)
+			runner.hostMap.Set(url, nil)
 		}
 		input.Close()
 	}
 
 	if dupeCount > 0 {
-		gologger.Labelf("Supplied input was automatically deduplicated (%d removed).", dupeCount)
+		gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", dupeCount)
 	}
 
 	// Create the output file if asked
-	if options.Output != "" {
-		output, errBufWriter := bufwriter.New(options.Output)
-		if errBufWriter != nil {
-			gologger.Fatalf("Could not create output file '%s': %s\n", options.Output, errBufWriter)
-		}
-		runner.output = output
+	outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.JSON, options.Output, options.TraceLogFile)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create output file")
 	}
+	runner.output = outputWriter
 
+	if options.JSON && options.EnableProgressBar {
+		options.StatsJSON = true
+	}
+	if options.StatsJSON {
+		options.EnableProgressBar = true
+	}
 	// Creates the progress tracking object
-	runner.progress = progress.NewProgress(options.EnableProgressBar)
+	var progressErr error
+	runner.progress, progressErr = progress.NewStatsTicker(options.StatsInterval, options.EnableProgressBar, options.StatsJSON, options.Metrics, options.MetricsPort)
+	if progressErr != nil {
+		return nil, progressErr
+	}
 
 	// create project file if requested or load existing one
 	if options.Project {
 		var projectFileErr error
-		runner.pf, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""})
+		runner.projectFile, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""})
 		if projectFileErr != nil {
 			return nil, projectFileErr
 		}
 	}
 
-	// Enable Polling
-	if options.BurpCollaboratorBiid != "" {
-		collaborator.DefaultCollaborator.Collab.AddBIID(options.BurpCollaboratorBiid)
-	}
-
-	// Create Dialer
-	var err error
-	runner.dialer, err = fastdialer.NewDialer(fastdialer.DefaultOptions)
-	if err != nil {
-		return nil, err
+	if !options.NoInteractsh {
+		interactshClient, err := interactsh.New(&interactsh.Options{
+			ServerURL:      options.InteractshURL,
+			CacheSize:      int64(options.InteractionsCacheSize),
+			Eviction:       time.Duration(options.InteractionsEviction) * time.Second,
+			ColldownPeriod: time.Duration(options.InteractionsColldownPeriod) * time.Second,
+			PollDuration:   time.Duration(options.InteractionsPollDuration) * time.Second,
+			Output:         runner.output,
+			IssuesClient:   runner.issuesClient,
+			Progress:       runner.progress,
+		})
+		if err != nil {
+			gologger.Error().Msgf("Could not create interactsh client: %s", err)
+		} else {
+			runner.interactsh = interactshClient
+		}
 	}
 
 	if options.RateLimit > 0 {
@@ -201,7 +241,6 @@ func New(options *Options) (*Runner, error) {
 	} else {
 		runner.ratelimiter = ratelimit.NewUnlimited()
 	}
-
 	return runner, nil
 }
 
@@ -210,106 +249,263 @@ func (r *Runner) Close() {
 	if r.output != nil {
 		r.output.Close()
 	}
-	r.hm.Close()
-	if r.pf != nil {
-		r.pf.Close()
+	r.hostMap.Close()
+	if r.projectFile != nil {
+		r.projectFile.Close()
 	}
+	protocolinit.Close()
 }
 
 // RunEnumeration sets up the input layer for giving input nuclei.
 // binary and runs the actual enumeration
-func (r *Runner) RunEnumeration() {
-	// resolves input templates definitions and any optional exclusion
-	includedTemplates := r.getTemplatesFor(r.options.Templates)
-	excludedTemplates := r.getTemplatesFor(r.options.ExcludedTemplates)
-	// defaults to all templates
-	allTemplates := includedTemplates
+func (r *Runner) RunEnumeration() error {
+	defer r.Close()
 
-	if len(excludedTemplates) > 0 {
-		excludedMap := make(map[string]struct{}, len(excludedTemplates))
-		for _, excl := range excludedTemplates {
-			excludedMap[excl] = struct{}{}
+	// If user asked for new templates to be executed, collect the list from template directory.
+	if r.options.NewTemplates {
+		templatesLoaded, err := r.readNewTemplatesFile()
+		if err != nil {
+			return errors.Wrap(err, "could not get newly added templates")
 		}
-		// rebuild list with only non-excluded templates
-		allTemplates = []string{}
+		r.options.Templates = append(r.options.Templates, templatesLoaded...)
+	}
+	ignoreFile := config.ReadIgnoreFile()
+	r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
+	r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
 
-		for _, incl := range includedTemplates {
-			if _, found := excludedMap[incl]; !found {
-				allTemplates = append(allTemplates, incl)
-			} else {
-				gologger.Warningf("Excluding '%s'", incl)
-			}
+	executerOpts := protocols.ExecuterOptions{
+		Output:       r.output,
+		Options:      r.options,
+		Progress:     r.progress,
+		Catalog:      r.catalog,
+		IssuesClient: r.issuesClient,
+		RateLimiter:  r.ratelimiter,
+		Interactsh:   r.interactsh,
+		ProjectFile:  r.projectFile,
+		Browser:      r.browser,
+	}
+	loaderConfig := loader.Config{
+		Templates:          r.options.Templates,
+		Workflows:          r.options.Workflows,
+		ExcludeTemplates:   r.options.ExcludedTemplates,
+		Tags:               r.options.Tags,
+		ExcludeTags:        r.options.ExcludeTags,
+		IncludeTemplates:   r.options.IncludeTemplates,
+		Authors:            r.options.Author,
+		Severities:         r.options.Severity,
+		IncludeTags:        r.options.IncludeTags,
+		TemplatesDirectory: r.options.TemplatesDirectory,
+		Catalog:            r.catalog,
+		ExecutorOptions:    executerOpts,
+	}
+	store, err := loader.New(&loaderConfig)
+	if err != nil {
+		return errors.Wrap(err, "could not load templates from config")
+	}
+	if r.options.Validate {
+		if !store.ValidateTemplates(r.options.Templates, r.options.Workflows) {
+			return errors.New("an error occurred during templates validation")
 		}
+		gologger.Info().Msgf("All templates validated successfully\n")
+		return nil // exit
+	}
+	store.Load()
+
+	builder := &strings.Builder{}
+	if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" {
+		builder.WriteString(" (")
+
+		if strings.Contains(config.Version, "-dev") {
+			builder.WriteString(r.colorizer.Blue("development").String())
+		} else if config.Version == r.templatesConfig.NucleiLatestVersion {
+			builder.WriteString(r.colorizer.Green("latest").String())
+		} else {
+			builder.WriteString(r.colorizer.Red("outdated").String())
+		}
+		builder.WriteString(")")
+	}
+	messageStr := builder.String()
+	builder.Reset()
+
+	gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr)
+
+	if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" {
+		builder.WriteString(" (")
+
+		if r.templatesConfig.CurrentVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
+			builder.WriteString(r.colorizer.Green("latest").String())
+		} else {
+			builder.WriteString(r.colorizer.Red("outdated").String())
+		}
+		builder.WriteString(")")
+	}
+	messageStr = builder.String()
+	builder.Reset()
+
+	gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.CurrentVersion, messageStr)
+
+	if r.interactsh != nil {
+		gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL)
+	}
+	if len(store.Templates()) > 0 {
+		gologger.Info().Msgf("Templates loaded: %d (New: %d)", len(store.Templates()), r.countNewTemplates())
+	}
+	if len(store.Workflows()) > 0 {
+		gologger.Info().Msgf("Workflows loaded: %d", len(store.Workflows()))
 	}
 
 	// pre-parse all the templates, apply filters
-	availableTemplates, workflowCount := r.getParsedTemplatesFor(allTemplates, r.options.Severity)
-	templateCount := len(availableTemplates)
-	hasWorkflows := workflowCount > 0
+	finalTemplates := []*templates.Template{}
+
+	var unclusteredRequests int64
+	for _, template := range store.Templates() {
+		// workflows will dynamically adjust the totals while running, as
+		// it can't be know in advance which requests will be called
+		if len(template.Workflows) > 0 {
+			continue
+		}
+		unclusteredRequests += int64(template.TotalRequests) * r.inputCount
+	}
+
+	if r.options.VerboseVerbose {
+		for _, template := range store.Templates() {
+			r.logAvailableTemplate(template.Path)
+		}
+		for _, template := range store.Workflows() {
+			r.logAvailableTemplate(template.Path)
+		}
+	}
+	templatesMap := make(map[string]*templates.Template)
+	for _, v := range store.Templates() {
+		templatesMap[v.ID] = v
+	}
+	originalTemplatesCount := len(store.Templates())
+	clusterCount := 0
+	clusters := clusterer.Cluster(templatesMap)
+	for _, cluster := range clusters {
+		if len(cluster) > 1 && !r.options.OfflineHTTP {
+			executerOpts := protocols.ExecuterOptions{
+				Output:       r.output,
+				Options:      r.options,
+				Progress:     r.progress,
+				Catalog:      r.catalog,
+				RateLimiter:  r.ratelimiter,
+				IssuesClient: r.issuesClient,
+				Browser:      r.browser,
+				ProjectFile:  r.projectFile,
+				Interactsh:   r.interactsh,
+			}
+			clusterID := fmt.Sprintf("cluster-%s", xid.New().String())
+
+			finalTemplates = append(finalTemplates, &templates.Template{
+				ID:            clusterID,
+				RequestsHTTP:  cluster[0].RequestsHTTP,
+				Executer:      clusterer.NewExecuter(cluster, &executerOpts),
+				TotalRequests: len(cluster[0].RequestsHTTP),
+			})
+			clusterCount += len(cluster)
+		} else {
+			finalTemplates = append(finalTemplates, cluster...)
+		}
+	}
+	finalTemplates = append(finalTemplates, store.Workflows()...)
+
+	var totalRequests int64
+	for _, t := range finalTemplates {
+		if len(t.Workflows) > 0 {
+			continue
+		}
+		totalRequests += int64(t.TotalRequests) * r.inputCount
+	}
+	if totalRequests < unclusteredRequests {
+		gologger.Info().Msgf("Templates clustered: %d (Reduced %d HTTP Requests)", clusterCount, unclusteredRequests-totalRequests)
+	}
+	workflowCount := len(store.Workflows())
+	templateCount := originalTemplatesCount + workflowCount
 
 	// 0 matches means no templates were found in directory
 	if templateCount == 0 {
-		gologger.Fatalf("Error, no templates were found.\n")
+		return errors.New("no templates were found")
 	}
 
-	gologger.Infof("Using %s rules (%s templates, %s workflows)",
-		r.colorizer.Colorizer.Bold(templateCount).String(),
-		r.colorizer.Colorizer.Bold(templateCount-workflowCount).String(),
-		r.colorizer.Colorizer.Bold(workflowCount).String())
-
-	// precompute total request count
-	var totalRequests int64 = 0
-
-	for _, t := range availableTemplates {
-		switch av := t.(type) {
-		case *templates.Template:
-			totalRequests += (av.GetHTTPRequestCount() + av.GetDNSRequestCount()) * r.inputCount
-		case *workflows.Workflow:
-			// workflows will dynamically adjust the totals while running, as
-			// it can't be know in advance which requests will be called
-		} // nolint:wsl // comment
-	}
-
-	results := atomicboolean.New()
+	results := &atomic.Bool{}
 	wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
-	// Starts polling or ignore
-	collaborator.DefaultCollaborator.Poll()
 
-	if r.inputCount == 0 {
-		gologger.Errorf("Could not find any valid input URLs.")
-	} else if totalRequests > 0 || hasWorkflows {
-		// tracks global progress and captures stdout/stderr until p.Wait finishes
-		p := r.progress
-		p.Init(r.inputCount, templateCount, totalRequests)
+	// tracks global progress and captures stdout/stderr until p.Wait finishes
+	r.progress.Init(r.inputCount, templateCount, totalRequests)
 
-		for _, t := range availableTemplates {
-			wgtemplates.Add()
-			go func(template interface{}) {
-				defer wgtemplates.Done()
-				switch tt := template.(type) {
-				case *templates.Template:
-					for _, request := range tt.RequestsDNS {
-						results.Or(r.processTemplateWithList(p, tt, request))
-					}
-					for _, request := range tt.BulkRequestsHTTP {
-						results.Or(r.processTemplateWithList(p, tt, request))
-					}
-				case *workflows.Workflow:
-					results.Or(r.processWorkflowWithList(p, template.(*workflows.Workflow)))
-				}
-			}(t)
-		}
+	for _, t := range finalTemplates {
+		wgtemplates.Add()
+		go func(template *templates.Template) {
+			defer wgtemplates.Done()
 
-		wgtemplates.Wait()
-		p.Stop()
+			if len(template.Workflows) > 0 {
+				results.CAS(false, r.processWorkflowWithList(template))
+			} else {
+				results.CAS(false, r.processTemplateWithList(template))
+			}
+		}(t)
 	}
+	wgtemplates.Wait()
 
-	if !results.Get() {
-		if r.output != nil {
-			r.output.Close()
-			os.Remove(r.options.Output)
+	if r.interactsh != nil {
+		matched := r.interactsh.Close()
+		if matched {
+			results.CAS(false, true)
 		}
-
-		gologger.Infof("No results found. Happy hacking!")
 	}
+	r.progress.Stop()
+
+	if r.issuesClient != nil {
+		r.issuesClient.Close()
+	}
+	if !results.Load() {
+		gologger.Info().Msgf("No results found. Better luck next time!")
+	}
+	if r.browser != nil {
+		r.browser.Close()
+	}
+	return nil
+}
+
+// readNewTemplatesFile reads newly added templates from directory if it exists
+func (r *Runner) readNewTemplatesFile() ([]string, error) {
+	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	file, err := os.Open(additionsFile)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	templatesList := []string{}
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text == "" {
+			continue
+		}
+		templatesList = append(templatesList, text)
+	}
+	return templatesList, nil
+}
+
+// readNewTemplatesFile reads newly added templates from directory if it exists
+func (r *Runner) countNewTemplates() int {
+	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	file, err := os.Open(additionsFile)
+	if err != nil {
+		return 0
+	}
+	defer file.Close()
+
+	count := 0
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text == "" {
+			continue
+		}
+		count++
+	}
+	return count
 }

v2/internal/runner/templates.go

@@ -1,209 +1,81 @@
 package runner
 
 import (
-	"errors"
+	"bytes"
 	"fmt"
+	"io/ioutil"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/karrick/godirwalk"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"gopkg.in/yaml.v2"
 )
 
-// getTemplatesFor parses the specified input template definitions and returns a list of unique, absolute template paths.
-func (r *Runner) getTemplatesFor(definitions []string) []string {
-	// keeps track of processed dirs and files
-	processed := make(map[string]bool)
-	allTemplates := []string{}
+// parseTemplateFile returns the parsed template file
+func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
 
-	// parses user input, handle file/directory cases and produce a list of unique templates
-	for _, t := range definitions {
-		var absPath string
-
-		var err error
-
-		if strings.Contains(t, "*") {
-			dirs := strings.Split(t, "/")
-			priorDir := strings.Join(dirs[:len(dirs)-1], "/")
-			absPath, err = r.resolvePathIfRelative(priorDir)
-			absPath += "/" + dirs[len(dirs)-1]
-		} else {
-			// resolve and convert relative to absolute path
-			absPath, err = r.resolvePathIfRelative(t)
-		}
-
-		if err != nil {
-			gologger.Errorf("Could not find template file '%s': %s\n", t, err)
-			continue
-		}
-
-		// Template input includes a wildcard
-		if strings.Contains(absPath, "*") {
-			var matches []string
-			matches, err = filepath.Glob(absPath)
-
-			if err != nil {
-				gologger.Labelf("Wildcard found, but unable to glob '%s': %s\n", absPath, err)
-
-				continue
-			}
-
-			// couldn't find templates in directory
-			if len(matches) == 0 {
-				gologger.Labelf("Error, no templates were found with '%s'.\n", absPath)
-				continue
-			} else {
-				gologger.Labelf("Identified %d templates\n", len(matches))
-			}
-
-			for _, match := range matches {
-				if !r.checkIfInNucleiIgnore(match) {
-					processed[match] = true
-
-					allTemplates = append(allTemplates, match)
-				}
-			}
-		} else {
-			// determine file/directory
-			isFile, err := isFilePath(absPath)
-			if err != nil {
-				gologger.Errorf("Could not stat '%s': %s\n", absPath, err)
-				continue
-			}
-			// test for uniqueness
-			if !isNewPath(absPath, processed) {
-				continue
-			}
-			// mark this absolute path as processed
-			// - if it's a file, we'll never process it again
-			// - if it's a dir, we'll never walk it again
-			processed[absPath] = true
-
-			if isFile {
-				allTemplates = append(allTemplates, absPath)
-			} else {
-				matches := []string{}
-
-				// Recursively walk down the Templates directory and run all the template file checks
-				err := directoryWalker(
-					absPath,
-					func(path string, d *godirwalk.Dirent) error {
-						if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
-							if !r.checkIfInNucleiIgnore(path) && isNewPath(path, processed) {
-								matches = append(matches, path)
-								processed[path] = true
-							}
-						}
-						return nil
-					},
-				)
-
-				// directory couldn't be walked
-				if err != nil {
-					gologger.Labelf("Could not find templates in directory '%s': %s\n", absPath, err)
-					continue
-				}
-
-				// couldn't find templates in directory
-				if len(matches) == 0 {
-					gologger.Labelf("Error, no templates were found in '%s'.\n", absPath)
-					continue
-				}
-
-				allTemplates = append(allTemplates, matches...)
-			}
-		}
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, err
 	}
 
-	return allTemplates
-}
-
-// getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered
-// by severity, along with a flag indicating if workflows are present.
-func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities string) (parsedTemplates []interface{}, workflowCount int) {
-	workflowCount = 0
-	severities = strings.ToLower(severities)
-	allSeverities := strings.Split(severities, ",")
-	filterBySeverity := len(severities) > 0
-
-	gologger.Infof("Loading templates...")
-
-	for _, match := range templatePaths {
-		t, err := r.parseTemplateFile(match)
-		switch tp := t.(type) {
-		case *templates.Template:
-			// only include if severity matches or no severity filtering
-			sev := strings.ToLower(tp.Info["severity"])
-			if !filterBySeverity || hasMatchingSeverity(sev, allSeverities) {
-				parsedTemplates = append(parsedTemplates, tp)
-				gologger.Infof("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"]))
-			} else {
-				gologger.Warningf("Excluding template %s due to severity filter (%s not in [%s])", tp.ID, sev, severities)
-			}
-		case *workflows.Workflow:
-			parsedTemplates = append(parsedTemplates, tp)
-			gologger.Infof("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"]))
-			workflowCount++
-		default:
-			gologger.Errorf("Could not parse file '%s': %s\n", match, err)
-		}
+	template := &templates.Template{}
+	err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
+	if err != nil {
+		return nil, err
 	}
-
-	return parsedTemplates, workflowCount
-}
-
-func (r *Runner) parseTemplateFile(file string) (interface{}, error) {
-	// check if it's a template
-	template, errTemplate := templates.Parse(file)
-	if errTemplate == nil {
-		return template, nil
-	}
-
-	// check if it's a workflow
-	workflow, errWorkflow := workflows.Parse(file)
-	if errWorkflow == nil {
-		return workflow, nil
-	}
-
-	if errTemplate != nil {
-		return nil, errTemplate
-	}
-
-	if errWorkflow != nil {
-		return nil, errWorkflow
-	}
-
-	return nil, errors.New("unknown error occurred")
+	return template, nil
 }
 
 func (r *Runner) templateLogMsg(id, name, author, severity string) string {
 	// Display the message for the template
 	message := fmt.Sprintf("[%s] %s (%s)",
-		r.colorizer.Colorizer.BrightBlue(id).String(),
-		r.colorizer.Colorizer.Bold(name).String(),
-		r.colorizer.Colorizer.BrightYellow("@"+author).String())
-
+		r.colorizer.BrightBlue(id).String(),
+		r.colorizer.Bold(name).String(),
+		r.colorizer.BrightYellow(appendAtSignToAuthors(author)).String())
 	if severity != "" {
-		message += " [" + r.colorizer.GetColorizedSeverity(severity) + "]"
+		message += " [" + r.severityColors.Data[severity] + "]"
 	}
-
 	return message
 }
 
+// appendAtSignToAuthors appends @ before each author and returns final string
+func appendAtSignToAuthors(author string) string {
+	authors := strings.Split(author, ",")
+	if len(authors) == 0 {
+		return "@none"
+	}
+	if len(authors) == 1 {
+		if !strings.HasPrefix(authors[0], "@") {
+			return fmt.Sprintf("@%s", authors[0])
+		}
+		return authors[0]
+	}
+	values := make([]string, 0, len(authors))
+	for _, k := range authors {
+		if !strings.HasPrefix(authors[0], "@") {
+			values = append(values, fmt.Sprintf("@%s", k))
+		} else {
+			values = append(values, k)
+		}
+	}
+	return strings.Join(values, ",")
+}
+
 func (r *Runner) logAvailableTemplate(tplPath string) {
 	t, err := r.parseTemplateFile(tplPath)
-	if t != nil {
-		switch tp := t.(type) {
-		case *templates.Template:
-			gologger.Silentf("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"]))
-		case *workflows.Workflow:
-			gologger.Silentf("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"]))
-		default:
-			gologger.Errorf("Could not parse file '%s': %s\n", tplPath, err)
-		}
+	if err != nil {
+		gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
+	} else {
+		gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
 	}
 }
 
@@ -214,11 +86,11 @@ func (r *Runner) listAvailableTemplates() {
 	}
 
 	if _, err := os.Stat(r.templatesConfig.TemplatesDirectory); os.IsNotExist(err) {
-		gologger.Errorf("%s does not exists", r.templatesConfig.TemplatesDirectory)
+		gologger.Error().Msgf("%s does not exists", r.templatesConfig.TemplatesDirectory)
 		return
 	}
 
-	gologger.Silentf(
+	gologger.Print().Msgf(
 		"\nListing available v.%s nuclei templates for %s",
 		r.templatesConfig.CurrentVersion,
 		r.templatesConfig.TemplatesDirectory,
@@ -227,76 +99,25 @@ func (r *Runner) listAvailableTemplates() {
 		r.templatesConfig.TemplatesDirectory,
 		func(path string, d *godirwalk.Dirent) error {
 			if d.IsDir() && path != r.templatesConfig.TemplatesDirectory {
-				gologger.Silentf("\n%s:\n\n", r.colorizer.Colorizer.Bold(r.colorizer.Colorizer.BgBrightBlue(d.Name())).String())
+				gologger.Print().Msgf("\n%s:\n\n", r.colorizer.Bold(r.colorizer.BgBrightBlue(d.Name())).String())
 			} else if strings.HasSuffix(path, ".yaml") {
 				r.logAvailableTemplate(path)
 			}
-
 			return nil
 		},
 	)
-
 	// directory couldn't be walked
 	if err != nil {
-		gologger.Labelf("Could not find templates in directory '%s': %s\n", r.templatesConfig.TemplatesDirectory, err)
+		gologger.Error().Msgf("Could not find templates in directory '%s': %s\n", r.templatesConfig.TemplatesDirectory, err)
 	}
 }
 
-func (r *Runner) resolvePathIfRelative(filePath string) (string, error) {
-	if isRelative(filePath) {
-		newPath, err := r.resolvePath(filePath)
-
-		if err != nil {
-			return "", err
-		}
-
-		return newPath, nil
-	}
-
-	return filePath, nil
-}
-
-func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool {
-	for _, s := range allowedSeverities {
-		if s != "" && strings.HasPrefix(templateSeverity, s) {
-			return true
-		}
-	}
-
-	return false
-}
-
 func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error {
-	err := godirwalk.Walk(fsPath, &godirwalk.Options{
+	return godirwalk.Walk(fsPath, &godirwalk.Options{
 		Callback: callback,
 		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
 			return godirwalk.SkipNode
 		},
 		Unsorted: true,
 	})
-
-	// directory couldn't be walked
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func isFilePath(filePath string) (bool, error) {
-	info, err := os.Stat(filePath)
-	if err != nil {
-		return false, err
-	}
-
-	return info.Mode().IsRegular(), nil
-}
-
-func isNewPath(filePath string, pathMap map[string]bool) bool {
-	if _, already := pathMap[filePath]; already {
-		gologger.Warningf("Skipping already specified path '%s'", filePath)
-		return false
-	}
-
-	return true
 }

v2/internal/runner/update.go

@@ -2,9 +2,12 @@ package runner
 
 import (
 	"archive/zip"
+	"bufio"
 	"bytes"
 	"context"
-	"errors"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -12,12 +15,23 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
+	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
+	"github.com/apex/log"
 	"github.com/blang/semver"
-	"github.com/google/go-github/v32/github"
+	"github.com/google/go-github/github"
+	"github.com/olekukonko/tablewriter"
+	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+
+	"github.com/tj/go-update"
+	"github.com/tj/go-update/progress"
+	githubUpdateStore "github.com/tj/go-update/stores/github"
 )
 
 const (
@@ -25,6 +39,13 @@ const (
 	repoName = "nuclei-templates"
 )
 
+const nucleiIgnoreFile = ".nuclei-ignore"
+
+// nucleiConfigFilename is the filename of nuclei configuration file.
+const nucleiConfigFilename = ".templates-config.json"
+
+var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
+
 // updateTemplates checks if the default list of nuclei-templates
 // exist in the users home directory, if not the latest revision
 // is downloaded from github.
@@ -36,57 +57,111 @@ func (r *Runner) updateTemplates() error {
 	if err != nil {
 		return err
 	}
+	configDir := path.Join(home, "/.config", "/nuclei")
+	_ = os.MkdirAll(configDir, os.ModePerm)
 
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
+	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
 	if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
-		config, readErr := readConfiguration()
+		configuration, readErr := config.ReadConfiguration()
 		if err != nil {
 			return readErr
 		}
-		r.templatesConfig = config
+		r.templatesConfig = configuration
+	}
+
+	ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
+	if r.templatesConfig == nil {
+		currentConfig := &config.Config{
+			TemplatesDirectory: path.Join(home, "nuclei-templates"),
+			IgnoreURL:          ignoreURL,
+			NucleiVersion:      config.Version,
+		}
+		if writeErr := config.WriteConfiguration(currentConfig, false, false); writeErr != nil {
+			return errors.Wrap(writeErr, "could not write template configuration")
+		}
+		r.templatesConfig = currentConfig
+	}
+
+	if r.options.NoUpdateTemplates {
+		return nil
+	}
+	// Check if last checked for nuclei-ignore is more than 1 hours.
+	// and if true, run the check.
+	//
+	// Also at the same time fetch latest version from github to do outdated nuclei
+	// and templates check.
+	checkedIgnore := false
+	if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
+		r.fetchLatestVersionsFromGithub()
+
+		if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" {
+			ignoreURL = r.templatesConfig.IgnoreURL
+		}
+		gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
+
+		checkedIgnore = true
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
+		if reqErr == nil {
+			resp, httpGet := http.DefaultClient.Do(req)
+			if httpGet != nil {
+				if resp != nil && resp.Body != nil {
+					resp.Body.Close()
+				}
+				gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
+			} else {
+				data, _ := ioutil.ReadAll(resp.Body)
+				resp.Body.Close()
+
+				if len(data) > 0 {
+					_ = ioutil.WriteFile(path.Join(configDir, nucleiIgnoreFile), data, 0644)
+				}
+				if r.templatesConfig != nil {
+					err = config.WriteConfiguration(r.templatesConfig, false, true)
+					if err != nil {
+						gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
+					}
+				}
+			}
+		}
+		cancel()
 	}
 
 	ctx := context.Background()
-
-	if r.templatesConfig == nil || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
-		if !r.options.UpdateTemplates {
-			gologger.Labelf("nuclei-templates are not installed, use update-templates flag.\n")
-			return nil
-		}
+	if r.templatesConfig.CurrentVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
+		gologger.Info().Msgf("nuclei-templates are not installed, installing...\n")
 
 		// Use custom location if user has given a template directory
-		if r.options.TemplatesDirectory != "" {
-			home = r.options.TemplatesDirectory
+		r.templatesConfig = &config.Config{
+			TemplatesDirectory: path.Join(home, "nuclei-templates"),
+		}
+		if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") {
+			r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
 		}
-
-		r.templatesConfig = &nucleiConfig{TemplatesDirectory: path.Join(home, "nuclei-templates")}
 
 		// Download the repository and also write the revision to a HEAD file.
 		version, asset, getErr := r.getLatestReleaseFromGithub()
 		if getErr != nil {
 			return getErr
 		}
+		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
 
-		gologger.Verbosef("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory)
-
-		err = r.downloadReleaseAndUnzip(ctx, asset.GetZipballURL())
+		r.fetchLatestVersionsFromGithub() // also fetch latest versions
+		_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
 		if err != nil {
 			return err
 		}
-
 		r.templatesConfig.CurrentVersion = version.String()
 
-		err = r.writeConfiguration(r.templatesConfig)
+		err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
 		if err != nil {
 			return err
 		}
-
-		gologger.Infof("Successfully downloaded nuclei-templates (v%s). Enjoy!\n", version.String())
-
+		gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). GoodLuck!\n", version.String())
 		return nil
 	}
 
-	// Check if last checked is more than 24 hours.
+	// Check if last checked is more than 24 hours and we don't have updateTemplates flag.
 	// If not, return since we don't want to do anything now.
 	if time.Since(r.templatesConfig.LastChecked) < 24*time.Hour && !r.options.UpdateTemplates {
 		return nil
@@ -95,17 +170,14 @@ func (r *Runner) updateTemplates() error {
 	// Get the configuration currently on disk.
 	verText := r.templatesConfig.CurrentVersion
 	indices := reVersion.FindStringIndex(verText)
-
 	if indices == nil {
 		return fmt.Errorf("invalid release found with tag %s", err)
 	}
-
 	if indices[0] > 0 {
 		verText = verText[indices[0]:]
 	}
 
 	oldVersion, err := semver.Make(verText)
-
 	if err != nil {
 		return err
 	}
@@ -116,38 +188,30 @@ func (r *Runner) updateTemplates() error {
 	}
 
 	if version.EQ(oldVersion) {
-		gologger.Infof("Your nuclei-templates are up to date: v%s\n", oldVersion.String())
-		return r.writeConfiguration(r.templatesConfig)
+		return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore)
 	}
 
 	if version.GT(oldVersion) {
-		if !r.options.UpdateTemplates {
-			gologger.Labelf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
-			return r.writeConfiguration(r.templatesConfig)
-		}
+		gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
+		gologger.Info().Msgf("Downloading latest release...")
 
 		if r.options.TemplatesDirectory != "" {
-			home = r.options.TemplatesDirectory
-			r.templatesConfig.TemplatesDirectory = path.Join(home, "nuclei-templates")
+			r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
 		}
-
 		r.templatesConfig.CurrentVersion = version.String()
 
-		gologger.Verbosef("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory)
-
-		err = r.downloadReleaseAndUnzip(ctx, asset.GetZipballURL())
+		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
+		r.fetchLatestVersionsFromGithub()
+		_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
 		if err != nil {
 			return err
 		}
-
-		err = r.writeConfiguration(r.templatesConfig)
+		err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
 		if err != nil {
 			return err
 		}
-
-		gologger.Infof("Successfully updated nuclei-templates (v%s). Enjoy!\n", version.String())
+		gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", version.String())
 	}
-
 	return nil
 }
 
@@ -162,17 +226,13 @@ func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.Repositor
 
 	// Find the most recent version based on semantic versioning.
 	var latestRelease semver.Version
-
 	var latestPublish *github.RepositoryRelease
-
 	for _, release := range rels {
 		verText := release.GetTagName()
 		indices := reVersion.FindStringIndex(verText)
-
 		if indices == nil {
 			return semver.Version{}, nil, fmt.Errorf("invalid release found with tag %s", err)
 		}
-
 		if indices[0] > 0 {
 			verText = verText[indices[0]:]
 		}
@@ -187,85 +247,349 @@ func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.Repositor
 			latestPublish = release
 		}
 	}
-
 	if latestPublish == nil {
 		return semver.Version{}, nil, errors.New("no version found for the templates")
 	}
-
 	return latestRelease, latestPublish, nil
 }
 
 // downloadReleaseAndUnzip downloads and unzips the release in a directory
-func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string) error {
+func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadURL string) (*templateUpdateResults, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
-		return fmt.Errorf("failed to create HTTP request to %s: %s", downloadURL, err)
+		return nil, fmt.Errorf("failed to create HTTP request to %s: %s", downloadURL, err)
 	}
 
 	res, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return fmt.Errorf("failed to download a release file from %s: %s", downloadURL, err)
+		return nil, fmt.Errorf("failed to download a release file from %s: %s", downloadURL, err)
 	}
 	defer res.Body.Close()
-
 	if res.StatusCode != http.StatusOK {
-		return fmt.Errorf("failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode)
+		return nil, fmt.Errorf("failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode)
 	}
 
 	buf, err := ioutil.ReadAll(res.Body)
 	if err != nil {
-		return fmt.Errorf("failed to create buffer for zip file: %s", err)
+		return nil, fmt.Errorf("failed to create buffer for zip file: %s", err)
 	}
 
 	reader := bytes.NewReader(buf)
 	z, err := zip.NewReader(reader, reader.Size())
-
 	if err != nil {
-		return fmt.Errorf("failed to uncompress zip file: %s", err)
+		return nil, fmt.Errorf("failed to uncompress zip file: %s", err)
 	}
 
 	// Create the template folder if it doesn't exists
 	err = os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm)
 	if err != nil {
-		return fmt.Errorf("failed to create template base folder: %s", err)
+		return nil, fmt.Errorf("failed to create template base folder: %s", err)
 	}
 
+	results, err := r.compareAndWriteTemplates(z)
+	if err != nil {
+		return nil, fmt.Errorf("failed to write templates: %s", err)
+	}
+
+	if r.options.Verbose {
+		r.printUpdateChangelog(results, version)
+	}
+	checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
+	err = writeTemplatesChecksum(checksumFile, results.checksums)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not write checksum")
+	}
+
+	// Write the additions to a cached file for new runs.
+	additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
+	buffer := &bytes.Buffer{}
+	for _, addition := range results.additions {
+		buffer.WriteString(addition)
+		buffer.WriteString("\n")
+	}
+	err = ioutil.WriteFile(additionsFile, buffer.Bytes(), os.ModePerm)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not write new additions file")
+	}
+	return results, err
+}
+
+type templateUpdateResults struct {
+	additions     []string
+	deletions     []string
+	modifications []string
+	totalCount    int
+	checksums     map[string]string
+}
+
+// compareAndWriteTemplates compares and returns the stats of a template
+// update operations.
+func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults, error) {
+	results := &templateUpdateResults{
+		checksums: make(map[string]string),
+	}
+
+	// We use file-checksums that are md5 hashes to store the list of files->hashes
+	// that have been downloaded previously.
+	// If the path isn't found in new update after being read from the previous checksum,
+	// it is removed. This allows us fine-grained control over the download process
+	// as well as solves a long problem with nuclei-template updates.
+	checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
+	previousChecksum, _ := readPreviousTemplatesChecksum(checksumFile)
 	for _, file := range z.File {
 		directory, name := filepath.Split(file.Name)
 		if name == "" {
 			continue
 		}
-
 		paths := strings.Split(directory, "/")
 		finalPath := strings.Join(paths[1:], "/")
 
+		if strings.HasPrefix(name, ".") || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") {
+			continue
+		}
+		results.totalCount++
 		templateDirectory := path.Join(r.templatesConfig.TemplatesDirectory, finalPath)
-		err = os.MkdirAll(templateDirectory, os.ModePerm)
-
+		err := os.MkdirAll(templateDirectory, os.ModePerm)
 		if err != nil {
-			return fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err)
+			return nil, fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err)
 		}
 
-		f, err := os.OpenFile(path.Join(templateDirectory, name), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777)
+		templatePath := path.Join(templateDirectory, name)
+
+		isAddition := false
+		if _, statErr := os.Stat(templatePath); os.IsNotExist(statErr) {
+			isAddition = true
+		}
+		f, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777)
 		if err != nil {
 			f.Close()
-			return fmt.Errorf("could not create uncompressed file: %s", err)
+			return nil, fmt.Errorf("could not create uncompressed file: %s", err)
 		}
 
 		reader, err := file.Open()
 		if err != nil {
 			f.Close()
-			return fmt.Errorf("could not open archive to extract file: %s", err)
+			return nil, fmt.Errorf("could not open archive to extract file: %s", err)
 		}
+		hasher := md5.New()
 
-		_, err = io.Copy(f, reader)
+		// Save file and also read into hasher for md5
+		_, err = io.Copy(f, io.TeeReader(reader, hasher))
 		if err != nil {
 			f.Close()
-			return fmt.Errorf("could not write template file: %s", err)
+			return nil, fmt.Errorf("could not write template file: %s", err)
 		}
-
 		f.Close()
+
+		oldChecksum, checksumOK := previousChecksum[templatePath]
+
+		checksum := hex.EncodeToString(hasher.Sum(nil))
+		if isAddition {
+			results.additions = append(results.additions, path.Join(finalPath, name))
+		} else if checksumOK && oldChecksum[0] != checksum {
+			results.modifications = append(results.modifications, path.Join(finalPath, name))
+		}
+		results.checksums[templatePath] = checksum
 	}
 
+	// If we don't find a previous file in new download and it hasn't been
+	// changed on the disk, delete it.
+	for k, v := range previousChecksum {
+		_, ok := results.checksums[k]
+		if !ok && v[0] == v[1] {
+			os.Remove(k)
+			results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(k, r.templatesConfig.TemplatesDirectory), "/"))
+		}
+	}
+	return results, nil
+}
+
+// readPreviousTemplatesChecksum reads the previous checksum file from the disk.
+//
+// It reads two checksums, the first checksum is what we expect and the second is
+// the actual checksum of the file on disk currently.
+func readPreviousTemplatesChecksum(file string) (map[string][2]string, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+
+	checksum := make(map[string][2]string)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text == "" {
+			continue
+		}
+		parts := strings.Split(text, ",")
+		if len(parts) < 2 {
+			continue
+		}
+		values := [2]string{parts[1]}
+
+		f, err := os.Open(parts[0])
+		if err != nil {
+			return nil, err
+		}
+
+		hasher := md5.New()
+		if _, err := io.Copy(hasher, f); err != nil {
+			return nil, err
+		}
+		f.Close()
+
+		values[1] = hex.EncodeToString(hasher.Sum(nil))
+		checksum[parts[0]] = values
+	}
+	return checksum, nil
+}
+
+// writeTemplatesChecksum writes the nuclei-templates checksum data to disk.
+func writeTemplatesChecksum(file string, checksum map[string]string) error {
+	f, err := os.Create(file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	builder := &strings.Builder{}
+	for k, v := range checksum {
+		builder.WriteString(k)
+		builder.WriteString(",")
+		builder.WriteString(v)
+		builder.WriteString("\n")
+
+		if _, checksumErr := f.WriteString(builder.String()); checksumErr != nil {
+			return err
+		}
+		builder.Reset()
+	}
+	return nil
+}
+
+func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version string) {
+	if len(results.additions) > 0 && r.options.Verbose {
+		gologger.Print().Msgf("\nNewly added templates: \n\n")
+
+		for _, addition := range results.additions {
+			gologger.Print().Msgf("%s", addition)
+		}
+	}
+
+	gologger.Print().Msgf("\nNuclei Templates v%s Changelog\n", version)
+	data := [][]string{
+		{strconv.Itoa(results.totalCount), strconv.Itoa(len(results.additions)), strconv.Itoa(len(results.deletions))},
+	}
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Total", "Added", "Removed"})
+	for _, v := range data {
+		table.Append(v)
+	}
+	table.Render()
+}
+
+// fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github
+func (r *Runner) fetchLatestVersionsFromGithub() {
+	nucleiLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei")
+	if err != nil {
+		gologger.Warning().Msgf("Could not fetch latest nuclei release: %s", err)
+	}
+	templatesLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei-templates")
+	if err != nil {
+		gologger.Warning().Msgf("Could not fetch latest nuclei-templates release: %s", err)
+	}
+	if r.templatesConfig != nil {
+		r.templatesConfig.NucleiLatestVersion = nucleiLatest
+		r.templatesConfig.NucleiTemplatesLatestVersion = templatesLatest
+	}
+}
+
+type githubTagData struct {
+	Name string
+}
+
+// githubFetchLatestTagRepo fetches latest tag from github
+// This function was half written by github copilot AI :D.
+func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	url := fmt.Sprintf("https://api.github.com/repos/%s/tags", repo)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return "", err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	var tags []githubTagData
+	err = json.Unmarshal(body, &tags)
+	if err != nil {
+		return "", err
+	}
+	if len(tags) == 0 {
+		return "", fmt.Errorf("no tags found for %s", repo)
+	}
+	return strings.TrimPrefix(tags[0].Name, "v"), nil
+}
+
+// updateNucleiVersionToLatest implements nuclei auto-updation using Github Releases.
+func updateNucleiVersionToLatest(verbose bool) error {
+	if verbose {
+		log.SetLevel(log.DebugLevel)
+	}
+	var command string
+	switch runtime.GOOS {
+	case "windows":
+		command = "nuclei.exe"
+	default:
+		command = "nuclei"
+	}
+	m := &update.Manager{
+		Command: command,
+		Store: &githubUpdateStore.Store{
+			Owner:   "projectdiscovery",
+			Repo:    "nuclei",
+			Version: config.Version,
+		},
+	}
+	releases, err := m.LatestReleases()
+	if err != nil {
+		return errors.Wrap(err, "could not fetch latest release")
+	}
+	if len(releases) == 0 {
+		gologger.Info().Msgf("No new updates found for nuclei engine!")
+		return nil
+	}
+
+	latest := releases[0]
+	var currentOS string
+	switch runtime.GOOS {
+	case "darwin":
+		currentOS = "macOS"
+	default:
+		currentOS = runtime.GOOS
+	}
+	final := latest.FindZip(currentOS, runtime.GOARCH)
+	if final == nil {
+		return fmt.Errorf("no compatible binary found for %s/%s", currentOS, runtime.GOARCH)
+	}
+	tarball, err := final.DownloadProxy(progress.Reader)
+	if err != nil {
+		return errors.Wrap(err, "could not download latest release")
+	}
+	if err := m.Install(tarball); err != nil {
+		return errors.Wrap(err, "could not install latest release")
+	}
+	gologger.Info().Msgf("Successfully updated to Nuclei %s\n", latest.Version)
 	return nil
 }

v2/internal/runner/update_test.go

@@ -0,0 +1,156 @@
+package runner
+
+import (
+	"archive/zip"
+	"context"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
+	gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{})
+
+	baseTemplates, err := ioutil.TempDir("", "old-temp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(baseTemplates)
+
+	err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create write base file")
+
+	err = zipFromDirectory("base.zip", baseTemplates)
+	require.Nil(t, err, "could not create zip from directory")
+	defer os.Remove("base.zip")
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "base.zip")
+	}))
+	defer ts.Close()
+
+	templatesDirectory, err := ioutil.TempDir("", "template-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(templatesDirectory)
+
+	r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}, options: testutils.DefaultOptions}
+	results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
+	require.Nil(t, err, "could not download release and unzip")
+	require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")
+
+	newTempDir, err := ioutil.TempDir("", "new-tmp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(newTempDir)
+
+	err = ioutil.WriteFile(path.Join(newTempDir, "base.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create base file")
+	err = ioutil.WriteFile(path.Join(newTempDir, "new.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create new file")
+
+	err = zipFromDirectory("new.zip", newTempDir)
+	require.Nil(t, err, "could not create new zip from directory")
+	defer os.Remove("new.zip")
+
+	ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "new.zip")
+	}))
+	defer ts2.Close()
+
+	results, err = r.downloadReleaseAndUnzip(context.Background(), "1.0.1", ts2.URL)
+	require.Nil(t, err, "could not download release and unzip")
+
+	require.Equal(t, "new.yaml", results.additions[0], "could not get correct new addition")
+}
+
+func TestDownloadReleaseAndUnzipDeletion(t *testing.T) {
+	gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{})
+
+	baseTemplates, err := ioutil.TempDir("", "old-temp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(baseTemplates)
+
+	err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777)
+	require.Nil(t, err, "could not create write base file")
+
+	err = zipFromDirectory("base.zip", baseTemplates)
+	require.Nil(t, err, "could not create zip from directory")
+	defer os.Remove("base.zip")
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "base.zip")
+	}))
+	defer ts.Close()
+
+	templatesDirectory, err := ioutil.TempDir("", "template-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(templatesDirectory)
+
+	r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}, options: testutils.DefaultOptions}
+
+	results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
+	require.Nil(t, err, "could not download release and unzip")
+	require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")
+
+	newTempDir, err := ioutil.TempDir("", "new-tmp-*")
+	require.Nil(t, err, "could not create temp directory")
+	defer os.RemoveAll(newTempDir)
+
+	err = zipFromDirectory("new.zip", newTempDir)
+	require.Nil(t, err, "could not create new zip from directory")
+	defer os.Remove("new.zip")
+
+	ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "new.zip")
+	}))
+	defer ts2.Close()
+
+	results, err = r.downloadReleaseAndUnzip(context.Background(), "1.0.1", ts2.URL)
+	require.Nil(t, err, "could not download release and unzip")
+
+	require.Equal(t, "base.yaml", results.deletions[0], "could not get correct new deletions")
+}
+
+func zipFromDirectory(zipPath, directory string) error {
+	file, err := os.Create(zipPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	w := zip.NewWriter(file)
+	defer w.Close()
+
+	walker := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		file, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		f, err := w.Create(strings.TrimPrefix(path, directory))
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(f, file)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+	return filepath.Walk(directory, walker)
+}

v2/internal/testutils/integration.go

@@ -0,0 +1,96 @@
+package testutils
+
+import (
+	"net"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+// RunNucleiAndGetResults returns a list of results for a template
+func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
+	cmd := exec.Command("./nuclei", "-t", template, "-target", url, "-silent")
+	if debug {
+		cmd = exec.Command("./nuclei", "-t", template, "-target", url, "-debug")
+		cmd.Stderr = os.Stderr
+	}
+	cmd.Args = append(cmd.Args, extra...)
+
+	data, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+	parts := []string{}
+	items := strings.Split(string(data), "\n")
+	for _, i := range items {
+		if i != "" {
+			parts = append(parts, i)
+		}
+	}
+	return parts, nil
+}
+
+// RunNucleiWorkflowAndGetResults returns a list of results for a workflow
+func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
+	cmd := exec.Command("./nuclei", "-w", template, "-target", url, "-silent")
+	if debug {
+		cmd = exec.Command("./nuclei", "-w", template, "-target", url, "-debug")
+		cmd.Stderr = os.Stderr
+	}
+	cmd.Args = append(cmd.Args, extra...)
+
+	data, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+	parts := []string{}
+	items := strings.Split(string(data), "\n")
+	for _, i := range items {
+		if i != "" {
+			parts = append(parts, i)
+		}
+	}
+	return parts, nil
+}
+
+// TestCase is a single integration test case
+type TestCase interface {
+	// Execute executes a test case and returns any errors if occurred
+	Execute(filePath string) error
+}
+
+// TCPServer creates a new tcp server that returns a response
+type TCPServer struct {
+	URL      string
+	listener net.Listener
+}
+
+// NewTCPServer creates a new TCP server from a handler
+func NewTCPServer(handler func(conn net.Conn)) *TCPServer {
+	server := &TCPServer{}
+
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		panic(err)
+	}
+	server.URL = l.Addr().String()
+	server.listener = l
+
+	go func() {
+		for {
+			// Listen for an incoming connection.
+			conn, err := l.Accept()
+			if err != nil {
+				continue
+			}
+			// Handle connections in a new goroutine.
+			go handler(conn)
+		}
+	}()
+	return server
+}
+
+// Close closes the TCP server
+func (s *TCPServer) Close() {
+	s.listener.Close()
+}

v2/internal/testutils/testutils.go

@@ -0,0 +1,125 @@
+package testutils
+
+import (
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"go.uber.org/ratelimit"
+)
+
+// Init initializes the protocols and their configurations
+func Init(options *types.Options) {
+	_ = protocolinit.Init(options)
+}
+
+// DefaultOptions is the default options structure for nuclei during mocking.
+var DefaultOptions = &types.Options{
+	Metrics:            false,
+	Debug:              false,
+	DebugRequests:      false,
+	DebugResponse:      false,
+	Silent:             false,
+	Version:            false,
+	Verbose:            false,
+	NoColor:            true,
+	UpdateTemplates:    false,
+	JSON:               false,
+	JSONRequests:       false,
+	EnableProgressBar:  false,
+	TemplatesVersion:   false,
+	TemplateList:       false,
+	Stdin:              false,
+	StopAtFirstMatch:   false,
+	NoMeta:             false,
+	Project:            false,
+	MetricsPort:        0,
+	BulkSize:           25,
+	TemplateThreads:    10,
+	Timeout:            5,
+	Retries:            1,
+	RateLimit:          150,
+	ProjectPath:        "",
+	Severity:           []string{},
+	Target:             "",
+	Targets:            "",
+	Output:             "",
+	ProxyURL:           "",
+	ProxySocksURL:      "",
+	TemplatesDirectory: "",
+	TraceLogFile:       "",
+	Templates:          []string{},
+	ExcludedTemplates:  []string{},
+	CustomHeaders:      []string{},
+}
+
+// MockOutputWriter is a mocked output writer.
+type MockOutputWriter struct {
+	aurora          aurora.Aurora
+	RequestCallback func(templateID, url, requestType string, err error)
+	WriteCallback   func(o *output.ResultEvent)
+}
+
+// NewMockOutputWriter creates a new mock output writer
+func NewMockOutputWriter() *MockOutputWriter {
+	return &MockOutputWriter{aurora: aurora.NewAurora(false)}
+}
+
+// Close closes the output writer interface
+func (m *MockOutputWriter) Close() {}
+
+// Colorizer returns the colorizer instance for writer
+func (m *MockOutputWriter) Colorizer() aurora.Aurora {
+	return m.aurora
+}
+
+// Write writes the event to file and/or screen.
+func (m *MockOutputWriter) Write(result *output.ResultEvent) error {
+	if m.WriteCallback != nil {
+		m.WriteCallback(result)
+	}
+	return nil
+}
+
+// Request writes a log the requests trace log
+func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) {
+	if m.RequestCallback != nil {
+		m.RequestCallback(templateID, url, requestType, err)
+	}
+}
+
+// TemplateInfo contains info for a mock executed template.
+type TemplateInfo struct {
+	ID   string
+	Info map[string]interface{}
+	Path string
+}
+
+// NewMockExecuterOptions creates a new mock executeroptions struct
+func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecuterOptions {
+	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
+	executerOpts := &protocols.ExecuterOptions{
+		TemplateID:   info.ID,
+		TemplateInfo: info.Info,
+		TemplatePath: info.Path,
+		Output:       NewMockOutputWriter(),
+		Options:      options,
+		Progress:     progressImpl,
+		ProjectFile:  nil,
+		IssuesClient: nil,
+		Browser:      nil,
+		Catalog:      catalog.New(options.TemplatesDirectory),
+		RateLimiter:  ratelimit.New(options.RateLimit),
+	}
+	return executerOpts
+}
+
+// NoopWriter is a NooP gologger writer.
+type NoopWriter struct{}
+
+// Write writes the data to an output writer.
+func (n *NoopWriter) Write(data []byte, level levels.Level) {}

v2/internal/tracelog/tracelog.go

@@ -1,76 +0,0 @@
-package tracelog
-
-import (
-	"os"
-	"sync"
-
-	jsoniter "github.com/json-iterator/go"
-)
-
-// Log is an interface for logging trace log of all the requests
-type Log interface {
-	// Close closes the log interface flushing data
-	Close()
-	// Request writes a log the requests trace log
-	Request(templateID, url, requestType string, err error)
-}
-
-// NoopLogger is a noop logger that simply does nothing
-type NoopLogger struct{}
-
-// Close closes the log interface flushing data
-func (n *NoopLogger) Close() {}
-
-// Request writes a log the requests trace log
-func (n *NoopLogger) Request(templateID, url, requestType string, err error) {}
-
-// FileLogger is a trace logger that writes request logs to a file.
-type FileLogger struct {
-	encoder *jsoniter.Encoder
-	file    *os.File
-	mutex   *sync.Mutex
-}
-
-// NewFileLogger creates a new file logger structure
-func NewFileLogger(path string) (*FileLogger, error) {
-	file, err := os.Create(path)
-	if err != nil {
-		return nil, err
-	}
-	return &FileLogger{file: file, encoder: jsoniter.NewEncoder(file), mutex: &sync.Mutex{}}, nil
-}
-
-// Close closes the log interface flushing data
-func (f *FileLogger) Close() {
-	f.mutex.Lock()
-	defer f.mutex.Unlock()
-
-	f.file.Close()
-}
-
-// JSONRequest is a trace log request written to file
-type JSONRequest struct {
-	ID    string `json:"id"`
-	URL   string `json:"url"`
-	Error string `json:"error"`
-	Type  string `json:"type"`
-}
-
-// Request writes a log the requests trace log
-func (f *FileLogger) Request(templateID, url, requestType string, err error) {
-	request := &JSONRequest{
-		ID:   templateID,
-		URL:  url,
-		Type: requestType,
-	}
-	if err != nil {
-		request.Error = err.Error()
-	} else {
-		request.Error = "none"
-	}
-
-	f.mutex.Lock()
-	defer f.mutex.Unlock()
-	//nolint:errcheck // We don't need to do anything here
-	f.encoder.Encode(request)
-}

v2/pkg/atomicboolean/bool.go

@@ -1,42 +0,0 @@
-package atomicboolean
-
-import (
-	"sync"
-)
-
-type AtomBool struct {
-	sync.RWMutex
-	flag bool
-}
-
-func New() *AtomBool {
-	return &AtomBool{}
-}
-
-func (b *AtomBool) Or(value bool) {
-	b.Lock()
-	defer b.Unlock()
-
-	b.flag = b.flag || value
-}
-
-func (b *AtomBool) And(value bool) {
-	b.Lock()
-	defer b.Unlock()
-
-	b.flag = b.flag && value
-}
-
-func (b *AtomBool) Set(value bool) {
-	b.Lock()
-	defer b.Unlock()
-
-	b.flag = value
-}
-
-func (b *AtomBool) Get() bool {
-	b.RLock()
-	defer b.RUnlock() //nolint
-
-	return b.flag
-}

v2/pkg/catalog/catalogue.go

@@ -0,0 +1,12 @@
+package catalog
+
+// Catalog is a template catalog helper implementation
+type Catalog struct {
+	templatesDirectory string
+}
+
+// New creates a new Catalog structure using provided input items
+func New(directory string) *Catalog {
+	catalog := &Catalog{templatesDirectory: directory}
+	return catalog
+}

v2/pkg/catalog/config/config.go

@@ -0,0 +1,138 @@
+package config
+
+import (
+	"os"
+	"path"
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+	"gopkg.in/yaml.v2"
+)
+
+// Config contains the internal nuclei engine configuration
+type Config struct {
+	TemplatesDirectory string    `json:"templates-directory,omitempty"`
+	CurrentVersion     string    `json:"current-version,omitempty"`
+	LastChecked        time.Time `json:"last-checked,omitempty"`
+	IgnoreURL          string    `json:"ignore-url,omitempty"`
+	NucleiVersion      string    `json:"nuclei-version,omitempty"`
+	LastCheckedIgnore  time.Time `json:"last-checked-ignore,omitempty"`
+
+	NucleiLatestVersion          string `json:"nuclei-latest-version"`
+	NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"`
+}
+
+// nucleiConfigFilename is the filename of nuclei configuration file.
+const nucleiConfigFilename = ".templates-config.json"
+
+// Version is the current version of nuclei
+const Version = `2.4.2`
+
+func getConfigDetails() (string, error) {
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		return "", errors.Wrap(err, "could not get home directory")
+	}
+	configDir := path.Join(homeDir, "/.config", "/nuclei")
+	_ = os.MkdirAll(configDir, os.ModePerm)
+	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
+	return templatesConfigFile, nil
+}
+
+// ReadConfiguration reads the nuclei configuration file from disk.
+func ReadConfiguration() (*Config, error) {
+	templatesConfigFile, err := getConfigDetails()
+	if err != nil {
+		return nil, err
+	}
+
+	file, err := os.Open(templatesConfigFile)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	config := &Config{}
+	err = jsoniter.NewDecoder(file).Decode(config)
+	if err != nil {
+		return nil, err
+	}
+	return config, nil
+}
+
+// WriteConfiguration writes the updated nuclei configuration to disk
+func WriteConfiguration(config *Config, checked, checkedIgnore bool) error {
+	if config.IgnoreURL == "" {
+		config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
+	}
+	if checked {
+		config.LastChecked = time.Now()
+	}
+	if checkedIgnore {
+		config.LastCheckedIgnore = time.Now()
+	}
+	config.NucleiVersion = Version
+
+	templatesConfigFile, err := getConfigDetails()
+	if err != nil {
+		return err
+	}
+	file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	err = jsoniter.NewEncoder(file).Encode(config)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+const nucleiIgnoreFile = ".nuclei-ignore"
+
+// IgnoreFile is an internal nuclei template blocking configuration file
+type IgnoreFile struct {
+	Tags  []string `yaml:"tags"`
+	Files []string `yaml:"files"`
+}
+
+// ReadIgnoreFile reads the nuclei ignore file returning blocked tags and paths
+func ReadIgnoreFile() IgnoreFile {
+	file, err := os.Open(getIgnoreFilePath())
+	if err != nil {
+		gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err)
+		return IgnoreFile{}
+	}
+	defer file.Close()
+
+	ignore := IgnoreFile{}
+	if err := yaml.NewDecoder(file).Decode(&ignore); err != nil {
+		gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err)
+		return IgnoreFile{}
+	}
+	return ignore
+}
+
+// getIgnoreFilePath returns the ignore file path for the runner
+func getIgnoreFilePath() string {
+	var defIgnoreFilePath string
+
+	home, err := os.UserHomeDir()
+	if err == nil {
+		configDir := path.Join(home, "/.config", "/nuclei")
+		_ = os.MkdirAll(configDir, os.ModePerm)
+
+		defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile)
+		return defIgnoreFilePath
+	}
+	cwd, err := os.Getwd()
+	if err != nil {
+		return defIgnoreFilePath
+	}
+	cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile)
+	return cwdIgnoreFilePath
+}

v2/pkg/catalog/find.go

@@ -0,0 +1,148 @@
+package catalog
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/karrick/godirwalk"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+)
+
+// GetTemplatesPath returns a list of absolute paths for the provided template list.
+func (c *Catalog) GetTemplatesPath(definitions []string) []string {
+	// keeps track of processed dirs and files
+	processed := make(map[string]bool)
+	allTemplates := []string{}
+
+	for _, t := range definitions {
+		paths, err := c.GetTemplatePath(t)
+		if err != nil {
+			gologger.Error().Msgf("Could not find template '%s': %s\n", t, err)
+		}
+		for _, path := range paths {
+			if _, ok := processed[path]; !ok {
+				processed[path] = true
+				allTemplates = append(allTemplates, path)
+			}
+		}
+	}
+	return allTemplates
+}
+
+// GetTemplatePath parses the specified input template path and returns a compiled
+// list of finished absolute paths to the templates evaluating any glob patterns
+// or folders provided as in.
+func (c *Catalog) GetTemplatePath(target string) ([]string, error) {
+	processed := make(map[string]struct{})
+
+	absPath, err := c.convertPathToAbsolute(target)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not find template file")
+	}
+
+	// Template input includes a wildcard
+	if strings.Contains(absPath, "*") {
+		matches, findErr := c.findGlobPathMatches(absPath, processed)
+		if findErr != nil {
+			return nil, errors.Wrap(findErr, "could not find glob matches")
+		}
+		if len(matches) == 0 {
+			return nil, errors.Errorf("no templates found for path")
+		}
+		return matches, nil
+	}
+
+	// Template input is either a file or a directory
+	match, file, err := c.findFileMatches(absPath, processed)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not find file")
+	}
+	if file {
+		if match != "" {
+			return []string{match}, nil
+		}
+		return nil, nil
+	}
+
+	// Recursively walk down the Templates directory and run all
+	// the template file checks
+	matches, err := c.findDirectoryMatches(absPath, processed)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not find directory matches")
+	}
+	if len(matches) == 0 {
+		return nil, errors.Errorf("no templates found in path")
+	}
+	return matches, nil
+}
+
+// convertPathToAbsolute resolves the paths provided to absolute paths
+// before doing any operations on them regardless of them being blob, folders, files, etc.
+func (c *Catalog) convertPathToAbsolute(t string) (string, error) {
+	if strings.Contains(t, "*") {
+		file := path.Base(t)
+		absPath, err := c.ResolvePath(path.Dir(t), "")
+		if err != nil {
+			return "", err
+		}
+		return path.Join(absPath, file), nil
+	}
+	return c.ResolvePath(t, "")
+}
+
+// findGlobPathMatches returns the matched files from a glob path
+func (c *Catalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) {
+	matches, err := filepath.Glob(absPath)
+	if err != nil {
+		return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
+	}
+	results := make([]string, 0, len(matches))
+	for _, match := range matches {
+		if _, ok := processed[match]; !ok {
+			processed[match] = struct{}{}
+			results = append(results, match)
+		}
+	}
+	return results, nil
+}
+
+// findFileMatches finds if a path is an absolute file. If the path
+// is a file, it returns true otherwise false with no errors.
+func (c *Catalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) {
+	info, err := os.Stat(absPath)
+	if err != nil {
+		return "", false, err
+	}
+	if !info.Mode().IsRegular() {
+		return "", false, nil
+	}
+	if _, ok := processed[absPath]; !ok {
+		processed[absPath] = struct{}{}
+		return absPath, true, nil
+	}
+	return "", true, nil
+}
+
+// findDirectoryMatches finds matches for templates from a directory
+func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) {
+	var results []string
+	err := godirwalk.Walk(absPath, &godirwalk.Options{
+		Unsorted: true,
+		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
+			return godirwalk.SkipNode
+		},
+		Callback: func(path string, d *godirwalk.Dirent) error {
+			if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
+				if _, ok := processed[path]; !ok {
+					results = append(results, path)
+					processed[path] = struct{}{}
+				}
+			}
+			return nil
+		},
+	})
+	return results, err
+}

v2/pkg/catalog/loader/filter/path_filter.go

@@ -0,0 +1,45 @@
+package filter
+
+import "github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+
+// PathFilter is a path based template filter
+type PathFilter struct {
+	excludedTemplates          []string
+	alwaysIncludedTemplatesMap map[string]struct{}
+}
+
+// PathFilterConfig contains configuration options for Path based templates Filter
+type PathFilterConfig struct {
+	IncludedTemplates []string
+	ExcludedTemplates []string
+}
+
+// NewPathFilter creates a new path filter from provided config
+func NewPathFilter(config *PathFilterConfig, catalogClient *catalog.Catalog) *PathFilter {
+	filter := &PathFilter{
+		excludedTemplates:          catalogClient.GetTemplatesPath(config.ExcludedTemplates),
+		alwaysIncludedTemplatesMap: make(map[string]struct{}),
+	}
+
+	alwaysIncludeTemplates := catalogClient.GetTemplatesPath(config.IncludedTemplates)
+	for _, tpl := range alwaysIncludeTemplates {
+		filter.alwaysIncludedTemplatesMap[tpl] = struct{}{}
+	}
+	return filter
+}
+
+// Match performs match for path filter on templates and returns final list
+func (p *PathFilter) Match(templates []string) map[string]struct{} {
+	templatesMap := make(map[string]struct{})
+	for _, tpl := range templates {
+		templatesMap[tpl] = struct{}{}
+	}
+	for _, template := range p.excludedTemplates {
+		if _, ok := p.alwaysIncludedTemplatesMap[template]; ok {
+			continue
+		} else {
+			delete(templatesMap, template)
+		}
+	}
+	return templatesMap
+}

v2/pkg/catalog/loader/filter/tag_filter.go

@@ -0,0 +1,178 @@
+package filter
+
+import (
+	"errors"
+	"strings"
+)
+
+// TagFilter is used to filter nuclei templates for tag based execution
+type TagFilter struct {
+	allowedTags map[string]struct{}
+	severities  map[string]struct{}
+	authors     map[string]struct{}
+	block       map[string]struct{}
+	matchAllows map[string]struct{}
+}
+
+// ErrExcluded is returned for execluded templates
+var ErrExcluded = errors.New("the template was excluded")
+
+// Match takes a tag and whether the template was matched from user
+// input and returns true or false using a tag filter.
+//
+// If the tag was specified in deny list, it will not return true
+// unless it is explicitly specified by user in includeTags which is the
+// matchAllows section.
+//
+// It returns true if the tag is specified, or false.
+func (t *TagFilter) Match(tag, author, severity string) (bool, error) {
+	matchedAny := false
+	if len(t.allowedTags) > 0 {
+		_, ok := t.allowedTags[tag]
+		if ok {
+			matchedAny = true
+		}
+	}
+	_, ok := t.block[tag]
+	if ok {
+		if _, allowOk := t.matchAllows[tag]; allowOk {
+			return true, nil
+		}
+		return false, ErrExcluded
+	}
+	if len(t.authors) > 0 {
+		_, ok = t.authors[author]
+		if !ok {
+			return false, nil
+		}
+		matchedAny = true
+	}
+	if len(t.severities) > 0 {
+		_, ok = t.severities[severity]
+		if !ok {
+			return false, nil
+		}
+		matchedAny = true
+	}
+	if len(t.allowedTags) == 0 && len(t.authors) == 0 && len(t.severities) == 0 {
+		return true, nil
+	}
+	return matchedAny, nil
+}
+
+// MatchWithAllowedTags takes an addition list of allowed tags
+// and returns true if the match was successful.
+func (t *TagFilter) MatchWithAllowedTags(allowed []string, tag, author, severity string) (bool, error) {
+	matchedAny := false
+
+	allowedMap := make(map[string]struct{})
+	for _, tag := range allowed {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := allowedMap[val]; !ok {
+				allowedMap[val] = struct{}{}
+			}
+		}
+	}
+	if len(allowedMap) > 0 {
+		_, ok := allowedMap[tag]
+		if ok {
+			matchedAny = true
+		}
+	}
+	_, ok := t.block[tag]
+	if ok && !matchedAny {
+		if _, allowOk := t.matchAllows[tag]; allowOk {
+			return true, nil
+		}
+		return false, ErrExcluded
+	}
+	if len(t.authors) > 0 {
+		_, ok = t.authors[author]
+		if !ok {
+			return false, nil
+		}
+		matchedAny = true
+	}
+	if len(t.severities) > 0 {
+		_, ok = t.severities[severity]
+		if !ok {
+			return false, nil
+		}
+		matchedAny = true
+	}
+	if len(allowedMap) == 0 && len(t.authors) == 0 && len(t.severities) == 0 {
+		return true, nil
+	}
+	return matchedAny, nil
+}
+
+type Config struct {
+	Tags        []string
+	ExcludeTags []string
+	Authors     []string
+	Severities  []string
+	IncludeTags []string
+}
+
+// New returns a tag filter for nuclei tag based execution
+//
+// It takes into account Tags, Severities, Authors, IncludeTags, ExcludeTags.
+func New(config *Config) *TagFilter {
+	filter := &TagFilter{
+		allowedTags: make(map[string]struct{}),
+		authors:     make(map[string]struct{}),
+		severities:  make(map[string]struct{}),
+		block:       make(map[string]struct{}),
+		matchAllows: make(map[string]struct{}),
+	}
+	for _, tag := range config.ExcludeTags {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.block[val]; !ok {
+				filter.block[val] = struct{}{}
+			}
+		}
+	}
+	for _, tag := range config.Severities {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.severities[val]; !ok {
+				filter.severities[val] = struct{}{}
+			}
+		}
+	}
+	for _, tag := range config.Authors {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.authors[val]; !ok {
+				filter.authors[val] = struct{}{}
+			}
+		}
+	}
+	for _, tag := range config.Tags {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.allowedTags[val]; !ok {
+				filter.allowedTags[val] = struct{}{}
+			}
+			delete(filter.block, val)
+		}
+	}
+	for _, tag := range config.IncludeTags {
+		for _, val := range splitCommaTrim(tag) {
+			if _, ok := filter.matchAllows[val]; !ok {
+				filter.matchAllows[val] = struct{}{}
+			}
+			delete(filter.block, val)
+		}
+	}
+	return filter
+}
+
+func splitCommaTrim(value string) []string {
+	if !strings.Contains(value, ",") {
+		return []string{strings.ToLower(value)}
+	}
+	splitted := strings.Split(value, ",")
+	final := make([]string, len(splitted))
+	for i, value := range splitted {
+		final[i] = strings.ToLower(strings.TrimSpace(value))
+	}
+	return final
+}

v2/pkg/catalog/loader/filter/tag_filter_test.go

@@ -0,0 +1,84 @@
+package filter
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestTagBasedFilter(t *testing.T) {
+	config := &Config{
+		Tags: []string{"cves", "2021", "jira"},
+	}
+	filter := New(config)
+
+	t.Run("true", func(t *testing.T) {
+		matched, _ := filter.Match("jira", "pdteam", "low")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("false", func(t *testing.T) {
+		matched, _ := filter.Match("consul", "pdteam", "low")
+		require.False(t, matched, "could not get correct match")
+	})
+	t.Run("not-match-excludes", func(t *testing.T) {
+		config := &Config{
+			ExcludeTags: []string{"dos"},
+		}
+		filter := New(config)
+		matched, err := filter.Match("dos", "pdteam", "low")
+		require.False(t, matched, "could not get correct match")
+		require.Equal(t, ErrExcluded, err, "could not get correct error")
+	})
+	t.Run("match-includes", func(t *testing.T) {
+		config := &Config{
+			Tags:        []string{"cves", "fuzz"},
+			ExcludeTags: []string{"dos", "fuzz"},
+			IncludeTags: []string{"fuzz"},
+		}
+		filter := New(config)
+		matched, err := filter.Match("fuzz", "pdteam", "low")
+		require.Nil(t, err, "could not get match")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("match-author", func(t *testing.T) {
+		config := &Config{
+			Authors: []string{"pdteam"},
+		}
+		filter := New(config)
+		matched, _ := filter.Match("fuzz", "pdteam", "low")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("match-severity", func(t *testing.T) {
+		config := &Config{
+			Severities: []string{"high"},
+		}
+		filter := New(config)
+		matched, _ := filter.Match("fuzz", "pdteam", "high")
+		require.True(t, matched, "could not get correct match")
+	})
+	t.Run("match-exclude-with-tags", func(t *testing.T) {
+		config := &Config{
+			Tags:        []string{"tag"},
+			ExcludeTags: []string{"another"},
+		}
+		filter := New(config)
+		matched, _ := filter.Match("another", "pdteam", "high")
+		require.False(t, matched, "could not get correct match")
+	})
+	t.Run("match-conditions", func(t *testing.T) {
+		config := &Config{
+			Authors:    []string{"pdteam"},
+			Tags:       []string{"jira"},
+			Severities: []string{"high"},
+		}
+		filter := New(config)
+		matched, _ := filter.Match("jira", "pdteam", "high")
+		require.True(t, matched, "could not get correct match")
+		matched, _ = filter.Match("jira", "pdteam", "low")
+		require.False(t, matched, "could not get correct match")
+		matched, _ = filter.Match("jira", "random", "low")
+		require.False(t, matched, "could not get correct match")
+		matched, _ = filter.Match("consul", "random", "low")
+		require.False(t, matched, "could not get correct match")
+	})
+}

v2/pkg/catalog/loader/load/load.go

@@ -0,0 +1,100 @@
+package load
+
+import (
+	"bytes"
+	"errors"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"gopkg.in/yaml.v2"
+)
+
+// Load loads a template by parsing metadata and running
+// all tag and path based filters on the template.
+func Load(templatePath string, workflow bool, customTags []string, tagFilter *filter.TagFilter) (bool, error) {
+	f, err := os.Open(templatePath)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		return false, err
+	}
+
+	template := make(map[string]interface{})
+	err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
+	if err != nil {
+		return false, err
+	}
+
+	info, ok := template["info"]
+	if !ok {
+		return false, errors.New("no template info field provided")
+	}
+	infoMap, ok := info.(map[interface{}]interface{})
+	if !ok {
+		return false, errors.New("could not get info")
+	}
+
+	if _, nameOk := infoMap["name"]; !nameOk {
+		return false, errors.New("no template name field provided")
+	}
+	author, ok := infoMap["author"]
+	if !ok {
+		return false, errors.New("no template author field provided")
+	}
+	severity, ok := infoMap["severity"]
+	if !ok {
+		severity = ""
+	}
+
+	templateTags, ok := infoMap["tags"]
+	if !ok {
+		templateTags = ""
+	}
+	tagStr := types.ToString(templateTags)
+
+	tags := strings.Split(tagStr, ",")
+	severityStr := strings.ToLower(types.ToString(severity))
+	authors := strings.Split(types.ToString(author), ",")
+
+	matched := false
+
+	_, workflowsFound := template["workflows"]
+	if !workflowsFound && workflow {
+		return false, nil
+	}
+	if workflow {
+		return true, nil
+	}
+	for _, tag := range tags {
+		for _, author := range authors {
+			var match bool
+			var err error
+
+			if len(customTags) == 0 {
+				match, err = tagFilter.Match(strings.ToLower(strings.TrimSpace(tag)), strings.ToLower(strings.TrimSpace(author)), severityStr)
+			} else {
+				match, err = tagFilter.MatchWithAllowedTags(customTags, strings.ToLower(strings.TrimSpace(tag)), strings.ToLower(strings.TrimSpace(author)), severityStr)
+			}
+			if err == filter.ErrExcluded {
+				return false, filter.ErrExcluded
+			}
+			if !matched && match {
+				matched = true
+			}
+		}
+	}
+	if !matched {
+		return false, nil
+	}
+	if workflowsFound && !workflow {
+		return false, nil
+	}
+	return true, nil
+}

v2/pkg/catalog/loader/loader.go

@@ -0,0 +1,202 @@
+package loader
+
+import (
+	"strings"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/load"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+)
+
+// Config contains the configuration options for the loader
+type Config struct {
+	Templates        []string
+	Workflows        []string
+	ExcludeTemplates []string
+	IncludeTemplates []string
+
+	Tags        []string
+	ExcludeTags []string
+	Authors     []string
+	Severities  []string
+	IncludeTags []string
+
+	Catalog            *catalog.Catalog
+	ExecutorOptions    protocols.ExecuterOptions
+	TemplatesDirectory string
+}
+
+// Store is a storage for loaded nuclei templates
+type Store struct {
+	tagFilter      *filter.TagFilter
+	pathFilter     *filter.PathFilter
+	config         *Config
+	finalTemplates []string
+
+	templates []*templates.Template
+	workflows []*templates.Template
+
+	preprocessor templates.Preprocessor
+}
+
+// New creates a new template store based on provided configuration
+func New(config *Config) (*Store, error) {
+	// Create a tag filter based on provided configuration
+	store := &Store{
+		config: config,
+		tagFilter: filter.New(&filter.Config{
+			Tags:        config.Tags,
+			ExcludeTags: config.ExcludeTags,
+			Authors:     config.Authors,
+			Severities:  config.Severities,
+			IncludeTags: config.IncludeTags,
+		}),
+		pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{
+			IncludedTemplates: config.IncludeTemplates,
+			ExcludedTemplates: config.ExcludeTemplates,
+		}, config.Catalog),
+	}
+
+	// Handle a case with no templates or workflows, where we use base directory
+	if len(config.Templates) == 0 && len(config.Workflows) == 0 {
+		config.Templates = append(config.Templates, config.TemplatesDirectory)
+	}
+	store.finalTemplates = append(store.finalTemplates, config.Templates...)
+	return store, nil
+}
+
+// Templates returns all the templates in the store
+func (s *Store) Templates() []*templates.Template {
+	return s.templates
+}
+
+// Workflows returns all the workflows in the store
+func (s *Store) Workflows() []*templates.Template {
+	return s.workflows
+}
+
+// RegisterPreprocessor allows a custom preprocessor to be passed to the store to run against templates
+func (s *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
+	s.preprocessor = preprocessor
+}
+
+// Load loads all the templates from a store, performs filtering and returns
+// the complete compiled templates for a nuclei execution configuration.
+func (s *Store) Load() {
+	s.templates = s.LoadTemplates(s.finalTemplates)
+	s.workflows = s.LoadWorkflows(s.config.Workflows)
+}
+
+// ValidateTemplates takes a list of templates and validates them
+// erroring out on discovering any faulty templates.
+func (s *Store) ValidateTemplates(templatesList, workflowsList []string) bool {
+	includedTemplates := s.config.Catalog.GetTemplatesPath(templatesList)
+	includedWorkflows := s.config.Catalog.GetTemplatesPath(workflowsList)
+	templatesMap := s.pathFilter.Match(includedTemplates)
+	workflowsMap := s.pathFilter.Match(includedWorkflows)
+
+	notErrored := true
+	for k := range templatesMap {
+		_, err := s.loadTemplate(k, false)
+		if err != nil {
+			if strings.Contains(err.Error(), "cannot create template executer") {
+				continue
+			}
+			if err == filter.ErrExcluded {
+				continue
+			}
+			notErrored = false
+			gologger.Error().Msgf("Error occurred loading template %s: %s\n", k, err)
+			continue
+		}
+		_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
+		if err != nil {
+			if strings.Contains(err.Error(), "cannot create template executer") {
+				continue
+			}
+			if err == filter.ErrExcluded {
+				continue
+			}
+			notErrored = false
+			gologger.Error().Msgf("Error occurred parsing template %s: %s\n", k, err)
+		}
+	}
+	for k := range workflowsMap {
+		_, err := s.loadTemplate(k, true)
+		if err != nil {
+			if strings.Contains(err.Error(), "cannot create template executer") {
+				continue
+			}
+			if err == filter.ErrExcluded {
+				continue
+			}
+			notErrored = false
+			gologger.Error().Msgf("Error occurred loading workflow %s: %s\n", k, err)
+		}
+		_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
+		if err != nil {
+			if strings.Contains(err.Error(), "cannot create template executer") {
+				continue
+			}
+			if err == filter.ErrExcluded {
+				continue
+			}
+			notErrored = false
+			gologger.Error().Msgf("Error occurred parsing workflow %s: %s\n", k, err)
+		}
+	}
+	return notErrored
+}
+
+// LoadTemplates takes a list of templates and returns paths for them
+func (s *Store) LoadTemplates(templatesList []string) []*templates.Template {
+	includedTemplates := s.config.Catalog.GetTemplatesPath(templatesList)
+	templatesMap := s.pathFilter.Match(includedTemplates)
+
+	loadedTemplates := make([]*templates.Template, 0, len(templatesMap))
+	for k := range templatesMap {
+		loaded, err := s.loadTemplate(k, false)
+		if err != nil {
+			gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
+		}
+		if loaded {
+			parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
+			if err != nil {
+				gologger.Warning().Msgf("Could not parse template %s: %s\n", k, err)
+			} else if parsed != nil {
+				loadedTemplates = append(loadedTemplates, parsed)
+			}
+		}
+	}
+	return loadedTemplates
+}
+
+// LoadWorkflows takes a list of workflows and returns paths for them
+func (s *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
+	includedWorkflows := s.config.Catalog.GetTemplatesPath(s.config.Workflows)
+	workflowsMap := s.pathFilter.Match(includedWorkflows)
+
+	loadedWorkflows := make([]*templates.Template, 0, len(workflowsMap))
+	for k := range workflowsMap {
+		loaded, err := s.loadTemplate(k, true)
+		if err != nil {
+			gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err)
+		}
+		if loaded {
+			parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
+			if err != nil {
+				gologger.Warning().Msgf("Could not parse workflow %s: %s\n", k, err)
+			} else if parsed != nil {
+				loadedWorkflows = append(loadedWorkflows, parsed)
+			}
+		}
+	}
+	return loadedWorkflows
+}
+
+func (s *Store) loadTemplate(templatePath string, workflow bool) (bool, error) {
+	return load.Load(templatePath, workflow, nil, s.tagFilter)
+}

v2/pkg/catalog/loader/loader_test.go

@@ -0,0 +1,40 @@
+package loader
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadTemplates(t *testing.T) {
+	store, err := New(&Config{
+		Templates: []string{"cves/CVE-2021-21315.yaml"},
+	})
+	require.Nil(t, err, "could not load templates")
+	require.Equal(t, []string{"cves/CVE-2021-21315.yaml"}, store.finalTemplates, "could not get correct templates")
+
+	templatesDirectory := "/test"
+	t.Run("blank", func(t *testing.T) {
+		store, err := New(&Config{
+			TemplatesDirectory: templatesDirectory,
+		})
+		require.Nil(t, err, "could not load templates")
+		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
+	})
+	t.Run("only-tags", func(t *testing.T) {
+		store, err := New(&Config{
+			Tags:               []string{"cves"},
+			TemplatesDirectory: templatesDirectory,
+		})
+		require.Nil(t, err, "could not load templates")
+		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
+	})
+	t.Run("tags-with-path", func(t *testing.T) {
+		store, err := New(&Config{
+			Tags:               []string{"cves"},
+			TemplatesDirectory: templatesDirectory,
+		})
+		require.Nil(t, err, "could not load templates")
+		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
+	})
+}

v2/pkg/catalog/path.go

@@ -0,0 +1,45 @@
+package catalog
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+// ResolvePath resolves the path to an absolute one in various ways.
+//
+// It checks if the filename is an absolute path, looks in the current directory
+// or checking the nuclei templates directory. If a second path is given,
+// it also tries to find paths relative to that second path.
+func (c *Catalog) ResolvePath(templateName, second string) (string, error) {
+	if strings.HasPrefix(templateName, "/") || strings.Contains(templateName, ":\\") {
+		return templateName, nil
+	}
+
+	if second != "" {
+		secondBasePath := path.Join(filepath.Dir(second), templateName)
+		if _, err := os.Stat(secondBasePath); !os.IsNotExist(err) {
+			return secondBasePath, nil
+		}
+	}
+
+	curDirectory, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+
+	templatePath := path.Join(curDirectory, templateName)
+	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
+		return templatePath, nil
+	}
+
+	if c.templatesDirectory != "" {
+		templatePath := path.Join(c.templatesDirectory, templateName)
+		if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
+			return templatePath, nil
+		}
+	}
+	return "", fmt.Errorf("no such path found: %s", templateName)
+}

v2/pkg/collaborator/collaborator.go

@@ -1,59 +0,0 @@
-package collaborator
-
-import (
-	"strings"
-	"time"
-
-	"github.com/projectdiscovery/collaborator"
-)
-
-const (
-	PollSeconds           = 5
-	DefaultMaxBufferLimit = 150
-)
-
-var DefaultPollInterval time.Duration = time.Second * time.Duration(PollSeconds)
-
-var DefaultCollaborator BurpCollaborator = BurpCollaborator{Collab: collaborator.NewBurpCollaborator()}
-
-type BurpCollaborator struct {
-	options *Options // unused
-	Collab  *collaborator.BurpCollaborator
-}
-
-type Options struct {
-	BIID           string
-	PollInterval   time.Duration
-	MaxBufferLimit int
-}
-
-func New(options *Options) *BurpCollaborator {
-	collab := collaborator.NewBurpCollaborator()
-	collab.AddBIID(options.BIID)
-	collab.MaxBufferLimit = options.MaxBufferLimit
-	return &BurpCollaborator{Collab: collab, options: options}
-}
-
-func (b *BurpCollaborator) Poll() {
-	// if no valid biids were provided just return
-	if len(b.Collab.BIIDs) > 0 {
-		go b.Collab.PollEach(DefaultPollInterval)
-	}
-}
-
-func (b *BurpCollaborator) Has(s string) bool {
-	for _, r := range b.Collab.RespBuffer {
-		for i := 0; i < len(r.Responses); i++ {
-			// search in dns
-			if strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) {
-				return true
-			}
-			// search in http
-			if strings.Contains(r.Responses[i].Data.RequestDecoded, s) {
-				return true
-			}
-		}
-	}
-
-	return false
-}

v2/pkg/colorizer/colorizer.go

@@ -1,42 +0,0 @@
-package colorizer
-
-import (
-	"strings"
-
-	"github.com/logrusorgru/aurora"
-)
-
-const (
-	fgOrange  uint8  = 208
-	undefined string = "undefined"
-)
-
-// NucleiColorizer contains the severity color mapping
-type NucleiColorizer struct {
-	Colorizer   aurora.Aurora
-	SeverityMap map[string]string
-}
-
-// NewNucleiColorizer initializes the new nuclei colorizer
-func NewNucleiColorizer(colorizer aurora.Aurora) *NucleiColorizer {
-	return &NucleiColorizer{
-		Colorizer: colorizer,
-		SeverityMap: map[string]string{
-			"info":     colorizer.Blue("info").String(),
-			"low":      colorizer.Green("low").String(),
-			"medium":   colorizer.Yellow("medium").String(),
-			"high":     colorizer.Index(fgOrange, "high").String(),
-			"critical": colorizer.Red("critical").String(),
-		},
-	}
-}
-
-// GetColorizedSeverity returns the colorized severity string
-func (r *NucleiColorizer) GetColorizedSeverity(severity string) string {
-	sev := r.SeverityMap[strings.ToLower(severity)]
-	if sev == "" {
-		return undefined
-	}
-
-	return sev
-}

v2/pkg/executer/executer_dns.go

@@ -1,177 +0,0 @@
-package executer
-
-import (
-	"fmt"
-	"os"
-	"regexp"
-
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
-	"github.com/projectdiscovery/nuclei/v2/internal/progress"
-	"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
-	"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	retryabledns "github.com/projectdiscovery/retryabledns"
-	"go.uber.org/ratelimit"
-)
-
-// DNSExecuter is a client for performing a DNS request
-// for a template.
-type DNSExecuter struct {
-	// hm            *hybrid.HybridMap // Unused
-	coloredOutput bool
-	debug         bool
-	jsonOutput    bool
-	jsonRequest   bool
-	noMeta        bool
-	Results       bool
-	traceLog      tracelog.Log
-	dnsClient     *retryabledns.Client
-	template      *templates.Template
-	dnsRequest    *requests.DNSRequest
-	writer        *bufwriter.Writer
-	ratelimiter   ratelimit.Limiter
-
-	colorizer   colorizer.NucleiColorizer
-	decolorizer *regexp.Regexp
-}
-
-// DefaultResolvers contains the list of resolvers known to be trusted.
-var DefaultResolvers = []string{
-	"1.1.1.1:53", // Cloudflare
-	"1.0.0.1:53", // Cloudflare
-	"8.8.8.8:53", // Google
-	"8.8.4.4:53", // Google
-}
-
-// DNSOptions contains configuration options for the DNS executer.
-type DNSOptions struct {
-	ColoredOutput bool
-	Debug         bool
-	JSON          bool
-	JSONRequests  bool
-	NoMeta        bool
-	TraceLog      tracelog.Log
-	Template      *templates.Template
-	DNSRequest    *requests.DNSRequest
-	Writer        *bufwriter.Writer
-
-	Colorizer   colorizer.NucleiColorizer
-	Decolorizer *regexp.Regexp
-	RateLimiter ratelimit.Limiter
-}
-
-// NewDNSExecuter creates a new DNS executer from a template
-// and a DNS request query.
-func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
-	dnsClient := retryabledns.New(DefaultResolvers, options.DNSRequest.Retries)
-
-	executer := &DNSExecuter{
-		debug:         options.Debug,
-		noMeta:        options.NoMeta,
-		jsonOutput:    options.JSON,
-		traceLog:      options.TraceLog,
-		jsonRequest:   options.JSONRequests,
-		dnsClient:     dnsClient,
-		template:      options.Template,
-		dnsRequest:    options.DNSRequest,
-		writer:        options.Writer,
-		coloredOutput: options.ColoredOutput,
-		colorizer:     options.Colorizer,
-		decolorizer:   options.Decolorizer,
-		ratelimiter:   options.RateLimiter,
-	}
-
-	return executer
-}
-
-// ExecuteDNS executes the DNS request on a URL
-func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, reqURL string) *Result {
-	result := &Result{}
-
-	// Parse the URL and return domain if URL.
-	var domain string
-	if isURL(reqURL) {
-		domain = extractDomain(reqURL)
-	} else {
-		domain = reqURL
-	}
-
-	// Compile each request for the template based on the URL
-	compiledRequest, err := e.dnsRequest.MakeDNSRequest(domain)
-	if err != nil {
-		e.traceLog.Request(e.template.ID, domain, "dns", err)
-		result.Error = errors.Wrap(err, "could not make dns request")
-		p.Drop(1)
-		return result
-	}
-	e.traceLog.Request(e.template.ID, domain, "dns", nil)
-
-	if e.debug {
-		gologger.Infof("Dumped DNS request for %s (%s)\n\n", reqURL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s\n", compiledRequest.String())
-	}
-
-	// Send the request to the target servers
-	resp, err := e.dnsClient.Do(compiledRequest)
-	if err != nil {
-		result.Error = errors.Wrap(err, "could not send dns request")
-		p.Drop(1)
-		return result
-	}
-	p.Update()
-
-	gologger.Verbosef("Sent for [%s] to %s\n", "dns-request", e.template.ID, reqURL)
-
-	if e.debug {
-		gologger.Infof("Dumped DNS response for %s (%s)\n\n", reqURL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s\n", resp.String())
-	}
-
-	matcherCondition := e.dnsRequest.GetMatchersCondition()
-
-	for _, matcher := range e.dnsRequest.Matchers {
-		// Check if the matcher matched
-		if !matcher.MatchDNS(resp) {
-			// If the condition is AND we haven't matched, return.
-			if matcherCondition == matchers.ANDCondition {
-				return result
-			}
-		} else {
-			// If the matcher has matched, and its an OR
-			// write the first output then move to next matcher.
-			if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 {
-				e.writeOutputDNS(domain, compiledRequest, resp, matcher, nil)
-				result.GotResults = true
-			}
-		}
-	}
-
-	// All matchers have successfully completed so now start with the
-	// next task which is extraction of input from matchers.
-	var extractorResults []string
-
-	for _, extractor := range e.dnsRequest.Extractors {
-		for match := range extractor.ExtractDNS(resp) {
-			if !extractor.Internal {
-				extractorResults = append(extractorResults, match)
-			}
-		}
-	}
-
-	// Write a final string of output if matcher type is
-	// AND or if we have extractors for the mechanism too.
-	if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
-		e.writeOutputDNS(domain, compiledRequest, resp, nil, extractorResults)
-
-		result.GotResults = true
-	}
-
-	return result
-}
-
-// Close closes the dns executer for a template.
-func (e *DNSExecuter) Close() {}

v2/pkg/executer/executer_http.go

@@ -1,710 +0,0 @@
-package executer
-
-import (
-	"bytes"
-	"context"
-	"crypto/tls"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net"
-	"net/http"
-	"net/http/cookiejar"
-	"net/http/httputil"
-	"net/url"
-	"os"
-	"regexp"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/fastdialer/fastdialer"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
-	"github.com/projectdiscovery/nuclei/v2/internal/progress"
-	"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
-	"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
-	"github.com/projectdiscovery/nuclei/v2/pkg/generators"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	projetctfile "github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/rawhttp"
-	"github.com/projectdiscovery/retryablehttp-go"
-	"github.com/remeh/sizedwaitgroup"
-	"go.uber.org/ratelimit"
-	"golang.org/x/net/proxy"
-)
-
-const (
-	two                   = 2
-	ten                   = 10
-	defaultMaxWorkers     = 150
-	defaultMaxHistorydata = 150
-)
-
-// HTTPExecuter is client for performing HTTP requests
-// for a template.
-type HTTPExecuter struct {
-	pf               *projetctfile.ProjectFile
-	customHeaders    requests.CustomHeaders
-	colorizer        colorizer.NucleiColorizer
-	httpClient       *retryablehttp.Client
-	rawHTTPClient    *rawhttp.Client
-	template         *templates.Template
-	bulkHTTPRequest  *requests.BulkHTTPRequest
-	writer           *bufwriter.Writer
-	CookieJar        *cookiejar.Jar
-	traceLog         tracelog.Log
-	decolorizer      *regexp.Regexp
-	coloredOutput    bool
-	debug            bool
-	Results          bool
-	jsonOutput       bool
-	jsonRequest      bool
-	noMeta           bool
-	stopAtFirstMatch bool
-	ratelimiter      ratelimit.Limiter
-}
-
-// HTTPOptions contains configuration options for the HTTP executer.
-type HTTPOptions struct {
-	CustomHeaders    requests.CustomHeaders
-	ProxyURL         string
-	ProxySocksURL    string
-	Template         *templates.Template
-	BulkHTTPRequest  *requests.BulkHTTPRequest
-	Writer           *bufwriter.Writer
-	Timeout          int
-	Retries          int
-	CookieJar        *cookiejar.Jar
-	Colorizer        *colorizer.NucleiColorizer
-	Decolorizer      *regexp.Regexp
-	TraceLog         tracelog.Log
-	Debug            bool
-	JSON             bool
-	JSONRequests     bool
-	NoMeta           bool
-	CookieReuse      bool
-	ColoredOutput    bool
-	StopAtFirstMatch bool
-	PF               *projetctfile.ProjectFile
-	RateLimiter      ratelimit.Limiter
-	Dialer           *fastdialer.Dialer
-}
-
-// NewHTTPExecuter creates a new HTTP executer from a template
-// and a HTTP request query.
-func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
-	var (
-		proxyURL *url.URL
-		err      error
-	)
-
-	if options.ProxyURL != "" {
-		proxyURL, err = url.Parse(options.ProxyURL)
-	}
-
-	if err != nil {
-		return nil, err
-	}
-
-	// Create the HTTP Client
-	client := makeHTTPClient(proxyURL, options)
-	// nolint:bodyclose // false positive there is no body to close yet
-	client.CheckRetry = retryablehttp.HostSprayRetryPolicy()
-
-	if options.CookieJar != nil {
-		client.HTTPClient.Jar = options.CookieJar
-	} else if options.CookieReuse {
-		jar, err := cookiejar.New(nil)
-		if err != nil {
-			return nil, err
-		}
-		client.HTTPClient.Jar = jar
-	}
-
-	// initiate raw http client
-	rawClient := rawhttp.NewClient(rawhttp.DefaultOptions)
-
-	executer := &HTTPExecuter{
-		debug:            options.Debug,
-		jsonOutput:       options.JSON,
-		jsonRequest:      options.JSONRequests,
-		noMeta:           options.NoMeta,
-		httpClient:       client,
-		rawHTTPClient:    rawClient,
-		traceLog:         options.TraceLog,
-		template:         options.Template,
-		bulkHTTPRequest:  options.BulkHTTPRequest,
-		writer:           options.Writer,
-		customHeaders:    options.CustomHeaders,
-		CookieJar:        options.CookieJar,
-		coloredOutput:    options.ColoredOutput,
-		colorizer:        *options.Colorizer,
-		decolorizer:      options.Decolorizer,
-		stopAtFirstMatch: options.StopAtFirstMatch,
-		pf:               options.PF,
-		ratelimiter:      options.RateLimiter,
-	}
-
-	return executer, nil
-}
-
-func (e *HTTPExecuter) ExecuteRaceRequest(reqURL string) *Result {
-	result := &Result{
-		Matches:     make(map[string]interface{}),
-		Extractions: make(map[string]interface{}),
-	}
-
-	dynamicvalues := make(map[string]interface{})
-
-	// verify if the URL is already being processed
-	if e.bulkHTTPRequest.HasGenerator(reqURL) {
-		return result
-	}
-
-	e.bulkHTTPRequest.CreateGenerator(reqURL)
-
-	// Workers that keeps enqueuing new requests
-	maxWorkers := e.bulkHTTPRequest.RaceNumberRequests
-	swg := sizedwaitgroup.New(maxWorkers)
-	for i := 0; i < e.bulkHTTPRequest.RaceNumberRequests; i++ {
-		swg.Add()
-		// base request
-		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
-			result.Error = err
-			return result
-		}
-		go func(httpRequest *requests.HTTPRequest) {
-			defer swg.Done()
-
-			// If the request was built correctly then execute it
-			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
-			if err != nil {
-				result.Error = errors.Wrap(err, "could not handle http request")
-			}
-		}(request)
-	}
-
-	swg.Wait()
-
-	return result
-}
-
-func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string) *Result {
-	result := &Result{
-		Matches:     make(map[string]interface{}),
-		Extractions: make(map[string]interface{}),
-	}
-
-	dynamicvalues := make(map[string]interface{})
-
-	// verify if the URL is already being processed
-	if e.bulkHTTPRequest.HasGenerator(reqURL) {
-		return result
-	}
-
-	remaining := e.bulkHTTPRequest.GetRequestCount()
-	e.bulkHTTPRequest.CreateGenerator(reqURL)
-
-	// Workers that keeps enqueuing new requests
-	maxWorkers := e.bulkHTTPRequest.Threads
-	swg := sizedwaitgroup.New(maxWorkers)
-	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
-		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
-			result.Error = err
-			p.Drop(remaining)
-		} else {
-			swg.Add()
-			go func(httpRequest *requests.HTTPRequest) {
-				defer swg.Done()
-
-				e.ratelimiter.Take()
-
-				// If the request was built correctly then execute it
-				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
-				if err != nil {
-					e.traceLog.Request(e.template.ID, reqURL, "http", err)
-					result.Error = errors.Wrap(err, "could not handle http request")
-					p.Drop(remaining)
-				} else {
-					e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-				}
-			}(request)
-		}
-		p.Update()
-		e.bulkHTTPRequest.Increment(reqURL)
-	}
-	swg.Wait()
-
-	return result
-}
-
-func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result {
-	result := &Result{
-		Matches:     make(map[string]interface{}),
-		Extractions: make(map[string]interface{}),
-	}
-
-	dynamicvalues := make(map[string]interface{})
-
-	// verify if the URL is already being processed
-	if e.bulkHTTPRequest.HasGenerator(reqURL) {
-		return result
-	}
-
-	e.bulkHTTPRequest.CreateGenerator(reqURL)
-
-	// need to extract the target from the url
-	URL, err := url.Parse(reqURL)
-	if err != nil {
-		return result
-	}
-
-	pipeOptions := rawhttp.DefaultPipelineOptions
-	pipeOptions.Host = URL.Host
-	pipeOptions.MaxConnections = 1
-	if e.bulkHTTPRequest.PipelineConcurrentConnections > 0 {
-		pipeOptions.MaxConnections = e.bulkHTTPRequest.PipelineConcurrentConnections
-	}
-	if e.bulkHTTPRequest.PipelineRequestsPerConnection > 0 {
-		pipeOptions.MaxPendingRequests = e.bulkHTTPRequest.PipelineRequestsPerConnection
-	}
-	pipeclient := rawhttp.NewPipelineClient(pipeOptions)
-
-	// defaultMaxWorkers should be a sufficient value to keep queues always full
-	maxWorkers := defaultMaxWorkers
-	// in case the queue is bigger increase the workers
-	if pipeOptions.MaxPendingRequests > maxWorkers {
-		maxWorkers = pipeOptions.MaxPendingRequests
-	}
-	swg := sizedwaitgroup.New(maxWorkers)
-	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
-		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
-			result.Error = err
-		} else {
-			swg.Add()
-			go func(httpRequest *requests.HTTPRequest) {
-				defer swg.Done()
-
-				// HTTP pipelining ignores rate limit
-				// If the request was built correctly then execute it
-				request.Pipeline = true
-				request.PipelineClient = pipeclient
-				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
-				if err != nil {
-					e.traceLog.Request(e.template.ID, reqURL, "http", err)
-					result.Error = errors.Wrap(err, "could not handle http request")
-				} else {
-					e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-				}
-				request.PipelineClient = nil
-			}(request)
-		}
-
-		e.bulkHTTPRequest.Increment(reqURL)
-	}
-	swg.Wait()
-	return result
-}
-
-// ExecuteHTTP executes the HTTP request on a URL
-func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result {
-	// verify if pipeline was requested
-	if e.bulkHTTPRequest.Pipeline {
-		return e.ExecuteTurboHTTP(reqURL)
-	}
-
-	// verify if a basic race condition was requested
-	if e.bulkHTTPRequest.Race && e.bulkHTTPRequest.RaceNumberRequests > 0 {
-		return e.ExecuteRaceRequest(reqURL)
-	}
-
-	// verify if parallel elaboration was requested
-	if e.bulkHTTPRequest.Threads > 0 {
-		return e.ExecuteParallelHTTP(p, reqURL)
-	}
-
-	var requestNumber int
-
-	result := &Result{
-		Matches:     make(map[string]interface{}),
-		Extractions: make(map[string]interface{}),
-		historyData: make(map[string]interface{}),
-	}
-
-	dynamicvalues := make(map[string]interface{})
-
-	// verify if the URL is already being processed
-	if e.bulkHTTPRequest.HasGenerator(reqURL) {
-		return result
-	}
-
-	remaining := e.bulkHTTPRequest.GetRequestCount()
-	e.bulkHTTPRequest.CreateGenerator(reqURL)
-
-	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
-		requestNumber++
-		httpRequest, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
-		if err != nil {
-			result.Error = err
-			p.Drop(remaining)
-		} else {
-			e.ratelimiter.Take()
-			// If the request was built correctly then execute it
-			format := "%s_" + strconv.Itoa(requestNumber)
-			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, format)
-			if err != nil {
-				result.Error = errors.Wrap(err, "could not handle http request")
-				p.Drop(remaining)
-				e.traceLog.Request(e.template.ID, reqURL, "http", err)
-			} else {
-				e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-			}
-		}
-		p.Update()
-
-		// Check if has to stop processing at first valid result
-		if e.stopAtFirstMatch && result.GotResults {
-			p.Drop(remaining)
-			break
-		}
-
-		// move always forward with requests
-		e.bulkHTTPRequest.Increment(reqURL)
-		remaining--
-	}
-	gologger.Verbosef("Sent for [%s] to %s\n", "http-request", e.template.ID, reqURL)
-	return result
-}
-
-func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, format string) error {
-	e.setCustomHeaders(request)
-
-	var (
-		resp          *http.Response
-		err           error
-		dumpedRequest []byte
-		fromcache     bool
-	)
-
-	if e.debug || e.pf != nil {
-		dumpedRequest, err = requests.Dump(request, reqURL)
-		if err != nil {
-			return err
-		}
-	}
-
-	if e.debug {
-		gologger.Infof("Dumped HTTP request for %s (%s)\n\n", reqURL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
-	}
-
-	timeStart := time.Now()
-
-	if request.Pipeline {
-		resp, err = request.PipelineClient.DoRaw(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)))
-		if err != nil {
-			if resp != nil {
-				resp.Body.Close()
-			}
-			e.traceLog.Request(e.template.ID, reqURL, "http", err)
-			return err
-		}
-		e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-	} else if request.Unsafe {
-		// rawhttp
-		// burp uses "\r\n" as new line character
-		request.RawRequest.Data = strings.ReplaceAll(request.RawRequest.Data, "\n", "\r\n")
-		options := e.rawHTTPClient.Options
-		options.AutomaticContentLength = request.AutomaticContentLengthHeader
-		options.AutomaticHostHeader = request.AutomaticHostHeader
-		options.FollowRedirects = request.FollowRedirects
-		resp, err = e.rawHTTPClient.DoRawWithOptions(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)), options)
-		if err != nil {
-			if resp != nil {
-				resp.Body.Close()
-			}
-			e.traceLog.Request(e.template.ID, reqURL, "http", err)
-			return err
-		}
-		e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-	} else {
-		// if nuclei-project is available check if the request was already sent previously
-		if e.pf != nil {
-			// if unavailable fail silently
-			fromcache = true
-			// nolint:bodyclose // false positive the response is generated at runtime
-			resp, err = e.pf.Get(dumpedRequest)
-			if err != nil {
-				fromcache = false
-			}
-		}
-
-		// retryablehttp
-		if resp == nil {
-			resp, err = e.httpClient.Do(request.Request)
-			if err != nil {
-				if resp != nil {
-					resp.Body.Close()
-				}
-				e.traceLog.Request(e.template.ID, reqURL, "http", err)
-				return err
-			}
-			e.traceLog.Request(e.template.ID, reqURL, "http", nil)
-		}
-	}
-
-	duration := time.Since(timeStart)
-
-	// Dump response - Step 1 - Decompression not yet handled
-	var dumpedResponse []byte
-	if e.debug {
-		var dumpErr error
-		dumpedResponse, dumpErr = httputil.DumpResponse(resp, true)
-		if dumpErr != nil {
-			return errors.Wrap(dumpErr, "could not dump http response")
-		}
-	}
-
-	data, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		_, copyErr := io.Copy(ioutil.Discard, resp.Body)
-		if copyErr != nil {
-			resp.Body.Close()
-			return copyErr
-		}
-
-		resp.Body.Close()
-
-		return errors.Wrap(err, "could not read http body")
-	}
-
-	resp.Body.Close()
-
-	// net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request
-	// so in case we have to manually do it
-	dataOrig := data
-	data, err = requests.HandleDecompression(request, data)
-	if err != nil {
-		return errors.Wrap(err, "could not decompress http body")
-	}
-
-	// Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation)
-	if e.debug {
-		dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data)
-		gologger.Infof("Dumped HTTP response for %s (%s)\n\n", reqURL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
-	}
-
-	// if nuclei-project is enabled store the response if not previously done
-	if e.pf != nil && !fromcache {
-		err := e.pf.Set(dumpedRequest, resp, data)
-		if err != nil {
-			return errors.Wrap(err, "could not store in project file")
-		}
-	}
-
-	// Convert response body from []byte to string with zero copy
-	body := unsafeToString(data)
-
-	headers := headersToString(resp.Header)
-
-	// store for internal purposes the DSL matcher data
-	// hardcode stopping storing data after defaultMaxHistorydata items
-	if len(result.historyData) < defaultMaxHistorydata {
-		result.Lock()
-		result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format))
-		result.Unlock()
-	}
-
-	matcherCondition := e.bulkHTTPRequest.GetMatchersCondition()
-	for _, matcher := range e.bulkHTTPRequest.Matchers {
-		// Check if the matcher matched
-		if !matcher.Match(resp, body, headers, duration, result.historyData) {
-			// If the condition is AND we haven't matched, try next request.
-			if matcherCondition == matchers.ANDCondition {
-				return nil
-			}
-		} else {
-			// If the matcher has matched, and its an OR
-			// write the first output then move to next matcher.
-			if matcherCondition == matchers.ORCondition {
-				result.Lock()
-				result.Matches[matcher.Name] = nil
-				// probably redundant but ensures we snapshot current payload values when matchers are valid
-				result.Meta = request.Meta
-				result.GotResults = true
-				result.Unlock()
-				e.writeOutputHTTP(request, resp, body, matcher, nil, result.Meta, reqURL)
-			}
-		}
-	}
-
-	// All matchers have successfully completed so now start with the
-	// next task which is extraction of input from matchers.
-	var extractorResults, outputExtractorResults []string
-
-	for _, extractor := range e.bulkHTTPRequest.Extractors {
-		for match := range extractor.Extract(resp, body, headers) {
-			if _, ok := dynamicvalues[extractor.Name]; !ok {
-				dynamicvalues[extractor.Name] = match
-			}
-
-			extractorResults = append(extractorResults, match)
-
-			if !extractor.Internal {
-				outputExtractorResults = append(outputExtractorResults, match)
-			}
-		}
-		// probably redundant but ensures we snapshot current payload values when extractors are valid
-		result.Lock()
-		result.Meta = request.Meta
-		result.Extractions[extractor.Name] = extractorResults
-		result.Unlock()
-	}
-
-	// Write a final string of output if matcher type is
-	// AND or if we have extractors for the mechanism too.
-	if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition {
-		e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, result.Meta, reqURL)
-		result.Lock()
-		result.GotResults = true
-		result.Unlock()
-	}
-
-	return nil
-}
-
-// Close closes the http executer for a template.
-func (e *HTTPExecuter) Close() {}
-
-// makeHTTPClient creates a http client
-func makeHTTPClient(proxyURL *url.URL, options *HTTPOptions) *retryablehttp.Client {
-	// Multiple Host
-	retryablehttpOptions := retryablehttp.DefaultOptionsSpraying
-	disableKeepAlives := true
-	maxIdleConns := 0
-	maxConnsPerHost := 0
-	maxIdleConnsPerHost := -1
-
-	if options.BulkHTTPRequest.Threads > 0 {
-		// Single host
-		retryablehttpOptions = retryablehttp.DefaultOptionsSingle
-		disableKeepAlives = false
-		maxIdleConnsPerHost = 500
-		maxConnsPerHost = 500
-	}
-
-	retryablehttpOptions.RetryWaitMax = 10 * time.Second
-	retryablehttpOptions.RetryMax = options.Retries
-	followRedirects := options.BulkHTTPRequest.Redirects
-	maxRedirects := options.BulkHTTPRequest.MaxRedirects
-
-	transport := &http.Transport{
-		DialContext:         options.Dialer.Dial,
-		MaxIdleConns:        maxIdleConns,
-		MaxIdleConnsPerHost: maxIdleConnsPerHost,
-		MaxConnsPerHost:     maxConnsPerHost,
-		TLSClientConfig: &tls.Config{
-			Renegotiation:      tls.RenegotiateOnceAsClient,
-			InsecureSkipVerify: true,
-		},
-		DisableKeepAlives: disableKeepAlives,
-	}
-
-	// Attempts to overwrite the dial function with the socks proxied version
-	if options.ProxySocksURL != "" {
-		var proxyAuth *proxy.Auth
-
-		socksURL, err := url.Parse(options.ProxySocksURL)
-
-		if err == nil {
-			proxyAuth = &proxy.Auth{}
-			proxyAuth.User = socksURL.User.Username()
-			proxyAuth.Password, _ = socksURL.User.Password()
-		}
-
-		dialer, err := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct)
-		dc := dialer.(interface {
-			DialContext(ctx context.Context, network, addr string) (net.Conn, error)
-		})
-
-		if err == nil {
-			transport.DialContext = dc.DialContext
-		}
-	}
-
-	if proxyURL != nil {
-		transport.Proxy = http.ProxyURL(proxyURL)
-	}
-
-	return retryablehttp.NewWithHTTPClient(&http.Client{
-		Transport:     transport,
-		Timeout:       time.Duration(options.Timeout) * time.Second,
-		CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects),
-	}, retryablehttpOptions)
-}
-
-type checkRedirectFunc func(_ *http.Request, requests []*http.Request) error
-
-func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirectFunc {
-	return func(_ *http.Request, requests []*http.Request) error {
-		if !followRedirects {
-			return http.ErrUseLastResponse
-		}
-
-		if maxRedirects == 0 {
-			if len(requests) > ten {
-				return http.ErrUseLastResponse
-			}
-
-			return nil
-		}
-
-		if len(requests) > maxRedirects {
-			return http.ErrUseLastResponse
-		}
-
-		return nil
-	}
-}
-
-func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) {
-	for _, customHeader := range e.customHeaders {
-		// This should be pre-computed somewhere and done only once
-		tokens := strings.SplitN(customHeader, ":", two)
-		// if it's an invalid header skip it
-		if len(tokens) < two {
-			continue
-		}
-
-		headerName, headerValue := tokens[0], strings.Join(tokens[1:], "")
-		if r.RawRequest != nil {
-			// rawhttp
-			r.RawRequest.Headers[headerName] = headerValue
-		} else {
-			// retryablehttp
-			headerName = strings.TrimSpace(headerName)
-			headerValue = strings.TrimSpace(headerValue)
-			r.Request.Header[headerName] = []string{headerValue}
-		}
-	}
-}
-
-type Result struct {
-	sync.Mutex
-	GotResults  bool
-	Done        bool
-	Meta        map[string]interface{}
-	Matches     map[string]interface{}
-	Extractions map[string]interface{}
-	historyData map[string]interface{}
-	Error       error
-}

v2/pkg/executer/output_dns.go

@@ -1,106 +0,0 @@
-package executer
-
-import (
-	"strings"
-
-	"github.com/miekg/dns"
-
-	jsoniter "github.com/json-iterator/go"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-)
-
-// writeOutputDNS writes dns output to streams
-// nolint:interfacer // dns.Msg is out of current scope
-func (e *DNSExecuter) writeOutputDNS(domain string, req, resp *dns.Msg, matcher *matchers.Matcher, extractorResults []string) {
-	if e.jsonOutput {
-		output := make(jsonOutput)
-		output["matched"] = domain
-
-		if !e.noMeta {
-			output["template"] = e.template.ID
-			output["type"] = "dns"
-			output["host"] = domain
-			for k, v := range e.template.Info {
-				output[k] = v
-			}
-			if matcher != nil && len(matcher.Name) > 0 {
-				output["matcher_name"] = matcher.Name
-			}
-			if len(extractorResults) > 0 {
-				output["extracted_results"] = extractorResults
-			}
-			if e.jsonRequest {
-				output["request"] = req.String()
-				output["response"] = resp.String()
-			}
-		}
-
-		data, err := jsoniter.Marshal(output)
-		if err != nil {
-			gologger.Warningf("Could not marshal json output: %s\n", err)
-		}
-		gologger.Silentf("%s", string(data))
-		if e.writer != nil {
-			if err := e.writer.Write(data); err != nil {
-				gologger.Errorf("Could not write output data: %s\n", err)
-				return
-			}
-		}
-		return
-	}
-
-	builder := &strings.Builder{}
-	colorizer := e.colorizer
-
-	if !e.noMeta {
-		builder.WriteRune('[')
-		builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String())
-
-		if matcher != nil && len(matcher.Name) > 0 {
-			builder.WriteString(":")
-			builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String())
-		}
-
-		builder.WriteString("] [")
-		builder.WriteString(colorizer.Colorizer.BrightBlue("dns").String())
-		builder.WriteString("] ")
-
-		if e.template.Info["severity"] != "" {
-			builder.WriteString("[")
-			builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"]))
-			builder.WriteString("] ")
-		}
-	}
-	builder.WriteString(domain)
-
-	// If any extractors, write the results
-	if len(extractorResults) > 0 && !e.noMeta {
-		builder.WriteString(" [")
-
-		for i, result := range extractorResults {
-			builder.WriteString(colorizer.Colorizer.BrightCyan(result).String())
-
-			if i != len(extractorResults)-1 {
-				builder.WriteRune(',')
-			}
-		}
-		builder.WriteString("]")
-	}
-	builder.WriteRune('\n')
-
-	// Write output to screen as well as any output file
-	message := builder.String()
-	gologger.Silentf("%s", message)
-
-	if e.writer != nil {
-		if e.coloredOutput {
-			message = e.decolorizer.ReplaceAllString(message, "")
-		}
-
-		if err := e.writer.WriteString(message); err != nil {
-			gologger.Errorf("Could not write output data: %s\n", err)
-			return
-		}
-	}
-}

v2/pkg/executer/output_http.go

@@ -1,147 +0,0 @@
-package executer
-
-import (
-	"net/http"
-	"net/http/httputil"
-	"strings"
-
-	jsoniter "github.com/json-iterator/go"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-)
-
-// writeOutputHTTP writes http output to streams
-func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Response, body string, matcher *matchers.Matcher, extractorResults []string, meta map[string]interface{}, reqURL string) {
-	var URL string
-	if req.RawRequest != nil {
-		URL = req.RawRequest.FullURL
-	}
-	if req.Request != nil {
-		URL = req.Request.URL.String()
-	}
-
-	if e.jsonOutput {
-		output := make(jsonOutput)
-
-		output["matched"] = URL
-		if !e.noMeta {
-			output["template"] = e.template.ID
-			output["type"] = "http"
-			output["host"] = reqURL
-			if len(meta) > 0 {
-				output["meta"] = meta
-			}
-			for k, v := range e.template.Info {
-				output[k] = v
-			}
-			if matcher != nil && len(matcher.Name) > 0 {
-				output["matcher_name"] = matcher.Name
-			}
-			if len(extractorResults) > 0 {
-				output["extracted_results"] = extractorResults
-			}
-
-			// TODO: URL should be an argument
-			if e.jsonRequest {
-				dumpedRequest, err := requests.Dump(req, URL)
-				if err != nil {
-					gologger.Warningf("could not dump request: %s\n", err)
-				} else {
-					output["request"] = string(dumpedRequest)
-				}
-
-				dumpedResponse, err := httputil.DumpResponse(resp, false)
-				if err != nil {
-					gologger.Warningf("could not dump response: %s\n", err)
-				} else {
-					output["response"] = string(dumpedResponse) + body
-				}
-			}
-		}
-
-		data, err := jsoniter.Marshal(output)
-		if err != nil {
-			gologger.Warningf("Could not marshal json output: %s\n", err)
-		}
-		gologger.Silentf("%s", string(data))
-
-		if e.writer != nil {
-			if err := e.writer.Write(data); err != nil {
-				gologger.Errorf("Could not write output data: %s\n", err)
-				return
-			}
-		}
-		return
-	}
-
-	builder := &strings.Builder{}
-	colorizer := e.colorizer
-
-	if !e.noMeta {
-		builder.WriteRune('[')
-		builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String())
-
-		if matcher != nil && len(matcher.Name) > 0 {
-			builder.WriteString(":")
-			builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String())
-		}
-
-		builder.WriteString("] [")
-		builder.WriteString(colorizer.Colorizer.BrightBlue("http").String())
-		builder.WriteString("] ")
-
-		if e.template.Info["severity"] != "" {
-			builder.WriteString("[")
-			builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"]))
-			builder.WriteString("] ")
-		}
-	}
-	builder.WriteString(URL)
-
-	// If any extractors, write the results
-	if len(extractorResults) > 0 && !e.noMeta {
-		builder.WriteString(" [")
-
-		for i, result := range extractorResults {
-			builder.WriteString(colorizer.Colorizer.BrightCyan(result).String())
-
-			if i != len(extractorResults)-1 {
-				builder.WriteRune(',')
-			}
-		}
-
-		builder.WriteString("]")
-	}
-
-	// write meta if any
-	if len(req.Meta) > 0 && !e.noMeta {
-		builder.WriteString(" [")
-
-		var metas []string
-
-		for name, value := range req.Meta {
-			metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(value.(string)).String())
-		}
-
-		builder.WriteString(strings.Join(metas, ","))
-		builder.WriteString("]")
-	}
-
-	builder.WriteRune('\n')
-
-	// Write output to screen as well as any output file
-	message := builder.String()
-	gologger.Silentf("%s", message)
-
-	if e.writer != nil {
-		if e.coloredOutput {
-			message = e.decolorizer.ReplaceAllString(message, "")
-		}
-
-		if err := e.writer.WriteString(message); err != nil {
-			gologger.Errorf("Could not write output data: %s\n", err)
-			return
-		}
-	}
-}

v2/pkg/executer/utils.go

@@ -1,59 +0,0 @@
-package executer
-
-import (
-	"net/http"
-	"net/url"
-	"strings"
-	"unsafe"
-)
-
-type jsonOutput map[string]interface{}
-
-// unsafeToString converts byte slice to string with zero allocations
-func unsafeToString(bs []byte) string {
-	return *(*string)(unsafe.Pointer(&bs))
-}
-
-// headersToString converts http headers to string
-func headersToString(headers http.Header) string {
-	builder := &strings.Builder{}
-
-	for header, values := range headers {
-		builder.WriteString(header)
-		builder.WriteString(": ")
-
-		for i, value := range values {
-			builder.WriteString(value)
-
-			if i != len(values)-1 {
-				builder.WriteRune('\n')
-				builder.WriteString(header)
-				builder.WriteString(": ")
-			}
-		}
-		builder.WriteRune('\n')
-	}
-	return builder.String()
-}
-
-// isURL tests a string to determine if it is a well-structured url or not.
-func isURL(toTest string) bool {
-	_, err := url.ParseRequestURI(toTest)
-	if err != nil {
-		return false
-	}
-	u, err := url.Parse(toTest)
-	if err != nil || u.Scheme == "" || u.Host == "" {
-		return false
-	}
-	return true
-}
-
-// extractDomain extracts the domain name of a URL
-func extractDomain(theURL string) string {
-	u, err := url.Parse(theURL)
-	if err != nil {
-		return ""
-	}
-	return u.Hostname()
-}

v2/pkg/extractors/extract.go

@@ -1,95 +0,0 @@
-package extractors
-
-import (
-	"net/http"
-
-	"github.com/miekg/dns"
-)
-
-// Extract extracts response from the parts of request using a regex
-func (e *Extractor) Extract(resp *http.Response, body, headers string) map[string]struct{} {
-	switch e.extractorType {
-	case RegexExtractor:
-		if e.part == BodyPart {
-			return e.extractRegex(body)
-		} else if e.part == HeaderPart {
-			return e.extractRegex(headers)
-		} else {
-			matches := e.extractRegex(headers)
-			if len(matches) > 0 {
-				return matches
-			}
-			return e.extractRegex(body)
-		}
-	case KValExtractor:
-		if e.part == HeaderPart {
-			return e.extractKVal(resp)
-		}
-
-		matches := e.extractKVal(resp)
-
-		if len(matches) > 0 {
-			return matches
-		}
-
-		return e.extractCookieKVal(resp)
-	}
-
-	return nil
-}
-
-// ExtractDNS extracts response from dns message using a regex
-// nolint:interfacer // dns.Msg is out of current scope
-func (e *Extractor) ExtractDNS(msg *dns.Msg) map[string]struct{} {
-	switch e.extractorType {
-	case RegexExtractor:
-		return e.extractRegex(msg.String())
-	case KValExtractor:
-	}
-
-	return nil
-}
-
-// extractRegex extracts text from a corpus and returns it
-func (e *Extractor) extractRegex(corpus string) map[string]struct{} {
-	results := make(map[string]struct{})
-
-	groupPlusOne := e.RegexGroup + 1
-	for _, regex := range e.regexCompiled {
-		matches := regex.FindAllStringSubmatch(corpus, -1)
-		for _, match := range matches {
-			if len(match) >= groupPlusOne {
-				results[match[e.RegexGroup]] = struct{}{}
-			}
-		}
-	}
-	return results
-}
-
-// extractKVal extracts text from http response
-func (e *Extractor) extractKVal(r *http.Response) map[string]struct{} {
-	results := make(map[string]struct{})
-
-	for _, k := range e.KVal {
-		for _, v := range r.Header.Values(k) {
-			results[v] = struct{}{}
-		}
-	}
-
-	return results
-}
-
-// extractCookieKVal extracts text from cookies
-func (e *Extractor) extractCookieKVal(r *http.Response) map[string]struct{} {
-	results := make(map[string]struct{})
-
-	for _, k := range e.KVal {
-		for _, cookie := range r.Cookies() {
-			if cookie.Name == k {
-				results[cookie.Value] = struct{}{}
-			}
-		}
-	}
-
-	return results
-}

v2/pkg/generators/attack.go

@@ -1,20 +0,0 @@
-package generators
-
-// Type is type of attack
-type Type int
-
-const (
-	// Sniper attack - each variable replaced with values at a time
-	Sniper Type = iota + 1
-	// PitchFork attack - Each variable replaced with positional value in multiple wordlists
-	PitchFork
-	// ClusterBomb attack - Generate all possible combinations of values
-	ClusterBomb
-)
-
-// AttackTypes is an table for conversion of attack type from string.
-var AttackTypes = map[string]Type{
-	"sniper":      Sniper,
-	"pitchfork":   PitchFork,
-	"clusterbomb": ClusterBomb,
-}

v2/pkg/generators/clusterbomb.go

@@ -1,53 +0,0 @@
-package generators
-
-// ClusterbombGenerator Attack - Generate all possible combinations from an input map with all values listed
-// as slices of the same size
-func ClusterbombGenerator(payloads map[string][]string) (out chan map[string]interface{}) {
-	out = make(chan map[string]interface{})
-
-	// generator
-	go func() {
-		defer close(out)
-
-		var order []string
-
-		var parts [][]string
-
-		for name, wordlist := range payloads {
-			order = append(order, name)
-			parts = append(parts, wordlist)
-		}
-
-		var n = 1
-		for _, ar := range parts {
-			n *= len(ar)
-		}
-
-		var at = make([]int, len(parts))
-	loop:
-		for {
-			// increment position counters
-			for i := len(parts) - 1; i >= 0; i-- {
-				if at[i] > 0 && at[i] >= len(parts[i]) {
-					if i == 0 || (i == 1 && at[i-1] == len(parts[0])-1) {
-						break loop
-					}
-					at[i] = 0
-					at[i-1]++
-				}
-			}
-			// construct permutation
-			item := make(map[string]interface{})
-			for i, ar := range parts {
-				var p = at[i]
-				if p >= 0 && p < len(ar) {
-					item[order[i]] = ar[p]
-				}
-			}
-			out <- item
-			at[len(parts)-1]++
-		}
-	}()
-
-	return out
-}