Merge pull request #413 from projectdiscovery/dev

Nuclei 2.2.0

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,14 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: "[feature]"
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.

.github/ISSUE_TEMPLATE/issue-report.md

@@ -0,0 +1,18 @@
+---
+name: Issue report
+about: Create a report to help us improve
+title: "[issue]"
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Nuclei version**
+Please share the version of the nuclei you are running with `nuclei -version` 
+
+
+**Screenshot of the error or bug**
+please add the screenshot showing bug or issue you are facing.

.github/workflows/build.yaml

@@ -4,11 +4,34 @@ on:
     branches:
       - master
   pull_request:
-  
-jobs:          
+
+jobs:
+  lint:
+    name: golangci-lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v1
+        with:
+          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
+          version: v1.31
+          args: --timeout 5m
+          working-directory: v2/
+
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+
+          # Optional: golangci-lint command line arguments.
+          # args: --issues-exit-code=0
+
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true
+
   build:
     name: Build
-    runs-on: ubuntu-latest 
+    runs-on: ubuntu-latest
     steps:
       - name: Set up Go
         uses: actions/setup-go@v2
@@ -24,4 +47,4 @@ jobs:
 
       - name: Build
         run: go build .
-        working-directory: v2/cmd/nuclei/
+        working-directory: v2/cmd/nuclei/

.github/workflows/release.yml

@@ -4,22 +4,25 @@ on:
     tags:
       - v*
 
-jobs:
-  release:
-    name: Release on GitHub
+jobs: 
+  release: 
     runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v1
-
-      - name: Validates GO releaser config
-        uses: docker://goreleaser/goreleaser:latest
-        with:
-          args: check
-
-      - name: Create release on GitHub
-        uses: docker://goreleaser/goreleaser:latest
-        with:
-          args: release
-        env:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+    steps: 
+      - 
+        name: "Check out code"
+        uses: actions/checkout@v2
+        with: 
+          fetch-depth: 0
+      - 
+        name: "Set up Go"
+        uses: actions/setup-go@v2
+        with: 
+          go-version: 1.14
+      - 
+        env: 
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        name: "Create release on GitHub"
+        uses: goreleaser/goreleaser-action@v2
+        with: 
+          args: "release --rm-dist"
+          version: latest

.gitignore

@@ -1,2 +1,3 @@
 cmd/nuclei/nuclei*
 v2/cmd/nuclei/nuclei
+.idea

.golangci.yml

@@ -0,0 +1,115 @@
+linters-settings:
+  dupl:
+    threshold: 100
+  exhaustive:
+    default-signifies-exhaustive: false
+  # funlen:
+  #   lines: 100
+  #   statements: 50
+  goconst:
+    min-len: 2
+    min-occurrences: 2
+  gocritic:
+    enabled-tags:
+      - diagnostic
+      - experimental
+      - opinionated
+      - performance
+      - style
+    disabled-checks:
+      - dupImport # https://github.com/go-critic/go-critic/issues/845
+      - ifElseChain
+  # gocyclo:
+  #   min-complexity: 15
+  goimports:
+    local-prefixes: github.com/golangci/golangci-lint
+  golint:
+    min-confidence: 0
+  gomnd:
+    settings:
+      mnd:
+        # don't include the "operation" and "assign"
+        checks: argument,case,condition,return
+  govet:
+    check-shadowing: true
+    settings:
+      printf:
+        funcs:
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
+  # lll:
+  #   line-length: 140
+  maligned:
+    suggest-new: true
+  misspell:
+    locale: US
+  nolintlint:
+    allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
+    allow-unused: false # report any unused nolint directives
+    require-explanation: false # don't require an explanation for nolint directives
+    require-specific: false # don't require nolint directives to be specific about which linter is being skipped
+
+linters:
+  # please, do not use `enable-all`: it's deprecated and will be removed soon.
+  # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
+  disable-all: true
+  enable:
+    - bodyclose
+    - deadcode
+    - dogsled
+    - errcheck
+    - exhaustive
+    - gochecknoinits
+    - goconst
+    - gocritic
+    - gofmt
+    - goimports
+    - golint
+    - gomnd
+    - goprintffuncname
+    - gosimple
+    - govet
+    - ineffassign
+    - interfacer
+    - maligned
+    - misspell
+    - nakedret
+    - noctx
+    - nolintlint
+    - rowserrcheck
+    - scopelint
+    - staticcheck
+    - structcheck
+    - stylecheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+    - varcheck
+    - whitespace
+
+  # don't enable:
+  # - depguard
+  # - asciicheck
+  # - funlen
+  # - gochecknoglobals
+  # - gocognit
+  # - gocyclo
+  # - godot
+  # - godox
+  # - goerr113
+  # - gosec
+  # - lll
+  # - nestif
+  # - prealloc
+  # - testpackage
+  # - wsl
+
+# golangci.com configuration
+# https://github.com/golangci/golangci/wiki/Configuration
+service:
+  golangci-lint-version: 1.31.x # use the fixed version to not introduce new linters unexpectedly
+  prepare:
+    - echo "here I can run custom commands, but no preparation needed for this repo"

Dockerfile

@@ -1,10 +1,19 @@
-FROM golang:1.13.11-alpine3.10 AS build-env
+FROM golang:alpine as builder
 
-RUN apk add --no-cache --upgrade git openssh-client ca-certificates
-RUN go get -u github.com/golang/dep/cmd/dep
-WORKDIR /go/src/app
+RUN mkdir -p /app
+WORKDIR /app
+COPY ./v2/go.mod .
+RUN go mod download
 
-# Install
-RUN GO111MODULE=on go get -u github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+COPY . .
+RUN cd ./v2/cmd/nuclei && go build -o nuclei .
 
-ENTRYPOINT ["nuclei"]
+FROM alpine
+
+RUN mkdir /app
+RUN adduser --home /app --shell /bin/sh --disabled-password appuser
+COPY --from=builder /app/v2/cmd/nuclei/nuclei /app
+USER appuser
+
+WORKDIR /app
+ENTRYPOINT ["/app/nuclei"]

README.md

@@ -1,4 +1,4 @@
-<h1 align="left">
+<h1 align="center">
   <img src="static/nuclei-logo.png" alt="nuclei" width="200px"></a>
   <br>
 </h1>
@@ -6,172 +6,243 @@
 [![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT)
 [![Go Report Card](https://goreportcard.com/badge/github.com/projectdiscovery/nuclei)](https://goreportcard.com/report/github.com/projectdiscovery/nuclei)
 [![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectdiscovery/nuclei/issues)
+[![GitHub Release](https://img.shields.io/github/release/projectdiscovery/nuclei)](https://github.com/projectdiscovery/nuclei/releases)
+[![Follow on Twitter](https://img.shields.io/twitter/follow/pdnuclei.svg?logo=twitter)](https://twitter.com/pdnuclei)
+[![Docker Images](https://img.shields.io/docker/pulls/projectdiscovery/nuclei.svg)](https://hub.docker.com/r/projectdiscovery/nuclei)
+[![Chat on Discord](https://img.shields.io/discord/695645237418131507.svg?logo=discord)](https://discord.gg/KECAGdH)
 
-Nuclei is a fast tool for configurable targeted scanning based on templates offering massive extensibility and ease of use. 
+<p align="center">
+<a href="https://nuclei.projectdiscovery.io/templating-guide/" target="_blank"><img src="static/read-the-docs-button.png" height="42px"/></center></a>  <a href="https://github.com/projectdiscovery/nuclei-templates" target="_blank"><img src="static/download-templates-button.png" height="42px"/></a>
+</p>
+
+
+Nuclei is a fast tool for configurable targeted scanning based on templates offering massive extensibility and ease of use.
 
 Nuclei is used to send requests across targets based on a template leading to zero false positives and providing effective scanning for known paths. Main use cases for nuclei are during initial reconnaissance phase to quickly check for low hanging fruits or CVEs across targets that are known and easily detectable. It uses [retryablehttp-go library](https://github.com/projectdiscovery/retryablehttp-go) designed to handle various errors and retries in case of blocking by WAFs, this is also one of our core modules from custom-queries.
 
-We have also [open-sourced a dedicated repository](https://github.com/projectdiscovery/nuclei-templates) to maintain various type of templates, we hope that you will contribute there too. Templates are provided in hopes that these will be useful and will allow everyone to build their own templates for the scanner. Checkout the guide at [**GUIDE.md**](https://github.com/projectdiscovery/nuclei-templates/blob/master/GUIDE.md) for a primer on nuclei templates.
+We have also [open-sourced a template repository](https://github.com/projectdiscovery/nuclei-templates) to maintain various type of templates, we hope that you will contribute there too. Templates are provided in hopes that these will be useful and will allow everyone to build their own templates for the scanner. Checkout the templating guide at [**nuclei.projectdiscovery.io**](https://nuclei.projectdiscovery.io/templating-guide/) for a primer on nuclei templates.
 
-# Resources
-- [Resources](#resources)
-- [Features](#features)
-- [Usage](#usage)
-- [Installation Instructions](#installation-instructions)
-    - [From Binary](#from-binary)
-    - [From Source](#from-source)
-    - [Running in a Docker Container](#running-in-a-docker-container)
-- [Nuclei templates](#nuclei-templates)
-- [Running nuclei](#running-nuclei)
-    - [1. Running nuclei with a single template.](#1-running-nuclei-with-a-single-template)
-    - [2. Running nuclei with multiple templates.](#2-running-nuclei-with-multiple-templates)
-    - [3. Automating nuclei with subfinder and any other similar tool.](#3-automating-nuclei-with-subfinder-and-any-other-similar-tool)
-- [Thanks](#thanks)
+## Resources
 
- # Features
+-   [Features](#features)
+-   [Usage](#usage)
+-   [Installation Instructions](#installation-instructions)
+    -   [From Binary](#from-binary)
+    -   [From Source](#from-source)
+    -   [From Github](#from-github)
+-   [Nuclei templates](#nuclei-templates)
+-   [Running nuclei](#running-nuclei)
+    -   [Running with a single template.](#running-with-single-template)
+    -   [Running with multiple templates.](#running-with-multiple-templates)
+    -   [Running with subfinder](#running-with-subfinder)
+    -   [Running in Docker](#running-in-docker-container)
+-   [Template exclusion](#template-exclusion)
+-   [Thanks](#thanks)
+
+
+## Features
 
 <h1 align="left">
   <img src="static/nuclei-run.png" alt="nuclei" width="700px"></a>
   <br>
 </h1>
 
- - Simple and modular code base making it easy to contribute.
- - Fast And fully configurable using a template based engine.
- - Handles edge cases doing retries, backoffs etc for handling WAFs.
- - Smart matching functionality for zero false positive scanning.
+-   Simple and modular code base making it easy to contribute.
+-   Fast And fully configurable using a template based engine.
+-   Handles edge cases doing retries, backoffs etc for handling WAFs.
+-   Smart matching functionality for zero false positive scanning.
 
-# Usage
+## Usage
 
-```bash
+```sh
 nuclei -h
 ```
 
 This will display help for the tool. Here are all the switches it supports.
 
-| Flag              | Description                                           | Example                                            |
-|-------------------|-------------------------------------------------------|----------------------------------------------------|
-| -c                | Number of concurrent requests (default 10)            | nuclei -c 100                                      |
-| -l                | List of urls to run templates                         | nuclei -l urls.txt                                 |
-| -target           | Target to scan using templates                        | nuclei -target hxxps://example.com                 |
-| -t                | Templates input file/files to check across hosts      | nuclei -t git-core.yaml                            |
-| -t                | Templates input file/files to check across hosts      | nuclei -t nuclei-templates/cves/                   |
-| -nC               | Don't Use colors in output                            | nuclei -nC                                         |
-| -json             | Prints and write output in json format                | nuclei -json                                       |
-| -o                | File to save output result (optional)                 | nuclei -o output.txt                               |
-| -silent           | Show only found results in output                     | nuclei -silent                                     |
-| -retries          | Number of times to retry a failed request (default 1) | nuclei -retries 1                                  |
-| -timeout          | Seconds to wait before timeout (default 5)            | nuclei -timeout 5                                  |
-| -debug            | Allow debugging of request/responses.                 | nuclei -debug                                      |
-| -update-templates | Download and updates nuclei templates                 | nuclei -update-templates                           |
-| -update-directory | Directory for storing nuclei-templates(optional)      | nuclei -update-directory templates                 |
-| -v                | Shows verbose output of all sent requests             | nuclei -v                                          |
-| -version          | Show version of nuclei                                | nuclei -version                                    |
-| -proxy-url        | Proxy URL                                             | nuclei -proxy-url hxxp://127.0.0.1:8080            |
-| -proxy-socks-url  | Socks proxy  URL                                      | nuclei -proxy-socks-url socks5://127.0.0.1:8080    |
-| -H                | Custom Header                                         | nuclei -H "x-bug-bounty: hacker"                   |
-
-
-# Installation Instructions
+|        Flag       |                      Description                      |                     Example                     |
+| :---------------: | :---------------------------------------------------: | :---------------------------------------------: |
+|         -c        |       Number of concurrent requests (default 10)      |                  nuclei -c 100                  |
+|         -l        |             List of urls to run templates             |                nuclei -l urls.txt               |
+|      -target      |             Target to scan using templates            |        nuclei -target hxxps://example.com       |
+|         -t        |    Templates input file/files to check across hosts   |             nuclei -t git-core.yaml             |
+|         -t        |    Templates input file/files to check across hosts   |         nuclei -t nuclei-templates/cves/        |
+|        -nC        |               Don't Use colors in output              |                    nuclei -nC                   |
+|       -json       |         Prints and write output in json format        |                   nuclei -json                  |
+|   -json-requests  |  Write requests/responses for matches in JSON output  |           nuclei -json -json-requests           |
+|         -o        |         File to save output result (optional)         |               nuclei -o output.txt              |
+|       -pbar       |           Enable the progress bar (optional)          |                   nuclei -pbar                  |
+|      -silent      |           Show only found results in output           |                  nuclei -silent                 |
+|                   |             (except when using with pbar)             |                                                 |
+|      -retries     | Number of times to retry a failed request (default 1) |                nuclei -retries 1                |
+|      -timeout     |       Seconds to wait before timeout (default 5)      |                nuclei -timeout 5                |
+|      -rl          |       Rate-Limit of requests per specified target     |                nuclei -rl 100                   |
+|      -severity    |Run templates based on severity                        |                nuclei -severity critical, low                |
+|      -exclude     |Template input dir/file/files to exclude               |                nuclei -exclude panels -exclude tokens           |
+|       -debug      |         Allow debugging of request/responses.         |                  nuclei -debug                  |
+| -update-templates |         Download and updates nuclei templates         |             nuclei -update-templates            |
+| -update-directory |    Directory for storing nuclei-templates(optional)   |        nuclei -update-directory templates       |
+|        -tl        |                List available templates               |                    nuclei -tl                   |
+|         -v        |       Shows verbose output of all sent requests       |                    nuclei -v                    |
+|      -version     |                 Show version of nuclei                |                 nuclei -version                 |
+|     -proxy-url    |                       Proxy URL                       |     nuclei -proxy-url hxxp://127.0.0.1:8080     |
+|  -proxy-socks-url |                    Socks proxy  URL                   | nuclei -proxy-socks-url socks5://127.0.0.1:8080 |
+|         -H        |                     Custom Header                     |         nuclei -H "x-bug-bounty: hacker"        |
 
+## Installation Instructions
 
 ### From Binary
 
 The installation is easy. You can download the pre-built binaries for your platform from the [Releases](https://github.com/projectdiscovery/nuclei/releases/) page. Extract them using tar, move it to your `$PATH`and you're ready to go.
 
-```bash
-> tar -xzvf nuclei-linux-amd64.tar.gz
-> mv nuclei /usr/bin/
-> nuclei -h
+```sh
+Download latest binary from https://github.com/projectdiscovery/nuclei/releases
+
+▶ tar -xzvf nuclei-linux-amd64.tar.gz
+▶ mv nuclei /usr/local/bin/
+▶ nuclei -version
 ```
 
 ### From Source
 
-nuclei requires go1.13+ to install successfully. Run the following command to get the repo - 
+nuclei requires **go1.14+** to install successfully. Run the following command to get the repo -
 
-```bash
-> GO111MODULE=on go get -u -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+```sh
+▶ GO111MODULE=on go get -u -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
 ```
 
-In order to update the tool, you can use -u flag with `go get` command.
+### From Github
 
-### Running in a Docker Container
-
-You can use the [nuclei dockerhub image](https://hub.docker.com/r/projectdiscovery/nuclei). Simply run -
-
-```bash
-> docker pull projectdiscovery/nuclei
+```sh
+▶ git clone https://github.com/projectdiscovery/nuclei.git; cd nuclei/v2/cmd/nuclei/; go build; mv nuclei /usr/local/bin/; nuclei -version
 ```
 
-- After downloading or building the container, run the following:
-```bash
-> docker run -it projectdiscovery/nuclei
-```
+## Nuclei templates
 
-For example, this will run the tool against all the hosts in `urls.txt` and output the results to your host file system:
-```bash
-> cat urls.txt | docker run -v /path-to-nuclei-templates:/go/src/app/ -i projectdiscovery/nuclei -t ./files/git-config.yaml > results.txt
-```
-Remember to change `/path-to-nuclei-templates` to the real path on your host file system.
+You can download or update the nuclei templates using `update-templates` flag.
 
-# Nuclei templates
-
-You can download or update the nuclei templates using `update-templates` flag. 
-
-```bash
-> nuclei -update-templates
+```sh
+▶ nuclei -update-templates
 ```
 
 or download it from [nuclei templates](https://github.com/projectdiscovery/nuclei-templates) Github project.
 
-```bash
-> git clone https://github.com/projectdiscovery/nuclei-templates.git
+```sh
+▶ git clone https://github.com/projectdiscovery/nuclei-templates.git
 ```
 
-# Running nuclei
+**Please refer to nuclei [templating guide](https://nuclei.projectdiscovery.io/templating-guide/) to writing your own custom templates.**
 
-### 1. Running nuclei with a single template. 
+## Running nuclei
 
-This will run the tool against all the hosts in `urls.txt` and returns the matched results. 
+### Running with single template.
 
-```bash
-> nuclei -l urls.txt -t files/git-core.yaml -o results.txt
+This will run `git-core.yaml` template against all the hosts in `urls.txt` and returns the matched results.
+
+```sh
+▶ nuclei -l urls.txt -t files/git-core.yaml -o results.txt
 ```
 
-You can also pass the list of hosts at standard input (STDIN). This allows for easy integration in automation pipelines.
+You can also pass the list of urls at standard input (STDIN). This allows for easy integration in automation pipelines.
 
-This will run the tool against all the hosts in `urls.txt` and returns the matched results. 
-
-```bash
-> cat urls.txt | nuclei -t files/git-core.yaml -o results.txt
+```sh
+▶ cat urls.txt | nuclei -t files/git-core.yaml -o results.txt
 ```
 
-### 2. Running nuclei with multiple templates. 
+💡 Nuclei accepts list of URLs as input, for example here is how `urls.txt` looks like:- 
 
-This will run the tool against all the hosts in `urls.txt` with all the templates in the `path-to-templates` directory and returns the matched results. 
+```
+https://test.some-site.com
+http://vuls-testing.com
+https://test.com
+```
+### Running with multiple templates.
 
-```bash
-> nuclei -l urls.txt -t cves/ -o results.txt 
+This will run the tool against all the urls in `urls.txt` with all the templates in the `cves` and `files` directory and returns the matched results.
+
+```sh
+▶ nuclei -l urls.txt -t cves/ -t files/ -o results.txt
 ```
 
-### 3. Automating nuclei with subfinder and any other similar tool.
+### Running with subfinder.
 
-
-```bash
-> subfinder -d hackerone.com -silent | httpx -silent | nuclei -t cves/ -o results.txt
+```sh
+▶ subfinder -d hackerone.com -silent | httpx -silent | nuclei -t cves/ -o results.txt
 ```
 
--------
+### Running in Docker container
 
-**Please refer to [this guide](https://github.com/projectdiscovery/nuclei-templates/blob/master/GUIDE.md) to build your own custom templates.**
+You can use the [nuclei dockerhub image](https://hub.docker.com/r/projectdiscovery/nuclei). Simply run -
+
+```sh
+▶ docker pull projectdiscovery/nuclei
+```
+
+After downloading or building the container, run the following:
+
+```sh
+▶ docker run -it projectdiscovery/nuclei
+```
+
+For example, this will run the tool against all the hosts in `urls.txt` and output the results to your host file system:
+
+```sh
+▶ cat urls.txt | docker run -v /path/to/nuclei-templates:/app/nuclei-templates -v /path/to/nuclei/config:/app/.nuclei-config.json -i projectdiscovery/nuclei -t ./files/git-config.yaml > results.txt
+```
+
+Remember to change `/path-to-nuclei-templates` to the real path on your host file system.
+
+### Template Exclusion
+
+[Nuclei-templates](https://github.com/projectdiscovery/nuclei-templates) includes multiple checks including many that are useful for attack surface mapping and not necessarily a security issue, in cases where you only looking to scan few specific templates or directory, here are few options / flags to filter or exclude them from running. 
+
+#### Running templates based on severity
+
+You can run the templates based on the specific severity of the template, single and multiple severity can be used for scan. 
+
+```sh
+nuclei -l urls.txt -t cves/ -severity critical, medium
+```
+
+The above example will run all the templates under `cves` directory with `critical` and `medium` severity. 
+
+```sh
+nuclei -l urls.txt -t panels/ -t technologies -severity info
+```
+
+The above example will run all the templates under `panels` and `technologies` directory with **severity** marked as `info`
+
+#### Running templates with exclusion
+
+We do not suggest running all the nuclei-templates directory at once, in case of doing so, one can make use of `exclude` flag to exclude specific directory or templates to ignore from scanning. 
+
+```sh
+nuclei -l urls.txt -t nuclei-templates -exclude panels/ -exclude technologies -exclude files/wp-xmlrpc.yaml
+```
+
+Note:- both directory and specific templates case be excluded from scan as shared in the above example.
+
+#### Using `.nuclei-ignore` file for template exclusion
+
+Since release of nuclei [v2.1.1](https://github.com/projectdiscovery/nuclei/releases/tag/v2.1.1), we have added support of `.nuclei-ignore` file that works along with `update-templates` flag of nuclei, in **.nuclei-ignore** file, you can define all the template directory or template path that you wanted to exclude from all the nuclei scans, to start using this feature, make sure you installed nuclei templates using `nuclei -update-templates` flag, now you can add/update/remove templates in the file that you wanted to exclude from running. 
+
+```
+nano ~/nuclei-templates/.nuclei-ignore
+```
+
+Default **nuclei-ignore** list can be accessed from [here](https://github.com/projectdiscovery/nuclei-templates/blob/master/.nuclei-ignore), in case you don't want to exclude anything, simply remove the `.nuclei-ignore` file.
+
+* * *
+
+# 📋 Notes
+- Progress bar is experimental feature, might not work in few cases. 
+- Progress bar doesn't work with workflows, numbers are not accurate due to conditional execution.
 
 
-# Thanks
+## Thanks
 
-nuclei is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)** file for more details. Do also check out these similar awesome projects that may fit in your workflow:
+nuclei is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/nuclei/blob/master/THANKS.md)** file for more details.
 
-[https://portswigger.net/burp](https://portswigger.net/burp)</br>
-[https://github.com/ffuf/ffuf](https://github.com/ffuf/ffuf)</br>
-[https://github.com/jaeles-project/jaeles](https://github.com/jaeles-project/jaeles)</br>
-[https://github.com/ameenmaali/qsfuzz](https://github.com/ameenmaali/qsfuzz)</br>
-[https://github.com/proabiral/inception](https://github.com/proabiral/inception)</br>
-[https://github.com/hannob/snallygaster](https://github.com/hannob/snallygaster)</br>
+Do also check out these similar awesome projects that may fit in your workflow:
+
+[Burp Suite](https://portswigger.net/burp), [FFuF](https://github.com/ffuf/ffuf), [Jaeles](https://github.com/jaeles-project/jaeles), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [ChopChop](https://github.com/michelin/ChopChop)

THANKS.md

@@ -1 +1,17 @@
 
+### Thanks
+
+Many people have contributed to **nuclei** making it a wonderful tool either by making a pull request fixing some stuff. Here, we recognize these persons and thank them. 
+
+- All the contributors at [CONTRIBUTORS](https://github.com/projectdiscovery/nuclei/graphs/contributors) who made Nuclei what it is.
+
+We'd like to thank some additional amazing people, who contributed a lot in nuclei's journey - 
+
+- [@manuelbua](https://www.github.com/manuelbua) - Multiple feature additions including progress bar. 
+- [@dwisiswant0](https://www.github.com/dwisiswant0) - For fixing multiple bugs with HTTP Raw requests. 
+- [@toufik-airane](https://www.github.com/toufik-airane) - For adding binary matchers.
+- [@lc](https://www.github.com/lc) - For adding directory support for templates.
+- [@wdahlenburg](https://www.github.com/wdahlenburg) - For adding wildcard globing support to run templates.
+- [@Marmelatze](https://www.github.com/Marmelatze) - For adding support to save matched request/response in JSON format. 
+- [@ankh2054](https://www.github.com/ankh2054) - For adding description field for template information.
+- [@rotemreiss](https://www.github.com/rotemreiss) - For adding docker support.

go.mod

@@ -1,22 +0,0 @@
-module github.com/projectdiscovery/nuclei
-
-go 1.14
-
-require (
-	github.com/Knetic/govaluate v3.0.0+incompatible
-	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
-	github.com/blang/semver v3.5.1+incompatible
-	github.com/d5/tengo v1.24.8
-	github.com/d5/tengo/v2 v2.6.0
-	github.com/google/go-github v17.0.0+incompatible
-	github.com/google/go-github/v32 v32.0.0
-	github.com/json-iterator/go v1.1.10
-	github.com/karrick/godirwalk v1.15.6
-	github.com/miekg/dns v1.1.29
-	github.com/pkg/errors v0.9.1
-	github.com/projectdiscovery/gologger v1.0.0
-	github.com/projectdiscovery/retryabledns v1.0.4
-	github.com/projectdiscovery/retryablehttp-go v1.0.1
-	golang.org/x/net v0.0.0-20200528225125-3c3fba18258b
-	gopkg.in/yaml.v2 v2.3.0
-)

go.sum

@@ -1,71 +0,0 @@
-github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4=
-github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
-github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
-github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg=
-github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
-github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
-github.com/d5/tengo v1.24.8/go.mod h1:VhLq8Q2QFhCIJO3NhvM934qOThykMqJi9y9Siqd1ocQ=
-github.com/d5/tengo/v2 v2.6.0 h1:D0cJtpiBzaLJ/Smv6nnUc/LIfO46oKwDx85NZtIRNRI=
-github.com/d5/tengo/v2 v2.6.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
-github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
-github.com/google/go-github/v32 v32.0.0 h1:q74KVb22spUq0U5HqZ9VCYqQz8YRuOtL/39ZnfwO+NM=
-github.com/google/go-github/v32 v32.0.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
-github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
-github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
-github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA=
-github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
-github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
-github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s=
-github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
-github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60=
-github.com/projectdiscovery/retryabledns v1.0.4/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
-github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
-github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200528225125-3c3fba18258b h1:IYiJPiJfzktmDAO1HQiwjMjwjlYKHAL7KzeD544RJPs=
-golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

v2/cmd/nuclei/main.go

@@ -9,11 +9,11 @@ func main() {
 	// Parse the command line flags and read config files
 	options := runner.ParseOptions()
 
-	runner, err := runner.New(options)
+	nucleiRunner, err := runner.New(options)
 	if err != nil {
 		gologger.Fatalf("Could not create runner: %s\n", err)
 	}
 
-	runner.RunEnumeration()
-	runner.Close()
+	nucleiRunner.RunEnumeration()
+	nucleiRunner.Close()
 }

v2/go.mod

@@ -4,19 +4,27 @@ go 1.14
 
 require (
 	github.com/Knetic/govaluate v3.0.0+incompatible
-	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/d5/tengo v1.24.8
-	github.com/d5/tengo/v2 v2.6.0
-	github.com/google/go-github v17.0.0+incompatible
-	github.com/google/go-github/v32 v32.0.0
+	github.com/d5/tengo/v2 v2.6.2
+	github.com/google/go-github/v32 v32.1.0
 	github.com/json-iterator/go v1.1.10
-	github.com/karrick/godirwalk v1.15.6
-	github.com/miekg/dns v1.1.29
+	github.com/karrick/godirwalk v1.16.1
+	github.com/logrusorgru/aurora v2.0.3+incompatible
+	github.com/miekg/dns v1.1.35
 	github.com/pkg/errors v0.9.1
-	github.com/projectdiscovery/gologger v1.0.0
-	github.com/projectdiscovery/retryabledns v1.0.4
+	github.com/projectdiscovery/clistats v0.0.5
+	github.com/projectdiscovery/collaborator v0.0.1
+	github.com/projectdiscovery/fastdialer v0.0.1
+	github.com/projectdiscovery/gologger v1.0.1
+	github.com/projectdiscovery/hmap v0.0.1
+	github.com/projectdiscovery/rawhttp v0.0.4
+	github.com/projectdiscovery/retryabledns v1.0.5
 	github.com/projectdiscovery/retryablehttp-go v1.0.1
-	golang.org/x/net v0.0.0-20200528225125-3c3fba18258b
+	github.com/remeh/sizedwaitgroup v1.0.0
+	github.com/segmentio/ksuid v1.0.3
+	github.com/spaolacci/murmur3 v1.1.0
+	github.com/stretchr/testify v1.5.1
+	go.uber.org/ratelimit v0.1.0
+	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
 	gopkg.in/yaml.v2 v2.3.0
 )

v2/go.sum

@@ -1,69 +1,116 @@
+github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4=
 github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
+github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
-github.com/d5/tengo v1.24.8/go.mod h1:VhLq8Q2QFhCIJO3NhvM934qOThykMqJi9y9Siqd1ocQ=
-github.com/d5/tengo/v2 v2.6.0 h1:D0cJtpiBzaLJ/Smv6nnUc/LIfO46oKwDx85NZtIRNRI=
-github.com/d5/tengo/v2 v2.6.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
+github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc=
+github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
-github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
-github.com/google/go-github/v32 v32.0.0 h1:q74KVb22spUq0U5HqZ9VCYqQz8YRuOtL/39ZnfwO+NM=
-github.com/google/go-github/v32 v32.0.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
+github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
+github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA=
-github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
+github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
+github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
-github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
+github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
+github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs=
+github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s=
-github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
-github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60=
-github.com/projectdiscovery/retryabledns v1.0.4/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
+github.com/projectdiscovery/clistats v0.0.5 h1:vcvOR9PrFRawO/7FWD6pER9nYVSoSTD2F+/fkRs73a0=
+github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
+github.com/projectdiscovery/collaborator v0.0.1 h1:dbQ5BCL/a3c+BB9cGtrGgiLs23+EfSzoaTzX/pxqiTI=
+github.com/projectdiscovery/collaborator v0.0.1/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
+github.com/projectdiscovery/fastdialer v0.0.1 h1:MgBkJ/zkciFu/PcbAz0DYGiZn2aqv6b39NvfXxfN8qg=
+github.com/projectdiscovery/fastdialer v0.0.1/go.mod h1:d24GUzSb93wOY7lu4gJmXAzfomqAGEcRrInEVrM6zbc=
+github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ=
+github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
+github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
+github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
+github.com/projectdiscovery/rawhttp v0.0.4 h1:O5IreNGk83d4xTD9e6SpkKbX0sHTs8K1Q33Bz4eYl2E=
+github.com/projectdiscovery/rawhttp v0.0.4/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
+github.com/projectdiscovery/retryabledns v1.0.5 h1:bQivGy5CuqKlwcxRkgA5ENincqIed/BR2sA6t2gdwuI=
+github.com/projectdiscovery/retryabledns v1.0.5/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
 github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
+github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
+github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
+github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY=
+github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
+github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
+go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9 h1:umElSU9WZirRdgu2yFHY0ayQkEnKiOC1TtM3fWXFnoU=
+golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200528225125-3c3fba18258b h1:IYiJPiJfzktmDAO1HQiwjMjwjlYKHAL7KzeD544RJPs=
-golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba h1:xmhUJGQGbxlod18iJGqVEp9cHIPLl7QiX2aA3to708s=
+golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

v2/internal/bufwriter/bufwriter.go

@@ -0,0 +1,74 @@
+package bufwriter
+
+import (
+	"bufio"
+	"os"
+	"sync"
+)
+
+// Writer is a mutex protected buffered writer
+type Writer struct {
+	file   *os.File
+	writer *bufio.Writer
+	mutex  *sync.Mutex
+}
+
+// New creates a new mutex protected buffered writer for a file
+func New(file string) (*Writer, error) {
+	output, err := os.Create(file)
+	if err != nil {
+		return nil, err
+	}
+	return &Writer{file: output, writer: bufio.NewWriter(output), mutex: &sync.Mutex{}}, nil
+}
+
+// Write writes a byte slice to the underlying file
+//
+// It also writes a newline if the last byte isn't a newline character.
+func (w *Writer) Write(data []byte) error {
+	if len(data) == 0 {
+		return nil
+	}
+	w.mutex.Lock()
+	defer w.mutex.Unlock()
+
+	_, err := w.writer.Write(data)
+	if err != nil {
+		return err
+	}
+	if data[len(data)-1] != '\n' {
+		_, err = w.writer.WriteRune('\n')
+	}
+	return err
+}
+
+// WriteString writes a string to the underlying file
+//
+// It also writes a newline if the last byte isn't a newline character.
+func (w *Writer) WriteString(data string) error {
+	if data == "" {
+		return nil
+	}
+	w.mutex.Lock()
+	defer w.mutex.Unlock()
+
+	_, err := w.writer.WriteString(data)
+	if err != nil {
+		return err
+	}
+	if data[len(data)-1] != '\n' {
+		_, err = w.writer.WriteRune('\n')
+	}
+	return err
+}
+
+// Close closes the underlying writer flushing everything to disk
+func (w *Writer) Close() error {
+	w.mutex.Lock()
+	defer w.mutex.Unlock()
+
+	w.writer.Flush()
+	//nolint:errcheck // we don't care whether sync failed or succeeded.
+	w.file.Sync()
+	return w.file.Close()
+}

v2/internal/progress/doc.go

@@ -0,0 +1,3 @@
+// Package progress implements progress display mechanism with very
+// simple command line statistics printing on runtime.
+package progress

v2/internal/progress/progress.go

@@ -0,0 +1,146 @@
+package progress
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/projectdiscovery/clistats"
+	"github.com/projectdiscovery/gologger"
+)
+
+// Progress is a progress instance for showing program stats
+type Progress struct {
+	active       bool
+	stats        clistats.StatisticsClient
+	tickDuration time.Duration
+}
+
+// NewProgress creates and returns a new progress tracking object.
+func NewProgress(active bool) *Progress {
+	var tickDuration time.Duration
+	if active {
+		tickDuration = 5 * time.Second
+	} else {
+		tickDuration = -1
+	}
+
+	var progress Progress
+	if active {
+		stats, err := clistats.New()
+		if err != nil {
+			gologger.Warningf("Couldn't create progress engine: %s\n", err)
+		}
+		progress.active = active
+		progress.stats = stats
+		progress.tickDuration = tickDuration
+	}
+
+	return &progress
+}
+
+// Init initializes the progress display mechanism by setting counters, etc.
+func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) {
+	if p.active {
+		p.stats.AddStatic("templates", rulesCount)
+		p.stats.AddStatic("hosts", hostCount)
+		p.stats.AddStatic("startedAt", time.Now())
+		p.stats.AddCounter("requests", uint64(0))
+		p.stats.AddCounter("errors", uint64(0))
+		p.stats.AddCounter("total", uint64(requestCount))
+		if err := p.stats.Start(makePrintCallback(), p.tickDuration); err != nil {
+			gologger.Warningf("Couldn't start statistics: %s\n", err)
+		}
+	}
+}
+
+// AddToTotal adds a value to the total request count
+func (p *Progress) AddToTotal(delta int64) {
+	if p.active {
+		p.stats.IncrementCounter("total", int(delta))
+	}
+}
+
+// Update progress tracking information and increments the request counter by one unit.
+func (p *Progress) Update() {
+	if p.active {
+		p.stats.IncrementCounter("requests", 1)
+	}
+}
+
+// Drop drops the specified number of requests from the progress bar total.
+// This may be the case when uncompleted requests are encountered and shouldn't be part of the total count.
+func (p *Progress) Drop(count int64) {
+	if p.active {
+		// mimic dropping by incrementing the completed requests
+		p.stats.IncrementCounter("errors", int(count))
+	}
+}
+
+const bufferSize = 128
+
+func makePrintCallback() func(stats clistats.StatisticsClient) {
+	builder := &strings.Builder{}
+	builder.Grow(bufferSize)
+
+	return func(stats clistats.StatisticsClient) {
+		builder.WriteRune('[')
+		startedAt, _ := stats.GetStatic("startedAt")
+		duration := time.Since(startedAt.(time.Time))
+		builder.WriteString(fmtDuration(duration))
+		builder.WriteRune(']')
+
+		templates, _ := stats.GetStatic("templates")
+		builder.WriteString(" | Templates: ")
+		builder.WriteString(clistats.String(templates))
+		hosts, _ := stats.GetStatic("hosts")
+		builder.WriteString(" | Hosts: ")
+		builder.WriteString(clistats.String(hosts))
+
+		requests, _ := stats.GetCounter("requests")
+		total, _ := stats.GetCounter("total")
+
+		builder.WriteString(" | RPS: ")
+		builder.WriteString(clistats.String(uint64(float64(requests) / duration.Seconds())))
+
+		errors, _ := stats.GetCounter("errors")
+		builder.WriteString(" | Errors: ")
+		builder.WriteString(clistats.String(errors))
+
+		builder.WriteString(" | Requests: ")
+		builder.WriteString(clistats.String(requests))
+		builder.WriteRune('/')
+		builder.WriteString(clistats.String(total))
+		builder.WriteRune(' ')
+		builder.WriteRune('(')
+		//nolint:gomnd // this is not a magic number
+		builder.WriteString(clistats.String(uint64(float64(requests) / float64(total) * 100.0)))
+		builder.WriteRune('%')
+		builder.WriteRune(')')
+		builder.WriteRune('\n')
+
+		fmt.Fprintf(os.Stderr, "%s", builder.String())
+		builder.Reset()
+	}
+}
+
+// fmtDuration formats the duration for the time elapsed
+func fmtDuration(d time.Duration) string {
+	d = d.Round(time.Second)
+	h := d / time.Hour
+	d -= h * time.Hour
+	m := d / time.Minute
+	d -= m * time.Minute
+	s := d / time.Second
+	return fmt.Sprintf("%d:%02d:%02d", h, m, s)
+}
+
+// Stop stops the progress bar execution
+func (p *Progress) Stop() {
+	if p.active {
+		if err := p.stats.Stop(); err != nil {
+			gologger.Warningf("Couldn't stop statistics: %s\n", err)
+		}
+	}
+}

v2/internal/runner/banner.go

@@ -3,15 +3,15 @@ package runner
 import "github.com/projectdiscovery/gologger"
 
 const banner = `
-                       __     _ 
+                       __     _
      ____  __  _______/ /__  (_)
-    / __ \/ / / / ___/ / _ \/ / 
-   / / / / /_/ / /__/ /  __/ /  
-  /_/ /_/\__,_/\___/_/\___/_/   v2									  
+    / __ \/ / / / ___/ / _ \/ /
+   / / / / /_/ / /__/ /  __/ /
+  /_/ /_/\__,_/\___/_/\___/_/   v2.2.0
 `
 
 // Version is the current version of nuclei
-const Version = `2.0.2`
+const Version = `2.2.0`
 
 // showBanner is used to show the banner to the user
 func showBanner() {

v2/internal/runner/config.go

@@ -0,0 +1,136 @@
+package runner
+
+import (
+	"bufio"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+)
+
+// nucleiConfig contains some configuration options for nuclei
+type nucleiConfig struct {
+	TemplatesDirectory string    `json:"templates-directory,omitempty"`
+	CurrentVersion     string    `json:"current-version,omitempty"`
+	LastChecked        time.Time `json:"last-checked,omitempty"`
+
+	// IgnorePaths ignores all the paths listed unless specified manually
+	IgnorePaths []string `json:"ignore-paths,omitempty"`
+}
+
+// nucleiConfigFilename is the filename of nuclei configuration file.
+const nucleiConfigFilename = ".nuclei-config.json"
+
+var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
+
+// readConfiguration reads the nuclei configuration file from disk.
+func readConfiguration() (*nucleiConfig, error) {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return nil, err
+	}
+
+	templatesConfigFile := path.Join(home, nucleiConfigFilename)
+	file, err := os.Open(templatesConfigFile)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	config := &nucleiConfig{}
+	err = jsoniter.NewDecoder(file).Decode(config)
+	if err != nil {
+		return nil, err
+	}
+	return config, nil
+}
+
+// readConfiguration reads the nuclei configuration file from disk.
+func (r *Runner) writeConfiguration(config *nucleiConfig) error {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return err
+	}
+
+	config.LastChecked = time.Now()
+	templatesConfigFile := path.Join(home, nucleiConfigFilename)
+	file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	err = jsoniter.NewEncoder(file).Encode(config)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+const nucleiIgnoreFile = ".nuclei-ignore"
+
+// readNucleiIgnoreFile reads the nuclei ignore file marking it in map
+func (r *Runner) readNucleiIgnoreFile() {
+	file, err := os.Open(r.getIgnoreFilePath())
+	if err != nil {
+		return
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text == "" {
+			continue
+		}
+		if strings.HasPrefix(text, "#") {
+			continue
+		}
+		r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, text)
+	}
+}
+
+// checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules.
+func (r *Runner) checkIfInNucleiIgnore(item string) bool {
+	if r.templatesConfig == nil {
+		return false
+	}
+
+	for _, paths := range r.templatesConfig.IgnorePaths {
+		// If we have a path to ignore, check if it's in the item.
+		if paths[len(paths)-1] == '/' {
+			if strings.Contains(item, paths) {
+				return true
+			}
+
+			continue
+		}
+		// Check for file based extension in ignores
+		if strings.HasSuffix(item, paths) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (r *Runner) getIgnoreFilePath() string {
+	defIgnoreFilePath := path.Join(r.templatesConfig.TemplatesDirectory, nucleiIgnoreFile)
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return defIgnoreFilePath
+	}
+
+	cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile)
+
+	cwdIfpInfo, err := os.Stat(cwdIgnoreFilePath)
+	if os.IsNotExist(err) || cwdIfpInfo.IsDir() {
+		return defIgnoreFilePath
+	}
+
+	return cwdIgnoreFilePath
+}

v2/internal/runner/options.go

@@ -1,7 +1,9 @@
 package runner
 
 import (
+	"errors"
 	"flag"
+	"net/url"
 	"os"
 
 	"github.com/projectdiscovery/gologger"
@@ -10,27 +12,53 @@ import (
 
 // Options contains the configuration options for tuning
 // the template requesting process.
+// nolint // false positive, options are allocated once and are necessary as is
 type Options struct {
-	Debug              bool                   // Debug mode allows debugging request/responses for the engine
-	Templates          string                 // Signature specifies the template/templates to use
-	Target             string                 // Target is a single URL/Domain to scan usng a template
-	Targets            string                 // Targets specifies the targets to scan using templates.
-	Threads            int                    // Thread controls the number of concurrent requests to make.
-	Timeout            int                    // Timeout is the seconds to wait for a response from the server.
-	Retries            int                    // Retries is the number of times to retry the request
-	Output             string                 // Output is the file to write found subdomains to.
-	ProxyURL           string                 // ProxyURL is the URL for the proxy server
-	ProxySocksURL      string                 // ProxySocksURL is the URL for the proxy socks server
-	Silent             bool                   // Silent suppresses any extra text and only writes found URLs on screen.
-	Version            bool                   // Version specifies if we should just show version and exit
-	Verbose            bool                   // Verbose flag indicates whether to show verbose output or not
-	NoColor            bool                   // No-Color disables the colored output.
-	CustomHeaders      requests.CustomHeaders // Custom global headers
-	UpdateTemplates    bool                   // UpdateTemplates updates the templates installed at startup
-	TemplatesDirectory string                 // TemplatesDirectory is the directory to use for storing templates
-	JSON               bool                   // JSON writes json output to files
+	Debug                bool                   // Debug mode allows debugging request/responses for the engine
+	Silent               bool                   // Silent suppresses any extra text and only writes found URLs on screen.
+	Version              bool                   // Version specifies if we should just show version and exit
+	Verbose              bool                   // Verbose flag indicates whether to show verbose output or not
+	NoColor              bool                   // No-Color disables the colored output.
+	UpdateTemplates      bool                   // UpdateTemplates updates the templates installed at startup
+	JSON                 bool                   // JSON writes json output to files
+	JSONRequests         bool                   // write requests/responses for matches in JSON output
+	EnableProgressBar    bool                   // Enable progrss bar
+	TemplatesVersion     bool                   // Show the templates installed version
+	TemplateList         bool                   // List available templates
+	Stdin                bool                   // Stdin specifies whether stdin input was given to the process
+	StopAtFirstMatch     bool                   // Stop processing template at first full match (this may break chained requests)
+	NoMeta               bool                   // Don't display metadata for the matches
+	BulkSize             int                    // Number of targets analyzed in parallel for each template
+	TemplateThreads      int                    // Number of templates executed in parallel
+	Project              bool                   // Nuclei uses project folder to avoid sending same HTTP request multiple times
+	ProjectPath          string                 // Nuclei uses a user defined project folder
+	Timeout              int                    // Timeout is the seconds to wait for a response from the server.
+	Retries              int                    // Retries is the number of times to retry the request
+	RateLimit            int                    // Rate-Limit of requests per specified target
+	Severity             string                 // Filter templates based on their severity and only run the matching ones.
+	Target               string                 // Target is a single URL/Domain to scan usng a template
+	Targets              string                 // Targets specifies the targets to scan using templates.
+	Output               string                 // Output is the file to write found subdomains to.
+	ProxyURL             string                 // ProxyURL is the URL for the proxy server
+	ProxySocksURL        string                 // ProxySocksURL is the URL for the proxy socks server
+	TemplatesDirectory   string                 // TemplatesDirectory is the directory to use for storing templates
+	TraceLogFile         string                 // TraceLogFile specifies a file to write with the trace of all requests
+	Templates            multiStringFlag        // Signature specifies the template/templates to use
+	ExcludedTemplates    multiStringFlag        // Signature specifies the template/templates to exclude
+	CustomHeaders        requests.CustomHeaders // Custom global headers
+	Threads              int                    // Thread controls the number of concurrent requests to make.
+	BurpCollaboratorBiid string                 // Burp Collaborator BIID for polling
+}
 
-	Stdin bool // Stdin specifies whether stdin input was given to the process
+type multiStringFlag []string
+
+func (m *multiStringFlag) String() string {
+	return ""
+}
+
+func (m *multiStringFlag) Set(value string) error {
+	*m = append(*m, value)
+	return nil
 }
 
 // ParseOptions parses the command line flags provided by a user
@@ -38,7 +66,9 @@ func ParseOptions() *Options {
 	options := &Options{}
 
 	flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template")
-	flag.StringVar(&options.Templates, "t", "", "Template input file/files to run on host")
+	flag.Var(&options.Templates, "t", "Template input dir/file/files to run on host. Can be used multiple times. Supports globbing.")
+	flag.Var(&options.ExcludedTemplates, "exclude", "Template input dir/file/files to exclude. Can be used multiple times. Supports globbing.")
+	flag.StringVar(&options.Severity, "severity", "", "Filter templates based on their severity and only run the matching ones. Comma-separated values can be used to specify multiple severities.")
 	flag.StringVar(&options.Targets, "l", "", "List of URLs to run templates on")
 	flag.StringVar(&options.Output, "o", "", "File to write output to (optional)")
 	flag.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server")
@@ -46,16 +76,27 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.Silent, "silent", false, "Show only results in output")
 	flag.BoolVar(&options.Version, "version", false, "Show version of nuclei")
 	flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output")
-	flag.BoolVar(&options.NoColor, "nC", false, "Don't Use colors in output")
-	flag.IntVar(&options.Threads, "c", 10, "Number of concurrent requests to make")
+	flag.BoolVar(&options.NoColor, "no-color", false, "Disable colors in output")
 	flag.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout")
 	flag.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
 	flag.Var(&options.CustomHeaders, "H", "Custom Header.")
 	flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses")
 	flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)")
+	flag.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log")
 	flag.StringVar(&options.TemplatesDirectory, "update-directory", "", "Directory to use for storing nuclei-templates")
 	flag.BoolVar(&options.JSON, "json", false, "Write json output to files")
-
+	flag.BoolVar(&options.JSONRequests, "include-rr", false, "Write requests/responses for matches in JSON output")
+	flag.BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan")
+	flag.BoolVar(&options.TemplateList, "tl", false, "List available templates")
+	flag.IntVar(&options.RateLimit, "rate-limit", 150, "Rate-Limit (maximum requests/second")
+	flag.BoolVar(&options.StopAtFirstMatch, "stop-at-first-match", false, "Stop processing http requests at first match (this may break template/workflow logic)")
+	flag.IntVar(&options.BulkSize, "bulk-size", 25, "Maximum Number of hosts analyzed in parallel per template")
+	flag.IntVar(&options.TemplateThreads, "c", 10, "Maximum Number of templates executed in parallel")
+	flag.BoolVar(&options.Project, "project", false, "Use a project folder to avoid sending same request multiple times")
+	flag.StringVar(&options.ProjectPath, "project-path", "", "Use a user defined project folder, temporary folder is used if not specified but enabled")
+	flag.BoolVar(&options.NoMeta, "no-meta", false, "Don't display metadata for the matches")
+	flag.BoolVar(&options.TemplatesVersion, "templates-version", false, "Shows the installed nuclei-templates version")
+	flag.StringVar(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "", "Burp Collaborator BIID")
 	flag.Parse()
 
 	// Check if stdin pipe was given
@@ -71,6 +112,14 @@ func ParseOptions() *Options {
 		gologger.Infof("Current Version: %s\n", Version)
 		os.Exit(0)
 	}
+	if options.TemplatesVersion {
+		config, err := readConfiguration()
+		if err != nil {
+			gologger.Fatalf("Could not read template configuration: %s\n", err)
+		}
+		gologger.Infof("Current nuclei-templates version: %s (%s)\n", config.CurrentVersion, config.TemplatesDirectory)
+		os.Exit(0)
+	}
 
 	// Validate the options passed by the user and if any
 	// invalid options have been used, exit.
@@ -78,16 +127,86 @@ func ParseOptions() *Options {
 	if err != nil {
 		gologger.Fatalf("Program exiting: %s\n", err)
 	}
+
 	return options
 }
 
 func hasStdin() bool {
-	fi, err := os.Stdin.Stat()
+	stat, err := os.Stdin.Stat()
 	if err != nil {
 		return false
 	}
-	if fi.Mode()&os.ModeNamedPipe == 0 {
-		return false
+
+	isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0
+	isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0
+
+	return isPipedFromChrDev || isPipedFromFIFO
+}
+
+// validateOptions validates the configuration options passed
+func (options *Options) validateOptions() error {
+	// Both verbose and silent flags were used
+	if options.Verbose && options.Silent {
+		return errors.New("both verbose and silent mode specified")
 	}
-	return true
+
+	if !options.TemplateList {
+		// Check if a list of templates was provided and it exists
+		if len(options.Templates) == 0 && !options.UpdateTemplates {
+			return errors.New("no template/templates provided")
+		}
+
+		if options.Targets == "" && !options.Stdin && options.Target == "" && !options.UpdateTemplates {
+			return errors.New("no target input provided")
+		}
+	}
+
+	// Validate proxy options if provided
+	err := validateProxyURL(
+		options.ProxyURL,
+		"invalid http proxy format (It should be http://username:password@host:port)",
+	)
+	if err != nil {
+		return err
+	}
+
+	err = validateProxyURL(
+		options.ProxySocksURL,
+		"invalid socks proxy format (It should be socks5://username:password@host:port)",
+	)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func validateProxyURL(proxyURL, message string) error {
+	if proxyURL != "" && !isValidURL(proxyURL) {
+		return errors.New(message)
+	}
+
+	return nil
+}
+
+func isValidURL(urlString string) bool {
+	_, err := url.Parse(urlString)
+
+	return err == nil
+}
+
+// configureOutput configures the output on the screen
+func (options *Options) configureOutput() {
+	// If the user desires verbose output, show verbose output
+	if options.Verbose {
+		gologger.MaxLevel = gologger.Verbose
+	}
+
+	if options.NoColor {
+		gologger.UseColors = false
+	}
+
+	if options.Silent {
+		gologger.MaxLevel = gologger.Silent
+	}
 }

v2/internal/runner/paths.go

@@ -0,0 +1,48 @@
+package runner
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/projectdiscovery/gologger"
+)
+
+// isRelative checks if a given path is a relative path
+func isRelative(filePath string) bool {
+	if strings.HasPrefix(filePath, "/") || strings.Contains(filePath, ":\\") {
+		return false
+	}
+
+	return true
+}
+
+// resolvePath gets the absolute path to the template by either
+// looking in the current directory or checking the nuclei templates directory.
+//
+// Current directory is given preference over the nuclei-templates directory.
+func (r *Runner) resolvePath(templateName string) (string, error) {
+	curDirectory, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+
+	templatePath := path.Join(curDirectory, templateName)
+	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
+		gologger.Debugf("Found template in current directory: %s\n", templatePath)
+
+		return templatePath, nil
+	}
+
+	if r.templatesConfig != nil {
+		templatePath := path.Join(r.templatesConfig.TemplatesDirectory, templateName)
+		if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
+			gologger.Debugf("Found template in nuclei-templates directory: %s\n", templatePath)
+
+			return templatePath, nil
+		}
+	}
+
+	return "", fmt.Errorf("no such path found: %s", templateName)
+}

v2/internal/runner/processor.go

@@ -0,0 +1,332 @@
+package runner
+
+import (
+	"context"
+	"fmt"
+	"net/http/cookiejar"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	tengo "github.com/d5/tengo/v2"
+	"github.com/d5/tengo/v2/stdlib"
+	"github.com/karrick/godirwalk"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/internal/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
+	"github.com/projectdiscovery/nuclei/v2/pkg/executer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
+	"github.com/remeh/sizedwaitgroup"
+)
+
+// workflowTemplates contains the initialized workflow templates per template group
+type workflowTemplates struct {
+	Name      string
+	Templates []*workflows.Template
+}
+
+// processTemplateWithList processes a template and runs the enumeration on all the targets
+func (r *Runner) processTemplateWithList(p *progress.Progress, template *templates.Template, request interface{}) bool {
+	var httpExecuter *executer.HTTPExecuter
+	var dnsExecuter *executer.DNSExecuter
+	var err error
+
+	// Create an executer based on the request type.
+	switch value := request.(type) {
+	case *requests.DNSRequest:
+		dnsExecuter = executer.NewDNSExecuter(&executer.DNSOptions{
+			TraceLog:      r.traceLog,
+			Debug:         r.options.Debug,
+			Template:      template,
+			DNSRequest:    value,
+			Writer:        r.output,
+			JSON:          r.options.JSON,
+			JSONRequests:  r.options.JSONRequests,
+			NoMeta:        r.options.NoMeta,
+			ColoredOutput: !r.options.NoColor,
+			Colorizer:     r.colorizer,
+			Decolorizer:   r.decolorizer,
+			RateLimiter:   r.ratelimiter,
+		})
+	case *requests.BulkHTTPRequest:
+		httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{
+			TraceLog:         r.traceLog,
+			Debug:            r.options.Debug,
+			Template:         template,
+			BulkHTTPRequest:  value,
+			Writer:           r.output,
+			Timeout:          r.options.Timeout,
+			Retries:          r.options.Retries,
+			ProxyURL:         r.options.ProxyURL,
+			ProxySocksURL:    r.options.ProxySocksURL,
+			CustomHeaders:    r.options.CustomHeaders,
+			JSON:             r.options.JSON,
+			JSONRequests:     r.options.JSONRequests,
+			NoMeta:           r.options.NoMeta,
+			CookieReuse:      value.CookieReuse,
+			ColoredOutput:    !r.options.NoColor,
+			Colorizer:        &r.colorizer,
+			Decolorizer:      r.decolorizer,
+			StopAtFirstMatch: r.options.StopAtFirstMatch,
+			PF:               r.pf,
+			Dialer:           r.dialer,
+			RateLimiter:      r.ratelimiter,
+		})
+	}
+
+	if err != nil {
+		p.Drop(request.(*requests.BulkHTTPRequest).GetRequestCount())
+		gologger.Warningf("Could not create http client: %s\n", err)
+
+		return false
+	}
+
+	var globalresult atomicboolean.AtomBool
+
+	wg := sizedwaitgroup.New(r.options.BulkSize)
+
+	r.hm.Scan(func(k, _ []byte) error {
+		URL := string(k)
+		wg.Add()
+		go func(URL string) {
+			defer wg.Done()
+
+			var result *executer.Result
+
+			if httpExecuter != nil {
+				result = httpExecuter.ExecuteHTTP(p, URL)
+				globalresult.Or(result.GotResults)
+			}
+
+			if dnsExecuter != nil {
+				result = dnsExecuter.ExecuteDNS(p, URL)
+				globalresult.Or(result.GotResults)
+			}
+
+			if result.Error != nil {
+				gologger.Warningf("[%s] Could not execute step: %s\n", r.colorizer.Colorizer.BrightBlue(template.ID), result.Error)
+			}
+		}(URL)
+
+		return nil
+	})
+
+	wg.Wait()
+
+	// See if we got any results from the executers
+	return globalresult.Get()
+}
+
+// ProcessWorkflowWithList coming from stdin or list of targets
+func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflows.Workflow) bool {
+	result := false
+
+	workflowTemplatesList, err := r.preloadWorkflowTemplates(p, workflow)
+	if err != nil {
+		gologger.Warningf("Could not preload templates for workflow %s: %s\n", workflow.ID, err)
+		return result
+	}
+
+	logicBytes := []byte(workflow.Logic)
+
+	wg := sizedwaitgroup.New(r.options.BulkSize)
+
+	r.hm.Scan(func(k, _ []byte) error {
+		targetURL := string(k)
+		wg.Add()
+
+		go func(targetURL string) {
+			defer wg.Done()
+
+			script := tengo.NewScript(logicBytes)
+			script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
+
+			variables := make(map[string]*workflows.NucleiVar)
+
+			for _, workflowTemplate := range *workflowTemplatesList {
+				name := workflowTemplate.Name
+				variable := &workflows.NucleiVar{Templates: workflowTemplate.Templates, URL: targetURL}
+				err := script.Add(name, variable)
+				if err != nil {
+					gologger.Errorf("Could not initialize script for workflow '%s': %s\n", workflow.ID, err)
+					continue
+				}
+				variables[name] = variable
+			}
+
+			_, err := script.RunContext(context.Background())
+			if err != nil {
+				gologger.Errorf("Could not execute workflow '%s': %s\n", workflow.ID, err)
+			}
+
+			for _, variable := range variables {
+				result = !variable.IsFalsy()
+				if result {
+					break
+				}
+			}
+		}(targetURL)
+		return nil
+	})
+
+	wg.Wait()
+
+	return result
+}
+
+func (r *Runner) preloadWorkflowTemplates(p *progress.Progress, workflow *workflows.Workflow) (*[]workflowTemplates, error) {
+	var jar *cookiejar.Jar
+
+	if workflow.CookieReuse {
+		var err error
+		jar, err = cookiejar.New(nil)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Single yaml provided
+	var wflTemplatesList []workflowTemplates
+
+	for name, value := range workflow.Variables {
+		// Check if the template is an absolute path or relative path.
+		// If the path is absolute, use it. Otherwise,
+		if isRelative(value) {
+			newPath, err := r.resolvePath(value)
+			if err != nil {
+				newPath, err = resolvePathWithBaseFolder(filepath.Dir(workflow.GetPath()), value)
+				if err != nil {
+					return nil, err
+				}
+			}
+
+			value = newPath
+		}
+
+		var wtlst []*workflows.Template
+
+		if strings.HasSuffix(value, ".yaml") {
+			t, err := templates.Parse(value)
+			if err != nil {
+				return nil, err
+			}
+
+			template := &workflows.Template{Progress: p}
+			if len(t.BulkRequestsHTTP) > 0 {
+				template.HTTPOptions = &executer.HTTPOptions{
+					TraceLog:         r.traceLog,
+					Debug:            r.options.Debug,
+					Writer:           r.output,
+					Template:         t,
+					Timeout:          r.options.Timeout,
+					Retries:          r.options.Retries,
+					ProxyURL:         r.options.ProxyURL,
+					ProxySocksURL:    r.options.ProxySocksURL,
+					CustomHeaders:    r.options.CustomHeaders,
+					JSON:             r.options.JSON,
+					JSONRequests:     r.options.JSONRequests,
+					CookieJar:        jar,
+					ColoredOutput:    !r.options.NoColor,
+					Colorizer:        &r.colorizer,
+					Decolorizer:      r.decolorizer,
+					PF:               r.pf,
+					RateLimiter:      r.ratelimiter,
+					NoMeta:           r.options.NoMeta,
+					StopAtFirstMatch: r.options.StopAtFirstMatch,
+					Dialer:           r.dialer,
+				}
+			} else if len(t.RequestsDNS) > 0 {
+				template.DNSOptions = &executer.DNSOptions{
+					TraceLog:      r.traceLog,
+					Debug:         r.options.Debug,
+					Template:      t,
+					Writer:        r.output,
+					JSON:          r.options.JSON,
+					JSONRequests:  r.options.JSONRequests,
+					ColoredOutput: !r.options.NoColor,
+					Colorizer:     r.colorizer,
+					Decolorizer:   r.decolorizer,
+					NoMeta:        r.options.NoMeta,
+					RateLimiter:   r.ratelimiter,
+				}
+			}
+
+			if template.DNSOptions != nil || template.HTTPOptions != nil {
+				wtlst = append(wtlst, template)
+			}
+		} else {
+			matches := []string{}
+
+			err := godirwalk.Walk(value, &godirwalk.Options{
+				Callback: func(path string, d *godirwalk.Dirent) error {
+					if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
+						matches = append(matches, path)
+					}
+
+					return nil
+				},
+				ErrorCallback: func(path string, err error) godirwalk.ErrorAction {
+					return godirwalk.SkipNode
+				},
+				Unsorted: true,
+			})
+
+			if err != nil {
+				return nil, err
+			}
+
+			// 0 matches means no templates were found in directory
+			if len(matches) == 0 {
+				return nil, fmt.Errorf("no match found in the directory %s", value)
+			}
+
+			for _, match := range matches {
+				t, err := templates.Parse(match)
+				if err != nil {
+					return nil, err
+				}
+				template := &workflows.Template{Progress: p}
+				if len(t.BulkRequestsHTTP) > 0 {
+					template.HTTPOptions = &executer.HTTPOptions{
+						Debug:         r.options.Debug,
+						Writer:        r.output,
+						Template:      t,
+						Timeout:       r.options.Timeout,
+						Retries:       r.options.Retries,
+						ProxyURL:      r.options.ProxyURL,
+						ProxySocksURL: r.options.ProxySocksURL,
+						CustomHeaders: r.options.CustomHeaders,
+						CookieJar:     jar,
+						TraceLog:      r.traceLog,
+					}
+				} else if len(t.RequestsDNS) > 0 {
+					template.DNSOptions = &executer.DNSOptions{
+						Debug:    r.options.Debug,
+						Template: t,
+						Writer:   r.output,
+						TraceLog: r.traceLog,
+					}
+				}
+				if template.DNSOptions != nil || template.HTTPOptions != nil {
+					wtlst = append(wtlst, template)
+				}
+			}
+		}
+		wflTemplatesList = append(wflTemplatesList, workflowTemplates{Name: name, Templates: wtlst})
+	}
+
+	return &wflTemplatesList, nil
+}
+
+func resolvePathWithBaseFolder(baseFolder, templateName string) (string, error) {
+	templatePath := path.Join(baseFolder, templateName)
+	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
+		gologger.Debugf("Found template in current directory: %s\n", templatePath)
+		return templatePath, nil
+	}
+
+	return "", fmt.Errorf("no such path found: %s", templateName)
+}

v2/internal/runner/runner.go

@@ -2,472 +2,314 @@ package runner
 
 import (
 	"bufio"
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
 	"os"
+	"regexp"
 	"strings"
-	"sync"
 
-	"github.com/d5/tengo/v2"
-	"github.com/karrick/godirwalk"
+	"github.com/logrusorgru/aurora"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/executor"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
+	"github.com/projectdiscovery/hmap/store/hybrid"
+	"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
+	"github.com/projectdiscovery/nuclei/v2/internal/progress"
+	"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
+	"github.com/projectdiscovery/nuclei/v2/pkg/collaborator"
+	"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
+	"github.com/remeh/sizedwaitgroup"
+	"go.uber.org/ratelimit"
 )
 
 // Runner is a client for running the enumeration process.
 type Runner struct {
+	inputCount int64
+
+	traceLog tracelog.Log
+
 	// output is the output file to write if any
-	output      *os.File
-	outputMutex *sync.Mutex
+	output *bufwriter.Writer
 
-	tempFile        string
 	templatesConfig *nucleiConfig
 	// options contains configuration options for runner
 	options *Options
+
+	pf *projectfile.ProjectFile
+
+	// progress tracking
+	progress *progress.Progress
+
+	// output coloring
+	colorizer   colorizer.NucleiColorizer
+	decolorizer *regexp.Regexp
+
+	// rate limiter
+	ratelimiter ratelimit.Limiter
+
+	// input deduplication
+	hm     *hybrid.HybridMap
+	dialer *fastdialer.Dialer
 }
 
 // New creates a new client for running enumeration process.
 func New(options *Options) (*Runner, error) {
 	runner := &Runner{
-		outputMutex: &sync.Mutex{},
-		options:     options,
+		traceLog: &tracelog.NoopLogger{},
+		options:  options,
+	}
+	if options.TraceLogFile != "" {
+		fileLog, err := tracelog.NewFileLogger(options.TraceLogFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create file trace logger")
+		}
+		runner.traceLog = fileLog
 	}
 
 	if err := runner.updateTemplates(); err != nil {
-		gologger.Warningf("Could not update templates: %s\n", err)
+		gologger.Labelf("Could not update templates: %s\n", err)
 	}
-	if (options.Templates == "" || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
+
+	// output coloring
+	useColor := !options.NoColor
+	runner.colorizer = *colorizer.NewNucleiColorizer(aurora.NewAurora(useColor))
+
+	if useColor {
+		// compile a decolorization regex to cleanup file output messages
+		runner.decolorizer = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
+	}
+
+	if options.TemplateList {
+		runner.listAvailableTemplates()
 		os.Exit(0)
 	}
 
-	// If we have stdin, write it to a new file
-	if options.Stdin {
-		tempInput, err := ioutil.TempFile("", "stdin-input-*")
-		if err != nil {
-			return nil, err
-		}
-		if _, err := io.Copy(tempInput, os.Stdin); err != nil {
-			return nil, err
-		}
-		runner.tempFile = tempInput.Name()
-		tempInput.Close()
+	if (len(options.Templates) == 0 || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
+		os.Exit(0)
 	}
-	// If we have single target, write it to a new file
+	// Read nucleiignore file if given a templateconfig
+	if runner.templatesConfig != nil {
+		runner.readNucleiIgnoreFile()
+	}
+
+	if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil {
+		gologger.Fatalf("Could not create temporary input file: %s\n", err)
+	} else {
+		runner.hm = hm
+	}
+
+	runner.inputCount = 0
+	dupeCount := 0
+
+	// Handle single target
 	if options.Target != "" {
-		tempInput, err := ioutil.TempFile("", "stdin-input-*")
-		if err != nil {
-			return nil, err
+		runner.inputCount++
+		// nolint:errcheck // ignoring error
+		runner.hm.Set(options.Target, nil)
+	}
+
+	// Handle stdin
+	if options.Stdin {
+		scanner := bufio.NewScanner(os.Stdin)
+		for scanner.Scan() {
+			url := strings.TrimSpace(scanner.Text())
+			// skip empty lines
+			if url == "" {
+				continue
+			}
+
+			// skip dupes
+			if _, ok := runner.hm.Get(url); ok {
+				dupeCount++
+				continue
+			}
+
+			runner.inputCount++
+			// nolint:errcheck // ignoring error
+			runner.hm.Set(url, nil)
 		}
-		tempInput.WriteString(options.Target)
-		runner.tempFile = tempInput.Name()
-		tempInput.Close()
+	}
+
+	// Handle taget file
+	if options.Targets != "" {
+		input, err := os.Open(options.Targets)
+		if err != nil {
+			gologger.Fatalf("Could not open targets file '%s': %s\n", options.Targets, err)
+		}
+		scanner := bufio.NewScanner(input)
+		for scanner.Scan() {
+			url := strings.TrimSpace(scanner.Text())
+			// skip empty lines
+			if url == "" {
+				continue
+			}
+
+			// skip dupes
+			if _, ok := runner.hm.Get(url); ok {
+				dupeCount++
+				continue
+			}
+
+			runner.inputCount++
+			// nolint:errcheck // ignoring error
+			runner.hm.Set(url, nil)
+		}
+		input.Close()
+	}
+
+	if dupeCount > 0 {
+		gologger.Labelf("Supplied input was automatically deduplicated (%d removed).", dupeCount)
 	}
 
 	// Create the output file if asked
 	if options.Output != "" {
-		output, err := os.Create(options.Output)
-		if err != nil {
-			gologger.Fatalf("Could not create output file '%s': %s\n", options.Output, err)
+		output, errBufWriter := bufwriter.New(options.Output)
+		if errBufWriter != nil {
+			gologger.Fatalf("Could not create output file '%s': %s\n", options.Output, errBufWriter)
 		}
 		runner.output = output
 	}
+
+	// Creates the progress tracking object
+	runner.progress = progress.NewProgress(options.EnableProgressBar)
+
+	// create project file if requested or load existing one
+	if options.Project {
+		var projectFileErr error
+		runner.pf, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""})
+		if projectFileErr != nil {
+			return nil, projectFileErr
+		}
+	}
+
+	// Enable Polling
+	if options.BurpCollaboratorBiid != "" {
+		collaborator.DefaultCollaborator.Collab.AddBIID(options.BurpCollaboratorBiid)
+	}
+
+	// Create Dialer
+	var err error
+	runner.dialer, err = fastdialer.NewDialer(fastdialer.DefaultOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	if options.RateLimit > 0 {
+		runner.ratelimiter = ratelimit.New(options.RateLimit)
+	} else {
+		runner.ratelimiter = ratelimit.NewUnlimited()
+	}
+
 	return runner, nil
 }
 
 // Close releases all the resources and cleans up
 func (r *Runner) Close() {
-	r.output.Close()
-	os.Remove(r.tempFile)
+	if r.output != nil {
+		r.output.Close()
+	}
+	r.hm.Close()
+	if r.pf != nil {
+		r.pf.Close()
+	}
 }
 
 // RunEnumeration sets up the input layer for giving input nuclei.
 // binary and runs the actual enumeration
 func (r *Runner) RunEnumeration() {
-	var err error
+	// resolves input templates definitions and any optional exclusion
+	includedTemplates := r.getTemplatesFor(r.options.Templates)
+	excludedTemplates := r.getTemplatesFor(r.options.ExcludedTemplates)
+	// defaults to all templates
+	allTemplates := includedTemplates
 
-	// Check if the template is an absolute path or relative path.
-	// If the path is absolute, use it. Otherwise,
-	if r.isRelative(r.options.Templates) {
-		newPath, err := r.resolvePath(r.options.Templates)
-		if err != nil {
-			gologger.Errorf("Could not find template file '%s': %s\n", r.options.Templates, err)
-			return
+	if len(excludedTemplates) > 0 {
+		excludedMap := make(map[string]struct{}, len(excludedTemplates))
+		for _, excl := range excludedTemplates {
+			excludedMap[excl] = struct{}{}
 		}
-		r.options.Templates = newPath
-	}
+		// rebuild list with only non-excluded templates
+		allTemplates = []string{}
 
-	// Single yaml provided
-	if strings.HasSuffix(r.options.Templates, ".yaml") {
-		t, err := r.parse(r.options.Templates)
-		switch t.(type) {
-		case *templates.Template:
-			var results bool
-			template := t.(*templates.Template)
-			// process http requests
-			for _, request := range template.RequestsHTTP {
-				results = r.processTemplateRequest(template, request)
+		for _, incl := range includedTemplates {
+			if _, found := excludedMap[incl]; !found {
+				allTemplates = append(allTemplates, incl)
+			} else {
+				gologger.Warningf("Excluding '%s'", incl)
 			}
-			// process dns requests
-			for _, request := range template.RequestsDNS {
-				dnsResults := r.processTemplateRequest(template, request)
-				if !results {
-					results = dnsResults
-				}
-			}
-
-			if !results {
-				if r.output != nil {
-					outputFile := r.output.Name()
-					r.output.Close()
-					os.Remove(outputFile)
-				}
-				gologger.Infof("No results found for the template. Happy hacking!")
-			}
-		case *workflows.Workflow:
-			workflow := t.(*workflows.Workflow)
-			r.ProcessWorkflowWithList(workflow)
-		default:
-			gologger.Errorf("Could not parse file '%s': %s\n", r.options.Templates, err)
 		}
-		return
 	}
 
-	// If the template passed is a directory
-	matches := []string{}
+	// pre-parse all the templates, apply filters
+	availableTemplates, workflowCount := r.getParsedTemplatesFor(allTemplates, r.options.Severity)
+	templateCount := len(availableTemplates)
+	hasWorkflows := workflowCount > 0
 
-	// Recursively walk down the Templates directory and run all the template file checks
-	err = godirwalk.Walk(r.options.Templates, &godirwalk.Options{
-		Callback: func(path string, d *godirwalk.Dirent) error {
-			if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
-				matches = append(matches, path)
-			}
-			return nil
-		},
-		ErrorCallback: func(path string, err error) godirwalk.ErrorAction {
-			return godirwalk.SkipNode
-		},
-		Unsorted: true,
-	})
-	if err != nil {
-		gologger.Fatalf("Could not find templates in directory '%s': %s\n", r.options.Templates, err)
-	}
 	// 0 matches means no templates were found in directory
-	if len(matches) == 0 {
-		gologger.Fatalf("Error, no templates found in directory: '%s'\n", r.options.Templates)
+	if templateCount == 0 {
+		gologger.Fatalf("Error, no templates were found.\n")
 	}
 
-	var results bool
-	for _, match := range matches {
-		t, err := r.parse(match)
-		switch t.(type) {
+	gologger.Infof("Using %s rules (%s templates, %s workflows)",
+		r.colorizer.Colorizer.Bold(templateCount).String(),
+		r.colorizer.Colorizer.Bold(templateCount-workflowCount).String(),
+		r.colorizer.Colorizer.Bold(workflowCount).String())
+
+	// precompute total request count
+	var totalRequests int64 = 0
+
+	for _, t := range availableTemplates {
+		switch av := t.(type) {
 		case *templates.Template:
-			template := t.(*templates.Template)
-			for _, request := range template.RequestsDNS {
-				dnsResults := r.processTemplateRequest(template, request)
-				if dnsResults {
-					results = dnsResults
-				}
-			}
-			for _, request := range template.RequestsHTTP {
-				httpResults := r.processTemplateRequest(template, request)
-				if httpResults {
-					results = httpResults
-				}
-			}
+			totalRequests += (av.GetHTTPRequestCount() + av.GetDNSRequestCount()) * r.inputCount
 		case *workflows.Workflow:
-			workflow := t.(*workflows.Workflow)
-			r.ProcessWorkflowWithList(workflow)
-		default:
-			gologger.Errorf("Could not parse file '%s': %s\n", r.options.Templates, err)
-		}
+			// workflows will dynamically adjust the totals while running, as
+			// it can't be know in advance which requests will be called
+		} // nolint:wsl // comment
 	}
-	if !results {
+
+	results := atomicboolean.New()
+	wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
+	// Starts polling or ignore
+	collaborator.DefaultCollaborator.Poll()
+
+	if r.inputCount == 0 {
+		gologger.Errorf("Could not find any valid input URLs.")
+	} else if totalRequests > 0 || hasWorkflows {
+		// tracks global progress and captures stdout/stderr until p.Wait finishes
+		p := r.progress
+		p.Init(r.inputCount, templateCount, totalRequests)
+
+		for _, t := range availableTemplates {
+			wgtemplates.Add()
+			go func(template interface{}) {
+				defer wgtemplates.Done()
+				switch tt := template.(type) {
+				case *templates.Template:
+					for _, request := range tt.RequestsDNS {
+						results.Or(r.processTemplateWithList(p, tt, request))
+					}
+					for _, request := range tt.BulkRequestsHTTP {
+						results.Or(r.processTemplateWithList(p, tt, request))
+					}
+				case *workflows.Workflow:
+					results.Or(r.processWorkflowWithList(p, template.(*workflows.Workflow)))
+				}
+			}(t)
+		}
+
+		wgtemplates.Wait()
+		p.Stop()
+	}
+
+	if !results.Get() {
 		if r.output != nil {
-			outputFile := r.output.Name()
 			r.output.Close()
-			os.Remove(outputFile)
+			os.Remove(r.options.Output)
 		}
-		gologger.Infof("No results found for the template. Happy hacking!")
-	}
-	return
-}
 
-// processTemplate processes a template and runs the enumeration on all the targets
-func (r *Runner) processTemplateRequest(template *templates.Template, request interface{}) bool {
-	var file *os.File
-	var err error
-
-	// Handle a list of hosts as argument
-	if r.options.Targets != "" {
-		file, err = os.Open(r.options.Targets)
-	} else if r.options.Stdin || r.options.Target != "" {
-		file, err = os.Open(r.tempFile)
-	}
-	if err != nil {
-		gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err)
-	}
-	results := r.processTemplateWithList(template, request, file)
-	file.Close()
-	return results
-}
-
-// processDomain processes the list with a template
-func (r *Runner) processTemplateWithList(template *templates.Template, request interface{}, reader io.Reader) bool {
-	// Display the message for the template
-	message := fmt.Sprintf("[%s] Loaded template %s (@%s)", template.ID, template.Info.Name, template.Info.Author)
-	if template.Info.Severity != "" {
-		message += " [" + template.Info.Severity + "]"
-	}
-	gologger.Infof("%s\n", message)
-
-	var writer *bufio.Writer
-	if r.output != nil {
-		writer = bufio.NewWriter(r.output)
-		defer writer.Flush()
-	}
-
-	var httpExecutor *executor.HTTPExecutor
-	var dnsExecutor *executor.DNSExecutor
-	var err error
-
-	// Create an executor based on the request type.
-	switch value := request.(type) {
-	case *requests.DNSRequest:
-		dnsExecutor = executor.NewDNSExecutor(&executor.DNSOptions{
-			Debug:      r.options.Debug,
-			Template:   template,
-			DNSRequest: value,
-			Writer:     writer,
-			JSON:       r.options.JSON,
-		})
-	case *requests.HTTPRequest:
-		httpExecutor, err = executor.NewHTTPExecutor(&executor.HTTPOptions{
-			Debug:         r.options.Debug,
-			Template:      template,
-			HTTPRequest:   value,
-			Writer:        writer,
-			Timeout:       r.options.Timeout,
-			Retries:       r.options.Retries,
-			ProxyURL:      r.options.ProxyURL,
-			ProxySocksURL: r.options.ProxySocksURL,
-			CustomHeaders: r.options.CustomHeaders,
-			JSON:          r.options.JSON,
-		})
-	}
-	if err != nil {
-		gologger.Warningf("Could not create http client: %s\n", err)
-		return false
-	}
-
-	limiter := make(chan struct{}, r.options.Threads)
-	wg := &sync.WaitGroup{}
-
-	scanner := bufio.NewScanner(reader)
-	for scanner.Scan() {
-		text := scanner.Text()
-		if text == "" {
-			continue
-		}
-		limiter <- struct{}{}
-		wg.Add(1)
-
-		go func(URL string) {
-			var err error
-
-			if httpExecutor != nil {
-				err = httpExecutor.ExecuteHTTP(URL)
-			}
-			if dnsExecutor != nil {
-				err = dnsExecutor.ExecuteDNS(URL)
-			}
-			if err != nil {
-				gologger.Warningf("Could not execute step: %s\n", err)
-			}
-			<-limiter
-			wg.Done()
-		}(text)
-	}
-	close(limiter)
-	wg.Wait()
-
-	// See if we got any results from the executors
-	var results bool
-	if httpExecutor != nil {
-		results = httpExecutor.GotResults()
-	}
-	if dnsExecutor != nil {
-		if !results {
-			results = dnsExecutor.GotResults()
-		}
-	}
-	return results
-}
-
-// ProcessWorkflowWithList coming from stdin or list of targets
-func (r *Runner) ProcessWorkflowWithList(workflow *workflows.Workflow) {
-	var file *os.File
-	var err error
-	// Handle a list of hosts as argument
-	if r.options.Targets != "" {
-		file, err = os.Open(r.options.Targets)
-	} else if r.options.Stdin {
-		file, err = os.Open(r.tempFile)
-	}
-	if err != nil {
-		gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err)
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		text := scanner.Text()
-		if text == "" {
-			continue
-		}
-		if err := r.ProcessWorkflow(workflow, text); err != nil {
-			gologger.Warningf("Could not run workflow for %s: %s\n", text, err)
-		}
+		gologger.Infof("No results found. Happy hacking!")
 	}
 }
-
-// ProcessWorkflow towards an URL
-func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error {
-	script := tengo.NewScript([]byte(workflow.Logic))
-
-	for name, value := range workflow.Variables {
-		var writer *bufio.Writer
-		if r.output != nil {
-			writer = bufio.NewWriter(r.output)
-			defer writer.Flush()
-		}
-
-		// Check if the template is an absolute path or relative path.
-		// If the path is absolute, use it. Otherwise,
-		if r.isRelative(value) {
-			newPath, err := r.resolvePath(value)
-			if err != nil {
-				return err
-			}
-			value = newPath
-		}
-
-		// Single yaml provided
-		var templatesList []*workflows.Template
-		if strings.HasSuffix(value, ".yaml") {
-			t, err := templates.Parse(value)
-			if err != nil {
-				return err
-			}
-			template := &workflows.Template{}
-			if len(t.RequestsHTTP) > 0 {
-				template.HTTPOptions = &executor.HTTPOptions{
-					Debug:         r.options.Debug,
-					Writer:        writer,
-					Template:      t,
-					Timeout:       r.options.Timeout,
-					Retries:       r.options.Retries,
-					ProxyURL:      r.options.ProxyURL,
-					ProxySocksURL: r.options.ProxySocksURL,
-					CustomHeaders: r.options.CustomHeaders,
-				}
-			} else if len(t.RequestsDNS) > 0 {
-				template.DNSOptions = &executor.DNSOptions{
-					Debug:    r.options.Debug,
-					Template: t,
-					Writer:   writer,
-				}
-			}
-			if template.DNSOptions != nil || template.HTTPOptions != nil {
-				templatesList = append(templatesList, template)
-			}
-		} else {
-			matches := []string{}
-
-			err := godirwalk.Walk(value, &godirwalk.Options{
-				Callback: func(path string, d *godirwalk.Dirent) error {
-					if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
-						matches = append(matches, path)
-					}
-					return nil
-				},
-				ErrorCallback: func(path string, err error) godirwalk.ErrorAction {
-					return godirwalk.SkipNode
-				},
-				Unsorted: true,
-			})
-			if err != nil {
-				return err
-			}
-			// 0 matches means no templates were found in directory
-			if len(matches) == 0 {
-				return errors.New("no match found in the directory")
-			}
-
-			for _, match := range matches {
-				t, err := templates.Parse(match)
-				if err != nil {
-					return err
-				}
-				template := &workflows.Template{}
-				if len(t.RequestsHTTP) > 0 {
-					template.HTTPOptions = &executor.HTTPOptions{
-						Debug:         r.options.Debug,
-						Writer:        writer,
-						Template:      t,
-						Timeout:       r.options.Timeout,
-						Retries:       r.options.Retries,
-						ProxyURL:      r.options.ProxyURL,
-						ProxySocksURL: r.options.ProxySocksURL,
-						CustomHeaders: r.options.CustomHeaders,
-					}
-				} else if len(t.RequestsDNS) > 0 {
-					template.DNSOptions = &executor.DNSOptions{
-						Debug:    r.options.Debug,
-						Template: t,
-						Writer:   writer,
-					}
-				}
-				if template.DNSOptions != nil || template.HTTPOptions != nil {
-					templatesList = append(templatesList, template)
-				}
-			}
-		}
-
-		script.Add(name, &workflows.NucleiVar{Templates: templatesList, URL: URL})
-	}
-
-	_, err := script.RunContext(context.Background())
-	if err != nil {
-		gologger.Errorf("Could not execute workflow '%s': %s\n", workflow.ID, err)
-		return err
-	}
-	return nil
-}
-
-func (r *Runner) parse(file string) (interface{}, error) {
-	// check if it's a template
-	template, errTemplate := templates.Parse(file)
-	if errTemplate == nil {
-		return template, nil
-	}
-
-	// check if it's a workflow
-	workflow, errWorkflow := workflows.Parse(file)
-	if errWorkflow == nil {
-		return workflow, nil
-	}
-
-	if errTemplate != nil {
-		return nil, errTemplate
-	}
-	if errWorkflow != nil {
-		return nil, errWorkflow
-	}
-	return nil, errors.New("unknown error occured")
-}

v2/internal/runner/templates.go

@@ -0,0 +1,302 @@
+package runner
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/karrick/godirwalk"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
+)
+
+// getTemplatesFor parses the specified input template definitions and returns a list of unique, absolute template paths.
+func (r *Runner) getTemplatesFor(definitions []string) []string {
+	// keeps track of processed dirs and files
+	processed := make(map[string]bool)
+	allTemplates := []string{}
+
+	// parses user input, handle file/directory cases and produce a list of unique templates
+	for _, t := range definitions {
+		var absPath string
+
+		var err error
+
+		if strings.Contains(t, "*") {
+			dirs := strings.Split(t, "/")
+			priorDir := strings.Join(dirs[:len(dirs)-1], "/")
+			absPath, err = r.resolvePathIfRelative(priorDir)
+			absPath += "/" + dirs[len(dirs)-1]
+		} else {
+			// resolve and convert relative to absolute path
+			absPath, err = r.resolvePathIfRelative(t)
+		}
+
+		if err != nil {
+			gologger.Errorf("Could not find template file '%s': %s\n", t, err)
+			continue
+		}
+
+		// Template input includes a wildcard
+		if strings.Contains(absPath, "*") {
+			var matches []string
+			matches, err = filepath.Glob(absPath)
+
+			if err != nil {
+				gologger.Labelf("Wildcard found, but unable to glob '%s': %s\n", absPath, err)
+
+				continue
+			}
+
+			// couldn't find templates in directory
+			if len(matches) == 0 {
+				gologger.Labelf("Error, no templates were found with '%s'.\n", absPath)
+				continue
+			} else {
+				gologger.Labelf("Identified %d templates\n", len(matches))
+			}
+
+			for _, match := range matches {
+				if !r.checkIfInNucleiIgnore(match) {
+					processed[match] = true
+
+					allTemplates = append(allTemplates, match)
+				}
+			}
+		} else {
+			// determine file/directory
+			isFile, err := isFilePath(absPath)
+			if err != nil {
+				gologger.Errorf("Could not stat '%s': %s\n", absPath, err)
+				continue
+			}
+			// test for uniqueness
+			if !isNewPath(absPath, processed) {
+				continue
+			}
+			// mark this absolute path as processed
+			// - if it's a file, we'll never process it again
+			// - if it's a dir, we'll never walk it again
+			processed[absPath] = true
+
+			if isFile {
+				allTemplates = append(allTemplates, absPath)
+			} else {
+				matches := []string{}
+
+				// Recursively walk down the Templates directory and run all the template file checks
+				err := directoryWalker(
+					absPath,
+					func(path string, d *godirwalk.Dirent) error {
+						if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
+							if !r.checkIfInNucleiIgnore(path) && isNewPath(path, processed) {
+								matches = append(matches, path)
+								processed[path] = true
+							}
+						}
+						return nil
+					},
+				)
+
+				// directory couldn't be walked
+				if err != nil {
+					gologger.Labelf("Could not find templates in directory '%s': %s\n", absPath, err)
+					continue
+				}
+
+				// couldn't find templates in directory
+				if len(matches) == 0 {
+					gologger.Labelf("Error, no templates were found in '%s'.\n", absPath)
+					continue
+				}
+
+				allTemplates = append(allTemplates, matches...)
+			}
+		}
+	}
+
+	return allTemplates
+}
+
+// getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered
+// by severity, along with a flag indicating if workflows are present.
+func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities string) (parsedTemplates []interface{}, workflowCount int) {
+	workflowCount = 0
+	severities = strings.ToLower(severities)
+	allSeverities := strings.Split(severities, ",")
+	filterBySeverity := len(severities) > 0
+
+	gologger.Infof("Loading templates...")
+
+	for _, match := range templatePaths {
+		t, err := r.parseTemplateFile(match)
+		switch tp := t.(type) {
+		case *templates.Template:
+			// only include if severity matches or no severity filtering
+			sev := strings.ToLower(tp.Info["severity"])
+			if !filterBySeverity || hasMatchingSeverity(sev, allSeverities) {
+				parsedTemplates = append(parsedTemplates, tp)
+				gologger.Infof("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"]))
+			} else {
+				gologger.Warningf("Excluding template %s due to severity filter (%s not in [%s])", tp.ID, sev, severities)
+			}
+		case *workflows.Workflow:
+			parsedTemplates = append(parsedTemplates, tp)
+			gologger.Infof("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"]))
+			workflowCount++
+		default:
+			gologger.Errorf("Could not parse file '%s': %s\n", match, err)
+		}
+	}
+
+	return parsedTemplates, workflowCount
+}
+
+func (r *Runner) parseTemplateFile(file string) (interface{}, error) {
+	// check if it's a template
+	template, errTemplate := templates.Parse(file)
+	if errTemplate == nil {
+		return template, nil
+	}
+
+	// check if it's a workflow
+	workflow, errWorkflow := workflows.Parse(file)
+	if errWorkflow == nil {
+		return workflow, nil
+	}
+
+	if errTemplate != nil {
+		return nil, errTemplate
+	}
+
+	if errWorkflow != nil {
+		return nil, errWorkflow
+	}
+
+	return nil, errors.New("unknown error occurred")
+}
+
+func (r *Runner) templateLogMsg(id, name, author, severity string) string {
+	// Display the message for the template
+	message := fmt.Sprintf("[%s] %s (%s)",
+		r.colorizer.Colorizer.BrightBlue(id).String(),
+		r.colorizer.Colorizer.Bold(name).String(),
+		r.colorizer.Colorizer.BrightYellow("@"+author).String())
+
+	if severity != "" {
+		message += " [" + r.colorizer.GetColorizedSeverity(severity) + "]"
+	}
+
+	return message
+}
+
+func (r *Runner) logAvailableTemplate(tplPath string) {
+	t, err := r.parseTemplateFile(tplPath)
+	if t != nil {
+		switch tp := t.(type) {
+		case *templates.Template:
+			gologger.Silentf("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"]))
+		case *workflows.Workflow:
+			gologger.Silentf("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"]))
+		default:
+			gologger.Errorf("Could not parse file '%s': %s\n", tplPath, err)
+		}
+	}
+}
+
+// ListAvailableTemplates prints available templates to stdout
+func (r *Runner) listAvailableTemplates() {
+	if r.templatesConfig == nil {
+		return
+	}
+
+	if _, err := os.Stat(r.templatesConfig.TemplatesDirectory); os.IsNotExist(err) {
+		gologger.Errorf("%s does not exists", r.templatesConfig.TemplatesDirectory)
+		return
+	}
+
+	gologger.Silentf(
+		"\nListing available v.%s nuclei templates for %s",
+		r.templatesConfig.CurrentVersion,
+		r.templatesConfig.TemplatesDirectory,
+	)
+	err := directoryWalker(
+		r.templatesConfig.TemplatesDirectory,
+		func(path string, d *godirwalk.Dirent) error {
+			if d.IsDir() && path != r.templatesConfig.TemplatesDirectory {
+				gologger.Silentf("\n%s:\n\n", r.colorizer.Colorizer.Bold(r.colorizer.Colorizer.BgBrightBlue(d.Name())).String())
+			} else if strings.HasSuffix(path, ".yaml") {
+				r.logAvailableTemplate(path)
+			}
+
+			return nil
+		},
+	)
+
+	// directory couldn't be walked
+	if err != nil {
+		gologger.Labelf("Could not find templates in directory '%s': %s\n", r.templatesConfig.TemplatesDirectory, err)
+	}
+}
+
+func (r *Runner) resolvePathIfRelative(filePath string) (string, error) {
+	if isRelative(filePath) {
+		newPath, err := r.resolvePath(filePath)
+
+		if err != nil {
+			return "", err
+		}
+
+		return newPath, nil
+	}
+
+	return filePath, nil
+}
+
+func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool {
+	for _, s := range allowedSeverities {
+		if s != "" && strings.HasPrefix(templateSeverity, s) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error {
+	err := godirwalk.Walk(fsPath, &godirwalk.Options{
+		Callback: callback,
+		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
+			return godirwalk.SkipNode
+		},
+		Unsorted: true,
+	})
+
+	// directory couldn't be walked
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func isFilePath(filePath string) (bool, error) {
+	info, err := os.Stat(filePath)
+	if err != nil {
+		return false, err
+	}
+
+	return info.Mode().IsRegular(), nil
+}
+
+func isNewPath(filePath string, pathMap map[string]bool) bool {
+	if _, already := pathMap[filePath]; already {
+		gologger.Warningf("Skipping already specified path '%s'", filePath)
+		return false
+	}
+
+	return true
+}

v2/internal/runner/update.go

@@ -12,66 +12,18 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"regexp"
 	"strings"
+	"time"
 
 	"github.com/blang/semver"
 	"github.com/google/go-github/v32/github"
-	jsoniter "github.com/json-iterator/go"
 	"github.com/projectdiscovery/gologger"
 )
 
-// nucleiConfig contains some configuration options for nuclei
-type nucleiConfig struct {
-	TemplatesDirectory string `json:"templates-directory,omitempty"`
-	CurrentVersion     string `json:"current-version,omitempty"`
-}
-
-// nucleiConfigFilename is the filename of nuclei configuration file.
-const nucleiConfigFilename = ".nuclei-config.json"
-
-var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
-
-// readConfiguration reads the nuclei configuration file from disk.
-func (r *Runner) readConfiguration() (*nucleiConfig, error) {
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return nil, err
-	}
-
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
-	file, err := os.Open(templatesConfigFile)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-
-	config := &nucleiConfig{}
-	if err = jsoniter.NewDecoder(file).Decode(config); err != nil {
-		return nil, err
-	}
-	return config, nil
-}
-
-// readConfiguration reads the nuclei configuration file from disk.
-func (r *Runner) writeConfiguration(config *nucleiConfig) error {
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return err
-	}
-
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
-	file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	if err = jsoniter.NewEncoder(file).Encode(config); err != nil {
-		return err
-	}
-	return nil
-}
+const (
+	userName = "projectdiscovery"
+	repoName = "nuclei-templates"
+)
 
 // updateTemplates checks if the default list of nuclei-templates
 // exist in the users home directory, if not the latest revision
@@ -86,14 +38,16 @@ func (r *Runner) updateTemplates() error {
 	}
 
 	templatesConfigFile := path.Join(home, nucleiConfigFilename)
-	if _, err := os.Stat(templatesConfigFile); !os.IsNotExist(err) {
-		config, err := r.readConfiguration()
+	if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
+		config, readErr := readConfiguration()
 		if err != nil {
-			return err
+			return readErr
 		}
 		r.templatesConfig = config
 	}
 
+	ctx := context.Background()
+
 	if r.templatesConfig == nil || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
 		if !r.options.UpdateTemplates {
 			gologger.Labelf("nuclei-templates are not installed, use update-templates flag.\n")
@@ -104,39 +58,54 @@ func (r *Runner) updateTemplates() error {
 		if r.options.TemplatesDirectory != "" {
 			home = r.options.TemplatesDirectory
 		}
+
 		r.templatesConfig = &nucleiConfig{TemplatesDirectory: path.Join(home, "nuclei-templates")}
-		os.RemoveAll(r.templatesConfig.TemplatesDirectory)
 
 		// Download the repository and also write the revision to a HEAD file.
-		version, asset, err := r.getLatestReleaseFromGithub()
-		if err != nil {
-			return err
+		version, asset, getErr := r.getLatestReleaseFromGithub()
+		if getErr != nil {
+			return getErr
 		}
 
 		gologger.Verbosef("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory)
 
-		if err = r.downloadReleaseAndUnzip(asset.GetZipballURL()); err != nil {
+		err = r.downloadReleaseAndUnzip(ctx, asset.GetZipballURL())
+		if err != nil {
 			return err
 		}
+
 		r.templatesConfig.CurrentVersion = version.String()
-		if err = r.writeConfiguration(r.templatesConfig); err != nil {
+
+		err = r.writeConfiguration(r.templatesConfig)
+		if err != nil {
 			return err
 		}
 
 		gologger.Infof("Successfully downloaded nuclei-templates (v%s). Enjoy!\n", version.String())
+
+		return nil
+	}
+
+	// Check if last checked is more than 24 hours.
+	// If not, return since we don't want to do anything now.
+	if time.Since(r.templatesConfig.LastChecked) < 24*time.Hour && !r.options.UpdateTemplates {
 		return nil
 	}
 
 	// Get the configuration currently on disk.
 	verText := r.templatesConfig.CurrentVersion
 	indices := reVersion.FindStringIndex(verText)
+
 	if indices == nil {
 		return fmt.Errorf("invalid release found with tag %s", err)
 	}
+
 	if indices[0] > 0 {
 		verText = verText[indices[0]:]
 	}
+
 	oldVersion, err := semver.Make(verText)
+
 	if err != nil {
 		return err
 	}
@@ -147,39 +116,41 @@ func (r *Runner) updateTemplates() error {
 	}
 
 	if version.EQ(oldVersion) {
-		gologger.Labelf("Latest version of nuclei-templates installed: v%s\n", oldVersion.String())
-		return nil
+		gologger.Infof("Your nuclei-templates are up to date: v%s\n", oldVersion.String())
+		return r.writeConfiguration(r.templatesConfig)
 	}
+
 	if version.GT(oldVersion) {
 		if !r.options.UpdateTemplates {
-			gologger.Labelf("You're using outdated nuclei-templates. Latest v%s\n", version.String())
-			return nil
+			gologger.Labelf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
+			return r.writeConfiguration(r.templatesConfig)
 		}
+
 		if r.options.TemplatesDirectory != "" {
 			home = r.options.TemplatesDirectory
 			r.templatesConfig.TemplatesDirectory = path.Join(home, "nuclei-templates")
 		}
+
 		r.templatesConfig.CurrentVersion = version.String()
-		os.RemoveAll(r.templatesConfig.TemplatesDirectory)
 
 		gologger.Verbosef("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory)
 
-		if err = r.downloadReleaseAndUnzip(asset.GetZipballURL()); err != nil {
+		err = r.downloadReleaseAndUnzip(ctx, asset.GetZipballURL())
+		if err != nil {
 			return err
 		}
-		if err = r.writeConfiguration(r.templatesConfig); err != nil {
+
+		err = r.writeConfiguration(r.templatesConfig)
+		if err != nil {
 			return err
 		}
+
 		gologger.Infof("Successfully updated nuclei-templates (v%s). Enjoy!\n", version.String())
 	}
+
 	return nil
 }
 
-const (
-	userName = "projectdiscovery"
-	repoName = "nuclei-templates"
-)
-
 // getLatestReleaseFromGithub returns the latest release from github
 func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.RepositoryRelease, error) {
 	client := github.NewClient(nil)
@@ -191,16 +162,21 @@ func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.Repositor
 
 	// Find the most recent version based on semantic versioning.
 	var latestRelease semver.Version
+
 	var latestPublish *github.RepositoryRelease
+
 	for _, release := range rels {
 		verText := release.GetTagName()
 		indices := reVersion.FindStringIndex(verText)
+
 		if indices == nil {
 			return semver.Version{}, nil, fmt.Errorf("invalid release found with tag %s", err)
 		}
+
 		if indices[0] > 0 {
 			verText = verText[indices[0]:]
 		}
+
 		ver, err := semver.Make(verText)
 		if err != nil {
 			return semver.Version{}, nil, err
@@ -211,97 +187,85 @@ func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.Repositor
 			latestPublish = release
 		}
 	}
+
 	if latestPublish == nil {
 		return semver.Version{}, nil, errors.New("no version found for the templates")
 	}
+
 	return latestRelease, latestPublish, nil
 }
 
 // downloadReleaseAndUnzip downloads and unzips the release in a directory
-func (r *Runner) downloadReleaseAndUnzip(downloadURL string) error {
-	req, err := http.NewRequest("GET", downloadURL, nil)
+func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
-		return fmt.Errorf("Failed to create HTTP request to %s: %s", downloadURL, err)
+		return fmt.Errorf("failed to create HTTP request to %s: %s", downloadURL, err)
 	}
 
 	res, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return fmt.Errorf("Failed to download a release file from %s: %s", downloadURL, err)
+		return fmt.Errorf("failed to download a release file from %s: %s", downloadURL, err)
 	}
 	defer res.Body.Close()
 
-	if res.StatusCode != 200 {
-		return fmt.Errorf("Failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode)
+	if res.StatusCode != http.StatusOK {
+		return fmt.Errorf("failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode)
 	}
 
 	buf, err := ioutil.ReadAll(res.Body)
 	if err != nil {
-		return fmt.Errorf("Failed to create buffer for zip file: %s", err)
+		return fmt.Errorf("failed to create buffer for zip file: %s", err)
 	}
 
 	reader := bytes.NewReader(buf)
 	z, err := zip.NewReader(reader, reader.Size())
+
 	if err != nil {
-		return fmt.Errorf("Failed to uncompress zip file: %s", err)
+		return fmt.Errorf("failed to uncompress zip file: %s", err)
 	}
 
 	// Create the template folder if it doesn't exists
-	os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm)
+	err = os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm)
+	if err != nil {
+		return fmt.Errorf("failed to create template base folder: %s", err)
+	}
 
 	for _, file := range z.File {
 		directory, name := filepath.Split(file.Name)
 		if name == "" {
 			continue
 		}
+
 		paths := strings.Split(directory, "/")
 		finalPath := strings.Join(paths[1:], "/")
 
 		templateDirectory := path.Join(r.templatesConfig.TemplatesDirectory, finalPath)
-		os.MkdirAll(templateDirectory, os.ModePerm)
+		err = os.MkdirAll(templateDirectory, os.ModePerm)
 
-		f, err := os.Create(path.Join(templateDirectory, name))
 		if err != nil {
-			return fmt.Errorf("Could not create uncompressed file: %s", err)
+			return fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err)
+		}
+
+		f, err := os.OpenFile(path.Join(templateDirectory, name), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777)
+		if err != nil {
+			f.Close()
+			return fmt.Errorf("could not create uncompressed file: %s", err)
 		}
-		defer f.Close()
 
 		reader, err := file.Open()
 		if err != nil {
-			return fmt.Errorf("Could not open archive to extract file: %s", err)
+			f.Close()
+			return fmt.Errorf("could not open archive to extract file: %s", err)
 		}
-		io.Copy(f, reader)
+
+		_, err = io.Copy(f, reader)
+		if err != nil {
+			f.Close()
+			return fmt.Errorf("could not write template file: %s", err)
+		}
+
+		f.Close()
 	}
+
 	return nil
 }
-
-// isRelative checks if a given path is a relative path
-func (r *Runner) isRelative(path string) bool {
-	if strings.HasPrefix(path, "/") || strings.Contains(path, ":\\") {
-		return false
-	}
-	return true
-}
-
-// resolvePath gets the absolute path to the template by either
-// looking in the current directory or checking the nuclei templates directory.
-//
-// Current directory is given preference over the nuclei-templates directory.
-func (r *Runner) resolvePath(templateName string) (string, error) {
-	curDirectory, err := os.Getwd()
-	if err != nil {
-		return "", err
-	}
-	templatePath := path.Join(curDirectory, templateName)
-	if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
-		gologger.Infof("Found template in current directory: %s\n", templatePath)
-		return templatePath, nil
-	}
-	if r.templatesConfig != nil {
-		templatePath := path.Join(r.templatesConfig.TemplatesDirectory, templateName)
-		if _, err := os.Stat(templatePath); !os.IsNotExist(err) {
-			gologger.Infof("Found template in nuclei-templates directory: %s\n", templatePath)
-			return templatePath, nil
-		}
-	}
-	return "", fmt.Errorf("no such path found: %s", templateName)
-}

v2/internal/runner/validate.go

@@ -1,57 +0,0 @@
-package runner
-
-import (
-	"errors"
-	"net/url"
-
-	"github.com/projectdiscovery/gologger"
-)
-
-// validateOptions validates the configuration options passed
-func (options *Options) validateOptions() error {
-	// Both verbose and silent flags were used
-	if options.Verbose && options.Silent {
-		return errors.New("both verbose and silent mode specified")
-	}
-
-	// Check if a list of templates was provided and it exists
-	if options.Templates == "" && !options.UpdateTemplates {
-		return errors.New("no template/templates provided")
-	}
-
-	if options.Targets == "" && !options.Stdin && options.Target == "" && !options.UpdateTemplates {
-		return errors.New("no target input provided")
-	}
-
-	// Validate proxy options if provided
-	if options.ProxyURL != "" && !isValidProxyURL(options.ProxyURL) {
-		return errors.New("invalid http proxy format (It should be http://username:password@host:port)")
-	}
-	if options.ProxySocksURL != "" && !isValidProxyURL(options.ProxySocksURL) {
-		return errors.New("invalid socks proxy format (It should be socks5://username:password@host:port)")
-	}
-
-	return nil
-}
-
-func isValidProxyURL(URL string) bool {
-	if _, err := url.Parse(URL); err != nil {
-		return false
-	}
-
-	return true
-}
-
-// configureOutput configures the output on the screen
-func (options *Options) configureOutput() {
-	// If the user desires verbose output, show verbose output
-	if options.Verbose {
-		gologger.MaxLevel = gologger.Verbose
-	}
-	if options.NoColor {
-		gologger.UseColors = false
-	}
-	if options.Silent {
-		gologger.MaxLevel = gologger.Silent
-	}
-}

v2/internal/tracelog/tracelog.go

@@ -0,0 +1,76 @@
+package tracelog
+
+import (
+	"os"
+	"sync"
+
+	jsoniter "github.com/json-iterator/go"
+)
+
+// Log is an interface for logging trace log of all the requests
+type Log interface {
+	// Close closes the log interface flushing data
+	Close()
+	// Request writes a log the requests trace log
+	Request(templateID, url, requestType string, err error)
+}
+
+// NoopLogger is a noop logger that simply does nothing
+type NoopLogger struct{}
+
+// Close closes the log interface flushing data
+func (n *NoopLogger) Close() {}
+
+// Request writes a log the requests trace log
+func (n *NoopLogger) Request(templateID, url, requestType string, err error) {}
+
+// FileLogger is a trace logger that writes request logs to a file.
+type FileLogger struct {
+	encoder *jsoniter.Encoder
+	file    *os.File
+	mutex   *sync.Mutex
+}
+
+// NewFileLogger creates a new file logger structure
+func NewFileLogger(path string) (*FileLogger, error) {
+	file, err := os.Create(path)
+	if err != nil {
+		return nil, err
+	}
+	return &FileLogger{file: file, encoder: jsoniter.NewEncoder(file), mutex: &sync.Mutex{}}, nil
+}
+
+// Close closes the log interface flushing data
+func (f *FileLogger) Close() {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+
+	f.file.Close()
+}
+
+// JSONRequest is a trace log request written to file
+type JSONRequest struct {
+	ID    string `json:"id"`
+	URL   string `json:"url"`
+	Error string `json:"error"`
+	Type  string `json:"type"`
+}
+
+// Request writes a log the requests trace log
+func (f *FileLogger) Request(templateID, url, requestType string, err error) {
+	request := &JSONRequest{
+		ID:   templateID,
+		URL:  url,
+		Type: requestType,
+	}
+	if err != nil {
+		request.Error = err.Error()
+	} else {
+		request.Error = "none"
+	}
+
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+	//nolint:errcheck // We don't need to do anything here
+	f.encoder.Encode(request)
+}

v2/pkg/atomicboolean/bool.go

@@ -0,0 +1,42 @@
+package atomicboolean
+
+import (
+	"sync"
+)
+
+type AtomBool struct {
+	sync.RWMutex
+	flag bool
+}
+
+func New() *AtomBool {
+	return &AtomBool{}
+}
+
+func (b *AtomBool) Or(value bool) {
+	b.Lock()
+	defer b.Unlock()
+
+	b.flag = b.flag || value
+}
+
+func (b *AtomBool) And(value bool) {
+	b.Lock()
+	defer b.Unlock()
+
+	b.flag = b.flag && value
+}
+
+func (b *AtomBool) Set(value bool) {
+	b.Lock()
+	defer b.Unlock()
+
+	b.flag = value
+}
+
+func (b *AtomBool) Get() bool {
+	b.RLock()
+	defer b.RUnlock() //nolint
+
+	return b.flag
+}

v2/pkg/collaborator/collaborator.go

@@ -0,0 +1,59 @@
+package collaborator
+
+import (
+	"strings"
+	"time"
+
+	"github.com/projectdiscovery/collaborator"
+)
+
+const (
+	PollSeconds           = 5
+	DefaultMaxBufferLimit = 150
+)
+
+var DefaultPollInterval time.Duration = time.Second * time.Duration(PollSeconds)
+
+var DefaultCollaborator BurpCollaborator = BurpCollaborator{Collab: collaborator.NewBurpCollaborator()}
+
+type BurpCollaborator struct {
+	options *Options // unused
+	Collab  *collaborator.BurpCollaborator
+}
+
+type Options struct {
+	BIID           string
+	PollInterval   time.Duration
+	MaxBufferLimit int
+}
+
+func New(options *Options) *BurpCollaborator {
+	collab := collaborator.NewBurpCollaborator()
+	collab.AddBIID(options.BIID)
+	collab.MaxBufferLimit = options.MaxBufferLimit
+	return &BurpCollaborator{Collab: collab, options: options}
+}
+
+func (b *BurpCollaborator) Poll() {
+	// if no valid biids were provided just return
+	if len(b.Collab.BIIDs) > 0 {
+		go b.Collab.PollEach(DefaultPollInterval)
+	}
+}
+
+func (b *BurpCollaborator) Has(s string) bool {
+	for _, r := range b.Collab.RespBuffer {
+		for i := 0; i < len(r.Responses); i++ {
+			// search in dns
+			if strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) {
+				return true
+			}
+			// search in http
+			if strings.Contains(r.Responses[i].Data.RequestDecoded, s) {
+				return true
+			}
+		}
+	}
+
+	return false
+}

v2/pkg/colorizer/colorizer.go

@@ -0,0 +1,42 @@
+package colorizer
+
+import (
+	"strings"
+
+	"github.com/logrusorgru/aurora"
+)
+
+const (
+	fgOrange  uint8  = 208
+	undefined string = "undefined"
+)
+
+// NucleiColorizer contains the severity color mapping
+type NucleiColorizer struct {
+	Colorizer   aurora.Aurora
+	SeverityMap map[string]string
+}
+
+// NewNucleiColorizer initializes the new nuclei colorizer
+func NewNucleiColorizer(colorizer aurora.Aurora) *NucleiColorizer {
+	return &NucleiColorizer{
+		Colorizer: colorizer,
+		SeverityMap: map[string]string{
+			"info":     colorizer.Blue("info").String(),
+			"low":      colorizer.Green("low").String(),
+			"medium":   colorizer.Yellow("medium").String(),
+			"high":     colorizer.Index(fgOrange, "high").String(),
+			"critical": colorizer.Red("critical").String(),
+		},
+	}
+}
+
+// GetColorizedSeverity returns the colorized severity string
+func (r *NucleiColorizer) GetColorizedSeverity(severity string) string {
+	sev := r.SeverityMap[strings.ToLower(severity)]
+	if sev == "" {
+		return undefined
+	}
+
+	return sev
+}

v2/pkg/executer/executer_dns.go

@@ -0,0 +1,177 @@
+package executer
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
+	"github.com/projectdiscovery/nuclei/v2/internal/progress"
+	"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	retryabledns "github.com/projectdiscovery/retryabledns"
+	"go.uber.org/ratelimit"
+)
+
+// DNSExecuter is a client for performing a DNS request
+// for a template.
+type DNSExecuter struct {
+	// hm            *hybrid.HybridMap // Unused
+	coloredOutput bool
+	debug         bool
+	jsonOutput    bool
+	jsonRequest   bool
+	noMeta        bool
+	Results       bool
+	traceLog      tracelog.Log
+	dnsClient     *retryabledns.Client
+	template      *templates.Template
+	dnsRequest    *requests.DNSRequest
+	writer        *bufwriter.Writer
+	ratelimiter   ratelimit.Limiter
+
+	colorizer   colorizer.NucleiColorizer
+	decolorizer *regexp.Regexp
+}
+
+// DefaultResolvers contains the list of resolvers known to be trusted.
+var DefaultResolvers = []string{
+	"1.1.1.1:53", // Cloudflare
+	"1.0.0.1:53", // Cloudflare
+	"8.8.8.8:53", // Google
+	"8.8.4.4:53", // Google
+}
+
+// DNSOptions contains configuration options for the DNS executer.
+type DNSOptions struct {
+	ColoredOutput bool
+	Debug         bool
+	JSON          bool
+	JSONRequests  bool
+	NoMeta        bool
+	TraceLog      tracelog.Log
+	Template      *templates.Template
+	DNSRequest    *requests.DNSRequest
+	Writer        *bufwriter.Writer
+
+	Colorizer   colorizer.NucleiColorizer
+	Decolorizer *regexp.Regexp
+	RateLimiter ratelimit.Limiter
+}
+
+// NewDNSExecuter creates a new DNS executer from a template
+// and a DNS request query.
+func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
+	dnsClient := retryabledns.New(DefaultResolvers, options.DNSRequest.Retries)
+
+	executer := &DNSExecuter{
+		debug:         options.Debug,
+		noMeta:        options.NoMeta,
+		jsonOutput:    options.JSON,
+		traceLog:      options.TraceLog,
+		jsonRequest:   options.JSONRequests,
+		dnsClient:     dnsClient,
+		template:      options.Template,
+		dnsRequest:    options.DNSRequest,
+		writer:        options.Writer,
+		coloredOutput: options.ColoredOutput,
+		colorizer:     options.Colorizer,
+		decolorizer:   options.Decolorizer,
+		ratelimiter:   options.RateLimiter,
+	}
+
+	return executer
+}
+
+// ExecuteDNS executes the DNS request on a URL
+func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, reqURL string) *Result {
+	result := &Result{}
+
+	// Parse the URL and return domain if URL.
+	var domain string
+	if isURL(reqURL) {
+		domain = extractDomain(reqURL)
+	} else {
+		domain = reqURL
+	}
+
+	// Compile each request for the template based on the URL
+	compiledRequest, err := e.dnsRequest.MakeDNSRequest(domain)
+	if err != nil {
+		e.traceLog.Request(e.template.ID, domain, "dns", err)
+		result.Error = errors.Wrap(err, "could not make dns request")
+		p.Drop(1)
+		return result
+	}
+	e.traceLog.Request(e.template.ID, domain, "dns", nil)
+
+	if e.debug {
+		gologger.Infof("Dumped DNS request for %s (%s)\n\n", reqURL, e.template.ID)
+		fmt.Fprintf(os.Stderr, "%s\n", compiledRequest.String())
+	}
+
+	// Send the request to the target servers
+	resp, err := e.dnsClient.Do(compiledRequest)
+	if err != nil {
+		result.Error = errors.Wrap(err, "could not send dns request")
+		p.Drop(1)
+		return result
+	}
+	p.Update()
+
+	gologger.Verbosef("Sent for [%s] to %s\n", "dns-request", e.template.ID, reqURL)
+
+	if e.debug {
+		gologger.Infof("Dumped DNS response for %s (%s)\n\n", reqURL, e.template.ID)
+		fmt.Fprintf(os.Stderr, "%s\n", resp.String())
+	}
+
+	matcherCondition := e.dnsRequest.GetMatchersCondition()
+
+	for _, matcher := range e.dnsRequest.Matchers {
+		// Check if the matcher matched
+		if !matcher.MatchDNS(resp) {
+			// If the condition is AND we haven't matched, return.
+			if matcherCondition == matchers.ANDCondition {
+				return result
+			}
+		} else {
+			// If the matcher has matched, and its an OR
+			// write the first output then move to next matcher.
+			if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 {
+				e.writeOutputDNS(domain, compiledRequest, resp, matcher, nil)
+				result.GotResults = true
+			}
+		}
+	}
+
+	// All matchers have successfully completed so now start with the
+	// next task which is extraction of input from matchers.
+	var extractorResults []string
+
+	for _, extractor := range e.dnsRequest.Extractors {
+		for match := range extractor.ExtractDNS(resp) {
+			if !extractor.Internal {
+				extractorResults = append(extractorResults, match)
+			}
+		}
+	}
+
+	// Write a final string of output if matcher type is
+	// AND or if we have extractors for the mechanism too.
+	if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
+		e.writeOutputDNS(domain, compiledRequest, resp, nil, extractorResults)
+
+		result.GotResults = true
+	}
+
+	return result
+}
+
+// Close closes the dns executer for a template.
+func (e *DNSExecuter) Close() {}

v2/pkg/executer/executer_http.go

@@ -0,0 +1,710 @@
+package executer
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/cookiejar"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
+	"github.com/projectdiscovery/nuclei/v2/internal/progress"
+	"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
+	projetctfile "github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
+	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/rawhttp"
+	"github.com/projectdiscovery/retryablehttp-go"
+	"github.com/remeh/sizedwaitgroup"
+	"go.uber.org/ratelimit"
+	"golang.org/x/net/proxy"
+)
+
+const (
+	two                   = 2
+	ten                   = 10
+	defaultMaxWorkers     = 150
+	defaultMaxHistorydata = 150
+)
+
+// HTTPExecuter is client for performing HTTP requests
+// for a template.
+type HTTPExecuter struct {
+	pf               *projetctfile.ProjectFile
+	customHeaders    requests.CustomHeaders
+	colorizer        colorizer.NucleiColorizer
+	httpClient       *retryablehttp.Client
+	rawHTTPClient    *rawhttp.Client
+	template         *templates.Template
+	bulkHTTPRequest  *requests.BulkHTTPRequest
+	writer           *bufwriter.Writer
+	CookieJar        *cookiejar.Jar
+	traceLog         tracelog.Log
+	decolorizer      *regexp.Regexp
+	coloredOutput    bool
+	debug            bool
+	Results          bool
+	jsonOutput       bool
+	jsonRequest      bool
+	noMeta           bool
+	stopAtFirstMatch bool
+	ratelimiter      ratelimit.Limiter
+}
+
+// HTTPOptions contains configuration options for the HTTP executer.
+type HTTPOptions struct {
+	CustomHeaders    requests.CustomHeaders
+	ProxyURL         string
+	ProxySocksURL    string
+	Template         *templates.Template
+	BulkHTTPRequest  *requests.BulkHTTPRequest
+	Writer           *bufwriter.Writer
+	Timeout          int
+	Retries          int
+	CookieJar        *cookiejar.Jar
+	Colorizer        *colorizer.NucleiColorizer
+	Decolorizer      *regexp.Regexp
+	TraceLog         tracelog.Log
+	Debug            bool
+	JSON             bool
+	JSONRequests     bool
+	NoMeta           bool
+	CookieReuse      bool
+	ColoredOutput    bool
+	StopAtFirstMatch bool
+	PF               *projetctfile.ProjectFile
+	RateLimiter      ratelimit.Limiter
+	Dialer           *fastdialer.Dialer
+}
+
+// NewHTTPExecuter creates a new HTTP executer from a template
+// and a HTTP request query.
+func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
+	var (
+		proxyURL *url.URL
+		err      error
+	)
+
+	if options.ProxyURL != "" {
+		proxyURL, err = url.Parse(options.ProxyURL)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the HTTP Client
+	client := makeHTTPClient(proxyURL, options)
+	// nolint:bodyclose // false positive there is no body to close yet
+	client.CheckRetry = retryablehttp.HostSprayRetryPolicy()
+
+	if options.CookieJar != nil {
+		client.HTTPClient.Jar = options.CookieJar
+	} else if options.CookieReuse {
+		jar, err := cookiejar.New(nil)
+		if err != nil {
+			return nil, err
+		}
+		client.HTTPClient.Jar = jar
+	}
+
+	// initiate raw http client
+	rawClient := rawhttp.NewClient(rawhttp.DefaultOptions)
+
+	executer := &HTTPExecuter{
+		debug:            options.Debug,
+		jsonOutput:       options.JSON,
+		jsonRequest:      options.JSONRequests,
+		noMeta:           options.NoMeta,
+		httpClient:       client,
+		rawHTTPClient:    rawClient,
+		traceLog:         options.TraceLog,
+		template:         options.Template,
+		bulkHTTPRequest:  options.BulkHTTPRequest,
+		writer:           options.Writer,
+		customHeaders:    options.CustomHeaders,
+		CookieJar:        options.CookieJar,
+		coloredOutput:    options.ColoredOutput,
+		colorizer:        *options.Colorizer,
+		decolorizer:      options.Decolorizer,
+		stopAtFirstMatch: options.StopAtFirstMatch,
+		pf:               options.PF,
+		ratelimiter:      options.RateLimiter,
+	}
+
+	return executer, nil
+}
+
+func (e *HTTPExecuter) ExecuteRaceRequest(reqURL string) *Result {
+	result := &Result{
+		Matches:     make(map[string]interface{}),
+		Extractions: make(map[string]interface{}),
+	}
+
+	dynamicvalues := make(map[string]interface{})
+
+	// verify if the URL is already being processed
+	if e.bulkHTTPRequest.HasGenerator(reqURL) {
+		return result
+	}
+
+	e.bulkHTTPRequest.CreateGenerator(reqURL)
+
+	// Workers that keeps enqueuing new requests
+	maxWorkers := e.bulkHTTPRequest.RaceNumberRequests
+	swg := sizedwaitgroup.New(maxWorkers)
+	for i := 0; i < e.bulkHTTPRequest.RaceNumberRequests; i++ {
+		swg.Add()
+		// base request
+		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
+		if err != nil {
+			result.Error = err
+			return result
+		}
+		go func(httpRequest *requests.HTTPRequest) {
+			defer swg.Done()
+
+			// If the request was built correctly then execute it
+			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
+			if err != nil {
+				result.Error = errors.Wrap(err, "could not handle http request")
+			}
+		}(request)
+	}
+
+	swg.Wait()
+
+	return result
+}
+
+func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string) *Result {
+	result := &Result{
+		Matches:     make(map[string]interface{}),
+		Extractions: make(map[string]interface{}),
+	}
+
+	dynamicvalues := make(map[string]interface{})
+
+	// verify if the URL is already being processed
+	if e.bulkHTTPRequest.HasGenerator(reqURL) {
+		return result
+	}
+
+	remaining := e.bulkHTTPRequest.GetRequestCount()
+	e.bulkHTTPRequest.CreateGenerator(reqURL)
+
+	// Workers that keeps enqueuing new requests
+	maxWorkers := e.bulkHTTPRequest.Threads
+	swg := sizedwaitgroup.New(maxWorkers)
+	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
+		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
+		if err != nil {
+			result.Error = err
+			p.Drop(remaining)
+		} else {
+			swg.Add()
+			go func(httpRequest *requests.HTTPRequest) {
+				defer swg.Done()
+
+				e.ratelimiter.Take()
+
+				// If the request was built correctly then execute it
+				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
+				if err != nil {
+					e.traceLog.Request(e.template.ID, reqURL, "http", err)
+					result.Error = errors.Wrap(err, "could not handle http request")
+					p.Drop(remaining)
+				} else {
+					e.traceLog.Request(e.template.ID, reqURL, "http", nil)
+				}
+			}(request)
+		}
+		p.Update()
+		e.bulkHTTPRequest.Increment(reqURL)
+	}
+	swg.Wait()
+
+	return result
+}
+
+func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result {
+	result := &Result{
+		Matches:     make(map[string]interface{}),
+		Extractions: make(map[string]interface{}),
+	}
+
+	dynamicvalues := make(map[string]interface{})
+
+	// verify if the URL is already being processed
+	if e.bulkHTTPRequest.HasGenerator(reqURL) {
+		return result
+	}
+
+	e.bulkHTTPRequest.CreateGenerator(reqURL)
+
+	// need to extract the target from the url
+	URL, err := url.Parse(reqURL)
+	if err != nil {
+		return result
+	}
+
+	pipeOptions := rawhttp.DefaultPipelineOptions
+	pipeOptions.Host = URL.Host
+	pipeOptions.MaxConnections = 1
+	if e.bulkHTTPRequest.PipelineConcurrentConnections > 0 {
+		pipeOptions.MaxConnections = e.bulkHTTPRequest.PipelineConcurrentConnections
+	}
+	if e.bulkHTTPRequest.PipelineRequestsPerConnection > 0 {
+		pipeOptions.MaxPendingRequests = e.bulkHTTPRequest.PipelineRequestsPerConnection
+	}
+	pipeclient := rawhttp.NewPipelineClient(pipeOptions)
+
+	// defaultMaxWorkers should be a sufficient value to keep queues always full
+	maxWorkers := defaultMaxWorkers
+	// in case the queue is bigger increase the workers
+	if pipeOptions.MaxPendingRequests > maxWorkers {
+		maxWorkers = pipeOptions.MaxPendingRequests
+	}
+	swg := sizedwaitgroup.New(maxWorkers)
+	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
+		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
+		if err != nil {
+			result.Error = err
+		} else {
+			swg.Add()
+			go func(httpRequest *requests.HTTPRequest) {
+				defer swg.Done()
+
+				// HTTP pipelining ignores rate limit
+				// If the request was built correctly then execute it
+				request.Pipeline = true
+				request.PipelineClient = pipeclient
+				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
+				if err != nil {
+					e.traceLog.Request(e.template.ID, reqURL, "http", err)
+					result.Error = errors.Wrap(err, "could not handle http request")
+				} else {
+					e.traceLog.Request(e.template.ID, reqURL, "http", nil)
+				}
+				request.PipelineClient = nil
+			}(request)
+		}
+
+		e.bulkHTTPRequest.Increment(reqURL)
+	}
+	swg.Wait()
+	return result
+}
+
+// ExecuteHTTP executes the HTTP request on a URL
+func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result {
+	// verify if pipeline was requested
+	if e.bulkHTTPRequest.Pipeline {
+		return e.ExecuteTurboHTTP(reqURL)
+	}
+
+	// verify if a basic race condition was requested
+	if e.bulkHTTPRequest.Race && e.bulkHTTPRequest.RaceNumberRequests > 0 {
+		return e.ExecuteRaceRequest(reqURL)
+	}
+
+	// verify if parallel elaboration was requested
+	if e.bulkHTTPRequest.Threads > 0 {
+		return e.ExecuteParallelHTTP(p, reqURL)
+	}
+
+	var requestNumber int
+
+	result := &Result{
+		Matches:     make(map[string]interface{}),
+		Extractions: make(map[string]interface{}),
+		historyData: make(map[string]interface{}),
+	}
+
+	dynamicvalues := make(map[string]interface{})
+
+	// verify if the URL is already being processed
+	if e.bulkHTTPRequest.HasGenerator(reqURL) {
+		return result
+	}
+
+	remaining := e.bulkHTTPRequest.GetRequestCount()
+	e.bulkHTTPRequest.CreateGenerator(reqURL)
+
+	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
+		requestNumber++
+		httpRequest, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
+		if err != nil {
+			result.Error = err
+			p.Drop(remaining)
+		} else {
+			e.ratelimiter.Take()
+			// If the request was built correctly then execute it
+			format := "%s_" + strconv.Itoa(requestNumber)
+			err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, format)
+			if err != nil {
+				result.Error = errors.Wrap(err, "could not handle http request")
+				p.Drop(remaining)
+				e.traceLog.Request(e.template.ID, reqURL, "http", err)
+			} else {
+				e.traceLog.Request(e.template.ID, reqURL, "http", nil)
+			}
+		}
+		p.Update()
+
+		// Check if has to stop processing at first valid result
+		if e.stopAtFirstMatch && result.GotResults {
+			p.Drop(remaining)
+			break
+		}
+
+		// move always forward with requests
+		e.bulkHTTPRequest.Increment(reqURL)
+		remaining--
+	}
+	gologger.Verbosef("Sent for [%s] to %s\n", "http-request", e.template.ID, reqURL)
+	return result
+}
+
+func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, format string) error {
+	e.setCustomHeaders(request)
+
+	var (
+		resp          *http.Response
+		err           error
+		dumpedRequest []byte
+		fromcache     bool
+	)
+
+	if e.debug || e.pf != nil {
+		dumpedRequest, err = requests.Dump(request, reqURL)
+		if err != nil {
+			return err
+		}
+	}
+
+	if e.debug {
+		gologger.Infof("Dumped HTTP request for %s (%s)\n\n", reqURL, e.template.ID)
+		fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
+	}
+
+	timeStart := time.Now()
+
+	if request.Pipeline {
+		resp, err = request.PipelineClient.DoRaw(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)))
+		if err != nil {
+			if resp != nil {
+				resp.Body.Close()
+			}
+			e.traceLog.Request(e.template.ID, reqURL, "http", err)
+			return err
+		}
+		e.traceLog.Request(e.template.ID, reqURL, "http", nil)
+	} else if request.Unsafe {
+		// rawhttp
+		// burp uses "\r\n" as new line character
+		request.RawRequest.Data = strings.ReplaceAll(request.RawRequest.Data, "\n", "\r\n")
+		options := e.rawHTTPClient.Options
+		options.AutomaticContentLength = request.AutomaticContentLengthHeader
+		options.AutomaticHostHeader = request.AutomaticHostHeader
+		options.FollowRedirects = request.FollowRedirects
+		resp, err = e.rawHTTPClient.DoRawWithOptions(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)), options)
+		if err != nil {
+			if resp != nil {
+				resp.Body.Close()
+			}
+			e.traceLog.Request(e.template.ID, reqURL, "http", err)
+			return err
+		}
+		e.traceLog.Request(e.template.ID, reqURL, "http", nil)
+	} else {
+		// if nuclei-project is available check if the request was already sent previously
+		if e.pf != nil {
+			// if unavailable fail silently
+			fromcache = true
+			// nolint:bodyclose // false positive the response is generated at runtime
+			resp, err = e.pf.Get(dumpedRequest)
+			if err != nil {
+				fromcache = false
+			}
+		}
+
+		// retryablehttp
+		if resp == nil {
+			resp, err = e.httpClient.Do(request.Request)
+			if err != nil {
+				if resp != nil {
+					resp.Body.Close()
+				}
+				e.traceLog.Request(e.template.ID, reqURL, "http", err)
+				return err
+			}
+			e.traceLog.Request(e.template.ID, reqURL, "http", nil)
+		}
+	}
+
+	duration := time.Since(timeStart)
+
+	// Dump response - Step 1 - Decompression not yet handled
+	var dumpedResponse []byte
+	if e.debug {
+		var dumpErr error
+		dumpedResponse, dumpErr = httputil.DumpResponse(resp, true)
+		if dumpErr != nil {
+			return errors.Wrap(dumpErr, "could not dump http response")
+		}
+	}
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		_, copyErr := io.Copy(ioutil.Discard, resp.Body)
+		if copyErr != nil {
+			resp.Body.Close()
+			return copyErr
+		}
+
+		resp.Body.Close()
+
+		return errors.Wrap(err, "could not read http body")
+	}
+
+	resp.Body.Close()
+
+	// net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request
+	// so in case we have to manually do it
+	dataOrig := data
+	data, err = requests.HandleDecompression(request, data)
+	if err != nil {
+		return errors.Wrap(err, "could not decompress http body")
+	}
+
+	// Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation)
+	if e.debug {
+		dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data)
+		gologger.Infof("Dumped HTTP response for %s (%s)\n\n", reqURL, e.template.ID)
+		fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
+	}
+
+	// if nuclei-project is enabled store the response if not previously done
+	if e.pf != nil && !fromcache {
+		err := e.pf.Set(dumpedRequest, resp, data)
+		if err != nil {
+			return errors.Wrap(err, "could not store in project file")
+		}
+	}
+
+	// Convert response body from []byte to string with zero copy
+	body := unsafeToString(data)
+
+	headers := headersToString(resp.Header)
+
+	// store for internal purposes the DSL matcher data
+	// hardcode stopping storing data after defaultMaxHistorydata items
+	if len(result.historyData) < defaultMaxHistorydata {
+		result.Lock()
+		result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format))
+		result.Unlock()
+	}
+
+	matcherCondition := e.bulkHTTPRequest.GetMatchersCondition()
+	for _, matcher := range e.bulkHTTPRequest.Matchers {
+		// Check if the matcher matched
+		if !matcher.Match(resp, body, headers, duration, result.historyData) {
+			// If the condition is AND we haven't matched, try next request.
+			if matcherCondition == matchers.ANDCondition {
+				return nil
+			}
+		} else {
+			// If the matcher has matched, and its an OR
+			// write the first output then move to next matcher.
+			if matcherCondition == matchers.ORCondition {
+				result.Lock()
+				result.Matches[matcher.Name] = nil
+				// probably redundant but ensures we snapshot current payload values when matchers are valid
+				result.Meta = request.Meta
+				result.GotResults = true
+				result.Unlock()
+				e.writeOutputHTTP(request, resp, body, matcher, nil, result.Meta, reqURL)
+			}
+		}
+	}
+
+	// All matchers have successfully completed so now start with the
+	// next task which is extraction of input from matchers.
+	var extractorResults, outputExtractorResults []string
+
+	for _, extractor := range e.bulkHTTPRequest.Extractors {
+		for match := range extractor.Extract(resp, body, headers) {
+			if _, ok := dynamicvalues[extractor.Name]; !ok {
+				dynamicvalues[extractor.Name] = match
+			}
+
+			extractorResults = append(extractorResults, match)
+
+			if !extractor.Internal {
+				outputExtractorResults = append(outputExtractorResults, match)
+			}
+		}
+		// probably redundant but ensures we snapshot current payload values when extractors are valid
+		result.Lock()
+		result.Meta = request.Meta
+		result.Extractions[extractor.Name] = extractorResults
+		result.Unlock()
+	}
+
+	// Write a final string of output if matcher type is
+	// AND or if we have extractors for the mechanism too.
+	if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition {
+		e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, result.Meta, reqURL)
+		result.Lock()
+		result.GotResults = true
+		result.Unlock()
+	}
+
+	return nil
+}
+
+// Close closes the http executer for a template.
+func (e *HTTPExecuter) Close() {}
+
+// makeHTTPClient creates a http client
+func makeHTTPClient(proxyURL *url.URL, options *HTTPOptions) *retryablehttp.Client {
+	// Multiple Host
+	retryablehttpOptions := retryablehttp.DefaultOptionsSpraying
+	disableKeepAlives := true
+	maxIdleConns := 0
+	maxConnsPerHost := 0
+	maxIdleConnsPerHost := -1
+
+	if options.BulkHTTPRequest.Threads > 0 {
+		// Single host
+		retryablehttpOptions = retryablehttp.DefaultOptionsSingle
+		disableKeepAlives = false
+		maxIdleConnsPerHost = 500
+		maxConnsPerHost = 500
+	}
+
+	retryablehttpOptions.RetryWaitMax = 10 * time.Second
+	retryablehttpOptions.RetryMax = options.Retries
+	followRedirects := options.BulkHTTPRequest.Redirects
+	maxRedirects := options.BulkHTTPRequest.MaxRedirects
+
+	transport := &http.Transport{
+		DialContext:         options.Dialer.Dial,
+		MaxIdleConns:        maxIdleConns,
+		MaxIdleConnsPerHost: maxIdleConnsPerHost,
+		MaxConnsPerHost:     maxConnsPerHost,
+		TLSClientConfig: &tls.Config{
+			Renegotiation:      tls.RenegotiateOnceAsClient,
+			InsecureSkipVerify: true,
+		},
+		DisableKeepAlives: disableKeepAlives,
+	}
+
+	// Attempts to overwrite the dial function with the socks proxied version
+	if options.ProxySocksURL != "" {
+		var proxyAuth *proxy.Auth
+
+		socksURL, err := url.Parse(options.ProxySocksURL)
+
+		if err == nil {
+			proxyAuth = &proxy.Auth{}
+			proxyAuth.User = socksURL.User.Username()
+			proxyAuth.Password, _ = socksURL.User.Password()
+		}
+
+		dialer, err := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct)
+		dc := dialer.(interface {
+			DialContext(ctx context.Context, network, addr string) (net.Conn, error)
+		})
+
+		if err == nil {
+			transport.DialContext = dc.DialContext
+		}
+	}
+
+	if proxyURL != nil {
+		transport.Proxy = http.ProxyURL(proxyURL)
+	}
+
+	return retryablehttp.NewWithHTTPClient(&http.Client{
+		Transport:     transport,
+		Timeout:       time.Duration(options.Timeout) * time.Second,
+		CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects),
+	}, retryablehttpOptions)
+}
+
+type checkRedirectFunc func(_ *http.Request, requests []*http.Request) error
+
+func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirectFunc {
+	return func(_ *http.Request, requests []*http.Request) error {
+		if !followRedirects {
+			return http.ErrUseLastResponse
+		}
+
+		if maxRedirects == 0 {
+			if len(requests) > ten {
+				return http.ErrUseLastResponse
+			}
+
+			return nil
+		}
+
+		if len(requests) > maxRedirects {
+			return http.ErrUseLastResponse
+		}
+
+		return nil
+	}
+}
+
+func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) {
+	for _, customHeader := range e.customHeaders {
+		// This should be pre-computed somewhere and done only once
+		tokens := strings.SplitN(customHeader, ":", two)
+		// if it's an invalid header skip it
+		if len(tokens) < two {
+			continue
+		}
+
+		headerName, headerValue := tokens[0], strings.Join(tokens[1:], "")
+		if r.RawRequest != nil {
+			// rawhttp
+			r.RawRequest.Headers[headerName] = headerValue
+		} else {
+			// retryablehttp
+			headerName = strings.TrimSpace(headerName)
+			headerValue = strings.TrimSpace(headerValue)
+			r.Request.Header[headerName] = []string{headerValue}
+		}
+	}
+}
+
+type Result struct {
+	sync.Mutex
+	GotResults  bool
+	Done        bool
+	Meta        map[string]interface{}
+	Matches     map[string]interface{}
+	Extractions map[string]interface{}
+	historyData map[string]interface{}
+	Error       error
+}

v2/pkg/executer/output_dns.go

@@ -0,0 +1,106 @@
+package executer
+
+import (
+	"strings"
+
+	"github.com/miekg/dns"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
+)
+
+// writeOutputDNS writes dns output to streams
+// nolint:interfacer // dns.Msg is out of current scope
+func (e *DNSExecuter) writeOutputDNS(domain string, req, resp *dns.Msg, matcher *matchers.Matcher, extractorResults []string) {
+	if e.jsonOutput {
+		output := make(jsonOutput)
+		output["matched"] = domain
+
+		if !e.noMeta {
+			output["template"] = e.template.ID
+			output["type"] = "dns"
+			output["host"] = domain
+			for k, v := range e.template.Info {
+				output[k] = v
+			}
+			if matcher != nil && len(matcher.Name) > 0 {
+				output["matcher_name"] = matcher.Name
+			}
+			if len(extractorResults) > 0 {
+				output["extracted_results"] = extractorResults
+			}
+			if e.jsonRequest {
+				output["request"] = req.String()
+				output["response"] = resp.String()
+			}
+		}
+
+		data, err := jsoniter.Marshal(output)
+		if err != nil {
+			gologger.Warningf("Could not marshal json output: %s\n", err)
+		}
+		gologger.Silentf("%s", string(data))
+		if e.writer != nil {
+			if err := e.writer.Write(data); err != nil {
+				gologger.Errorf("Could not write output data: %s\n", err)
+				return
+			}
+		}
+		return
+	}
+
+	builder := &strings.Builder{}
+	colorizer := e.colorizer
+
+	if !e.noMeta {
+		builder.WriteRune('[')
+		builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String())
+
+		if matcher != nil && len(matcher.Name) > 0 {
+			builder.WriteString(":")
+			builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String())
+		}
+
+		builder.WriteString("] [")
+		builder.WriteString(colorizer.Colorizer.BrightBlue("dns").String())
+		builder.WriteString("] ")
+
+		if e.template.Info["severity"] != "" {
+			builder.WriteString("[")
+			builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"]))
+			builder.WriteString("] ")
+		}
+	}
+	builder.WriteString(domain)
+
+	// If any extractors, write the results
+	if len(extractorResults) > 0 && !e.noMeta {
+		builder.WriteString(" [")
+
+		for i, result := range extractorResults {
+			builder.WriteString(colorizer.Colorizer.BrightCyan(result).String())
+
+			if i != len(extractorResults)-1 {
+				builder.WriteRune(',')
+			}
+		}
+		builder.WriteString("]")
+	}
+	builder.WriteRune('\n')
+
+	// Write output to screen as well as any output file
+	message := builder.String()
+	gologger.Silentf("%s", message)
+
+	if e.writer != nil {
+		if e.coloredOutput {
+			message = e.decolorizer.ReplaceAllString(message, "")
+		}
+
+		if err := e.writer.WriteString(message); err != nil {
+			gologger.Errorf("Could not write output data: %s\n", err)
+			return
+		}
+	}
+}

v2/pkg/executer/output_http.go

@@ -0,0 +1,147 @@
+package executer
+
+import (
+	"net/http"
+	"net/http/httputil"
+	"strings"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
+)
+
+// writeOutputHTTP writes http output to streams
+func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Response, body string, matcher *matchers.Matcher, extractorResults []string, meta map[string]interface{}, reqURL string) {
+	var URL string
+	if req.RawRequest != nil {
+		URL = req.RawRequest.FullURL
+	}
+	if req.Request != nil {
+		URL = req.Request.URL.String()
+	}
+
+	if e.jsonOutput {
+		output := make(jsonOutput)
+
+		output["matched"] = URL
+		if !e.noMeta {
+			output["template"] = e.template.ID
+			output["type"] = "http"
+			output["host"] = reqURL
+			if len(meta) > 0 {
+				output["meta"] = meta
+			}
+			for k, v := range e.template.Info {
+				output[k] = v
+			}
+			if matcher != nil && len(matcher.Name) > 0 {
+				output["matcher_name"] = matcher.Name
+			}
+			if len(extractorResults) > 0 {
+				output["extracted_results"] = extractorResults
+			}
+
+			// TODO: URL should be an argument
+			if e.jsonRequest {
+				dumpedRequest, err := requests.Dump(req, URL)
+				if err != nil {
+					gologger.Warningf("could not dump request: %s\n", err)
+				} else {
+					output["request"] = string(dumpedRequest)
+				}
+
+				dumpedResponse, err := httputil.DumpResponse(resp, false)
+				if err != nil {
+					gologger.Warningf("could not dump response: %s\n", err)
+				} else {
+					output["response"] = string(dumpedResponse) + body
+				}
+			}
+		}
+
+		data, err := jsoniter.Marshal(output)
+		if err != nil {
+			gologger.Warningf("Could not marshal json output: %s\n", err)
+		}
+		gologger.Silentf("%s", string(data))
+
+		if e.writer != nil {
+			if err := e.writer.Write(data); err != nil {
+				gologger.Errorf("Could not write output data: %s\n", err)
+				return
+			}
+		}
+		return
+	}
+
+	builder := &strings.Builder{}
+	colorizer := e.colorizer
+
+	if !e.noMeta {
+		builder.WriteRune('[')
+		builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String())
+
+		if matcher != nil && len(matcher.Name) > 0 {
+			builder.WriteString(":")
+			builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String())
+		}
+
+		builder.WriteString("] [")
+		builder.WriteString(colorizer.Colorizer.BrightBlue("http").String())
+		builder.WriteString("] ")
+
+		if e.template.Info["severity"] != "" {
+			builder.WriteString("[")
+			builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"]))
+			builder.WriteString("] ")
+		}
+	}
+	builder.WriteString(URL)
+
+	// If any extractors, write the results
+	if len(extractorResults) > 0 && !e.noMeta {
+		builder.WriteString(" [")
+
+		for i, result := range extractorResults {
+			builder.WriteString(colorizer.Colorizer.BrightCyan(result).String())
+
+			if i != len(extractorResults)-1 {
+				builder.WriteRune(',')
+			}
+		}
+
+		builder.WriteString("]")
+	}
+
+	// write meta if any
+	if len(req.Meta) > 0 && !e.noMeta {
+		builder.WriteString(" [")
+
+		var metas []string
+
+		for name, value := range req.Meta {
+			metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(value.(string)).String())
+		}
+
+		builder.WriteString(strings.Join(metas, ","))
+		builder.WriteString("]")
+	}
+
+	builder.WriteRune('\n')
+
+	// Write output to screen as well as any output file
+	message := builder.String()
+	gologger.Silentf("%s", message)
+
+	if e.writer != nil {
+		if e.coloredOutput {
+			message = e.decolorizer.ReplaceAllString(message, "")
+		}
+
+		if err := e.writer.WriteString(message); err != nil {
+			gologger.Errorf("Could not write output data: %s\n", err)
+			return
+		}
+	}
+}

v2/pkg/executer/utils.go

@@ -0,0 +1,59 @@
+package executer
+
+import (
+	"net/http"
+	"net/url"
+	"strings"
+	"unsafe"
+)
+
+type jsonOutput map[string]interface{}
+
+// unsafeToString converts byte slice to string with zero allocations
+func unsafeToString(bs []byte) string {
+	return *(*string)(unsafe.Pointer(&bs))
+}
+
+// headersToString converts http headers to string
+func headersToString(headers http.Header) string {
+	builder := &strings.Builder{}
+
+	for header, values := range headers {
+		builder.WriteString(header)
+		builder.WriteString(": ")
+
+		for i, value := range values {
+			builder.WriteString(value)
+
+			if i != len(values)-1 {
+				builder.WriteRune('\n')
+				builder.WriteString(header)
+				builder.WriteString(": ")
+			}
+		}
+		builder.WriteRune('\n')
+	}
+	return builder.String()
+}
+
+// isURL tests a string to determine if it is a well-structured url or not.
+func isURL(toTest string) bool {
+	_, err := url.ParseRequestURI(toTest)
+	if err != nil {
+		return false
+	}
+	u, err := url.Parse(toTest)
+	if err != nil || u.Scheme == "" || u.Host == "" {
+		return false
+	}
+	return true
+}
+
+// extractDomain extracts the domain name of a URL
+func extractDomain(theURL string) string {
+	u, err := url.Parse(theURL)
+	if err != nil {
+		return ""
+	}
+	return u.Hostname()
+}

v2/pkg/executor/dns_utils.go

@@ -1,38 +0,0 @@
-package executor
-
-import (
-	"net/url"
-
-	"github.com/asaskevich/govalidator"
-)
-
-// isURL tests a string to determine if it is a well-structured url or not.
-func isURL(toTest string) bool {
-	_, err := url.ParseRequestURI(toTest)
-	if err != nil {
-		return false
-	}
-
-	u, err := url.Parse(toTest)
-	if err != nil || u.Scheme == "" || u.Host == "" {
-		return false
-	}
-	return true
-}
-
-// extractDomain extracts the domain name of a URL
-func extractDomain(URL string) string {
-	u, err := url.Parse(URL)
-	if err != nil {
-		return ""
-	}
-	hostname := u.Hostname()
-	return hostname
-}
-
-// isDNS tests a string to determine if it is a well-structured dns or not
-// even if it's oneliner, we leave it wrapped in a function call for
-// future improvements
-func isDNS(toTest string) bool {
-	return govalidator.IsDNSName(toTest)
-}

v2/pkg/executor/executer_http.go

@@ -1,282 +0,0 @@
-package executor
-
-import (
-	"bufio"
-	"crypto/tls"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/http/httputil"
-	"net/url"
-	"os"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/retryablehttp-go"
-	"golang.org/x/net/proxy"
-)
-
-// HTTPExecutor is client for performing HTTP requests
-// for a template.
-type HTTPExecutor struct {
-	debug         bool
-	results       uint32
-	jsonOutput    bool
-	httpClient    *retryablehttp.Client
-	template      *templates.Template
-	httpRequest   *requests.HTTPRequest
-	writer        *bufio.Writer
-	outputMutex   *sync.Mutex
-	customHeaders requests.CustomHeaders
-}
-
-// HTTPOptions contains configuration options for the HTTP executor.
-type HTTPOptions struct {
-	Template      *templates.Template
-	HTTPRequest   *requests.HTTPRequest
-	Writer        *bufio.Writer
-	Timeout       int
-	Retries       int
-	ProxyURL      string
-	ProxySocksURL string
-	Debug         bool
-	JSON          bool
-	CustomHeaders requests.CustomHeaders
-}
-
-// NewHTTPExecutor creates a new HTTP executor from a template
-// and a HTTP request query.
-func NewHTTPExecutor(options *HTTPOptions) (*HTTPExecutor, error) {
-	var proxyURL *url.URL
-	var err error
-
-	if options.ProxyURL != "" {
-		proxyURL, err = url.Parse(options.ProxyURL)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	// Create the HTTP Client
-	client := makeHTTPClient(proxyURL, options)
-	client.CheckRetry = retryablehttp.HostSprayRetryPolicy()
-
-	executer := &HTTPExecutor{
-		debug:         options.Debug,
-		jsonOutput:    options.JSON,
-		results:       0,
-		httpClient:    client,
-		template:      options.Template,
-		httpRequest:   options.HTTPRequest,
-		outputMutex:   &sync.Mutex{},
-		writer:        options.Writer,
-		customHeaders: options.CustomHeaders,
-	}
-	return executer, nil
-}
-
-// GotResults returns true if there were any results for the executor
-func (e *HTTPExecutor) GotResults() bool {
-	if atomic.LoadUint32(&e.results) == 0 {
-		return false
-	}
-	return true
-}
-
-// ExecuteHTTP executes the HTTP request on a URL
-func (e *HTTPExecutor) ExecuteHTTP(URL string) error {
-	// Compile each request for the template based on the URL
-	compiledRequest, err := e.httpRequest.MakeHTTPRequest(URL)
-	if err != nil {
-		return errors.Wrap(err, "could not make http request")
-	}
-
-	// Send the request to the target servers
-mainLoop:
-	for compiledRequest := range compiledRequest {
-		if compiledRequest.Error != nil {
-			return errors.Wrap(err, "could not make http request")
-		}
-		e.setCustomHeaders(compiledRequest)
-		req := compiledRequest.Request
-
-		if e.debug {
-			gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID)
-			dumpedRequest, err := httputil.DumpRequest(req.Request, true)
-			if err != nil {
-				return errors.Wrap(err, "could not dump http request")
-			}
-			fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
-		}
-
-		resp, err := e.httpClient.Do(req)
-		if err != nil {
-			if resp != nil {
-				resp.Body.Close()
-			}
-			gologger.Warningf("Could not do request: %s\n", err)
-			continue
-		}
-
-		if e.debug {
-			gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID)
-			dumpedResponse, err := httputil.DumpResponse(resp, true)
-			if err != nil {
-				return errors.Wrap(err, "could not dump http response")
-			}
-			fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
-		}
-
-		data, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			io.Copy(ioutil.Discard, resp.Body)
-			resp.Body.Close()
-			return errors.Wrap(err, "could not read http body")
-		}
-		resp.Body.Close()
-
-		// net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request
-		// so in case we have to manually do it
-		data, err = requests.HandleDecompression(compiledRequest.Request, data)
-		if err != nil {
-			return errors.Wrap(err, "could not decompress http body")
-		}
-
-		// Convert response body from []byte to string with zero copy
-		body := unsafeToString(data)
-
-		var headers string
-		matcherCondition := e.httpRequest.GetMatchersCondition()
-		for _, matcher := range e.httpRequest.Matchers {
-			headers = headersToString(resp.Header)
-			// Check if the matcher matched
-			if !matcher.Match(resp, body, headers) {
-				// If the condition is AND we haven't matched, try next request.
-				if matcherCondition == matchers.ANDCondition {
-					continue mainLoop
-				}
-			} else {
-				// If the matcher has matched, and its an OR
-				// write the first output then move to next matcher.
-				if matcherCondition == matchers.ORCondition && len(e.httpRequest.Extractors) == 0 {
-					e.writeOutputHTTP(compiledRequest, matcher, nil)
-					atomic.CompareAndSwapUint32(&e.results, 0, 1)
-				}
-			}
-		}
-
-		// All matchers have successfully completed so now start with the
-		// next task which is extraction of input from matchers.
-		var extractorResults []string
-		for _, extractor := range e.httpRequest.Extractors {
-			headers = headersToString(resp.Header)
-			for match := range extractor.Extract(body, headers) {
-				extractorResults = append(extractorResults, match)
-			}
-		}
-
-		// Write a final string of output if matcher type is
-		// AND or if we have extractors for the mechanism too.
-		if len(e.httpRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
-			e.writeOutputHTTP(compiledRequest, nil, extractorResults)
-			atomic.CompareAndSwapUint32(&e.results, 0, 1)
-		}
-	}
-
-	gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL)
-
-	return nil
-}
-
-// Close closes the http executor for a template.
-func (e *HTTPExecutor) Close() {
-	e.outputMutex.Lock()
-	e.writer.Flush()
-	e.outputMutex.Unlock()
-}
-
-// makeHTTPClient creates a http client
-func makeHTTPClient(proxyURL *url.URL, options *HTTPOptions) *retryablehttp.Client {
-	retryablehttpOptions := retryablehttp.DefaultOptionsSpraying
-	retryablehttpOptions.RetryWaitMax = 10 * time.Second
-	retryablehttpOptions.RetryMax = options.Retries
-	followRedirects := options.HTTPRequest.Redirects
-	maxRedirects := options.HTTPRequest.MaxRedirects
-
-	transport := &http.Transport{
-		MaxIdleConnsPerHost: -1,
-		TLSClientConfig: &tls.Config{
-			Renegotiation:      tls.RenegotiateOnceAsClient,
-			InsecureSkipVerify: true,
-		},
-		DisableKeepAlives: true,
-	}
-
-	// Attempts to overwrite the dial function with the socks proxied version
-	if options.ProxySocksURL != "" {
-		var proxyAuth *proxy.Auth
-		socksURL, err := url.Parse(options.ProxySocksURL)
-		if err == nil {
-			proxyAuth = &proxy.Auth{}
-			proxyAuth.User = socksURL.User.Username()
-			proxyAuth.Password, _ = socksURL.User.Password()
-		}
-		dialer, err := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct)
-		if err == nil {
-			transport.Dial = dialer.Dial
-		}
-	}
-
-	if proxyURL != nil {
-		transport.Proxy = http.ProxyURL(proxyURL)
-	}
-	return retryablehttp.NewWithHTTPClient(&http.Client{
-		Transport:     transport,
-		Timeout:       time.Duration(options.Timeout) * time.Second,
-		CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects),
-	}, retryablehttpOptions)
-}
-
-type checkRedirectFunc func(_ *http.Request, requests []*http.Request) error
-
-func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirectFunc {
-	return func(_ *http.Request, requests []*http.Request) error {
-		if !followRedirects {
-			return http.ErrUseLastResponse
-		}
-		if maxRedirects == 0 {
-			if len(requests) > 10 {
-				return http.ErrUseLastResponse
-			}
-			return nil
-		}
-		if len(requests) > maxRedirects {
-			return http.ErrUseLastResponse
-		}
-		return nil
-	}
-}
-
-func (e *HTTPExecutor) setCustomHeaders(r *requests.CompiledHTTP) {
-	for _, customHeader := range e.customHeaders {
-		// This should be pre-computed somewhere and done only once
-		tokens := strings.Split(customHeader, ":")
-		// if it's an invalid header skip it
-		if len(tokens) < 2 {
-			continue
-		}
-
-		headerName, headerValue := tokens[0], strings.Join(tokens[1:], "")
-		headerName = strings.TrimSpace(headerName)
-		headerValue = strings.TrimSpace(headerValue)
-		r.Request.Header.Set(headerName, headerValue)
-	}
-}

v2/pkg/executor/executor_dns.go

@@ -1,149 +0,0 @@
-package executor
-
-import (
-	"bufio"
-	"fmt"
-	"os"
-	"sync"
-	"sync/atomic"
-
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	retryabledns "github.com/projectdiscovery/retryabledns"
-)
-
-// DNSExecutor is a client for performing a DNS request
-// for a template.
-type DNSExecutor struct {
-	debug       bool
-	jsonOutput  bool
-	results     uint32
-	dnsClient   *retryabledns.Client
-	template    *templates.Template
-	dnsRequest  *requests.DNSRequest
-	writer      *bufio.Writer
-	outputMutex *sync.Mutex
-}
-
-// DefaultResolvers contains the list of resolvers known to be trusted.
-var DefaultResolvers = []string{
-	"1.1.1.1:53", // Cloudflare
-	"1.0.0.1:53", // Cloudflare
-	"8.8.8.8:53", // Google
-	"8.8.4.4:53", // Google
-}
-
-// DNSOptions contains configuration options for the DNS executor.
-type DNSOptions struct {
-	Debug      bool
-	JSON       bool
-	Template   *templates.Template
-	DNSRequest *requests.DNSRequest
-	Writer     *bufio.Writer
-}
-
-// NewDNSExecutor creates a new DNS executor from a template
-// and a DNS request query.
-func NewDNSExecutor(options *DNSOptions) *DNSExecutor {
-	dnsClient := retryabledns.New(DefaultResolvers, options.DNSRequest.Retries)
-
-	executer := &DNSExecutor{
-		debug:       options.Debug,
-		jsonOutput:  options.JSON,
-		results:     0,
-		dnsClient:   dnsClient,
-		template:    options.Template,
-		dnsRequest:  options.DNSRequest,
-		writer:      options.Writer,
-		outputMutex: &sync.Mutex{},
-	}
-	return executer
-}
-
-// GotResults returns true if there were any results for the executor
-func (e *DNSExecutor) GotResults() bool {
-	if atomic.LoadUint32(&e.results) == 0 {
-		return false
-	}
-	return true
-}
-
-// ExecuteDNS executes the DNS request on a URL
-func (e *DNSExecutor) ExecuteDNS(URL string) error {
-	// Parse the URL and return domain if URL.
-	var domain string
-	if isURL(URL) {
-		domain = extractDomain(URL)
-	} else {
-		domain = URL
-	}
-
-	// Compile each request for the template based on the URL
-	compiledRequest, err := e.dnsRequest.MakeDNSRequest(domain)
-	if err != nil {
-		return errors.Wrap(err, "could not make dns request")
-	}
-
-	if e.debug {
-		gologger.Infof("Dumped DNS request for %s (%s)\n\n", URL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s\n", compiledRequest.String())
-	}
-
-	// Send the request to the target servers
-	resp, err := e.dnsClient.Do(compiledRequest)
-	if err != nil {
-		return errors.Wrap(err, "could not send dns request")
-	}
-
-	gologger.Verbosef("Sent DNS request to %s\n", "dns-request", URL)
-
-	if e.debug {
-		gologger.Infof("Dumped DNS response for %s (%s)\n\n", URL, e.template.ID)
-		fmt.Fprintf(os.Stderr, "%s\n", resp.String())
-	}
-
-	matcherCondition := e.dnsRequest.GetMatchersCondition()
-	for _, matcher := range e.dnsRequest.Matchers {
-		// Check if the matcher matched
-		if !matcher.MatchDNS(resp) {
-			// If the condition is AND we haven't matched, return.
-			if matcherCondition == matchers.ANDCondition {
-				return nil
-			}
-		} else {
-			// If the matcher has matched, and its an OR
-			// write the first output then move to next matcher.
-			if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 {
-				e.writeOutputDNS(domain, matcher, nil)
-				atomic.CompareAndSwapUint32(&e.results, 0, 1)
-			}
-		}
-	}
-
-	// All matchers have successfully completed so now start with the
-	// next task which is extraction of input from matchers.
-	var extractorResults []string
-	for _, extractor := range e.dnsRequest.Extractors {
-		for match := range extractor.ExtractDNS(resp.String()) {
-			extractorResults = append(extractorResults, match)
-		}
-	}
-
-	// Write a final string of output if matcher type is
-	// AND or if we have extractors for the mechanism too.
-	if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
-		e.writeOutputDNS(domain, nil, extractorResults)
-		atomic.CompareAndSwapUint32(&e.results, 0, 1)
-	}
-	return nil
-}
-
-// Close closes the dns executor for a template.
-func (e *DNSExecutor) Close() {
-	e.outputMutex.Lock()
-	e.writer.Flush()
-	e.outputMutex.Unlock()
-}

v2/pkg/executor/http_utils.go

@@ -1,41 +0,0 @@
-package executor
-
-import (
-	"net/http"
-	"strings"
-	"unsafe"
-)
-
-type jsonOutput struct {
-	Template         string   `json:"template"`
-	Type             string   `json:"type"`
-	Matched          string   `json:"matched"`
-	MatcherName      string   `json:"matcher_name,omitempty"`
-	ExtractedResults []string `json:"extracted_results,omitempty"`
-	Severity         string   `json:"severity"`
-	Author           string   `json:"author"`
-}
-
-// unsafeToString converts byte slice to string with zero allocations
-func unsafeToString(bs []byte) string {
-	return *(*string)(unsafe.Pointer(&bs))
-}
-
-// headersToString converts http headers to string
-func headersToString(headers http.Header) string {
-	builder := &strings.Builder{}
-
-	for header, values := range headers {
-		builder.WriteString(header)
-		builder.WriteString(": ")
-
-		for i, value := range values {
-			builder.WriteString(value)
-			if i != len(values)-1 {
-				builder.WriteRune(',')
-			}
-		}
-		builder.WriteRune('\n')
-	}
-	return builder.String()
-}

v2/pkg/executor/output_dns.go

@@ -1,76 +0,0 @@
-package executor
-
-import (
-	"strings"
-
-	jsoniter "github.com/json-iterator/go"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-)
-
-// writeOutputDNS writes dns output to streams
-func (e *DNSExecutor) writeOutputDNS(domain string, matcher *matchers.Matcher, extractorResults []string) {
-	if e.jsonOutput {
-		output := jsonOutput{
-			Template: e.template.ID,
-			Type:     "dns",
-			Matched:  domain,
-			Severity: e.template.Info.Severity,
-			Author:   e.template.Info.Author,
-		}
-		if matcher != nil && len(matcher.Name) > 0 {
-			output.MatcherName = matcher.Name
-		}
-		if len(extractorResults) > 0 {
-			output.ExtractedResults = extractorResults
-		}
-		data, err := jsoniter.Marshal(output)
-		if err != nil {
-			gologger.Warningf("Could not marshal json output: %s\n", err)
-		}
-
-		gologger.Silentf("%s", string(data))
-
-		if e.writer != nil {
-			e.outputMutex.Lock()
-			e.writer.Write(data)
-			e.writer.WriteRune('\n')
-			e.outputMutex.Unlock()
-		}
-		return
-	}
-
-	builder := &strings.Builder{}
-	builder.WriteRune('[')
-	builder.WriteString(e.template.ID)
-	if matcher != nil && len(matcher.Name) > 0 {
-		builder.WriteString(":")
-		builder.WriteString(matcher.Name)
-	}
-	builder.WriteString("] [dns] ")
-
-	builder.WriteString(domain)
-
-	// If any extractors, write the results
-	if len(extractorResults) > 0 {
-		builder.WriteString(" [")
-		for i, result := range extractorResults {
-			builder.WriteString(result)
-			if i != len(extractorResults)-1 {
-				builder.WriteRune(',')
-			}
-		}
-		builder.WriteString("]")
-	}
-	builder.WriteRune('\n')
-
-	// Write output to screen as well as any output file
-	message := builder.String()
-	gologger.Silentf("%s", message)
-
-	if e.writer != nil {
-		e.outputMutex.Lock()
-		e.writer.WriteString(message)
-		e.outputMutex.Unlock()
-	}
-}

v2/pkg/executor/output_http.go

@@ -1,94 +0,0 @@
-package executor
-
-import (
-	"strings"
-
-	jsoniter "github.com/json-iterator/go"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
-)
-
-// writeOutputHTTP writes http output to streams
-func (e *HTTPExecutor) writeOutputHTTP(req *requests.CompiledHTTP, matcher *matchers.Matcher, extractorResults []string) {
-	URL := req.Request.URL.String()
-
-	if e.jsonOutput {
-		output := jsonOutput{
-			Template: e.template.ID,
-			Type:     "http",
-			Matched:  URL,
-			Severity: e.template.Info.Severity,
-			Author:   e.template.Info.Author,
-		}
-		if matcher != nil && len(matcher.Name) > 0 {
-			output.MatcherName = matcher.Name
-		}
-		if len(extractorResults) > 0 {
-			output.ExtractedResults = extractorResults
-		}
-		data, err := jsoniter.Marshal(output)
-		if err != nil {
-			gologger.Warningf("Could not marshal json output: %s\n", err)
-		}
-
-		gologger.Silentf("%s", string(data))
-
-		if e.writer != nil {
-			e.outputMutex.Lock()
-			e.writer.Write(data)
-			e.writer.WriteRune('\n')
-			e.outputMutex.Unlock()
-		}
-		return
-	}
-
-	builder := &strings.Builder{}
-
-	builder.WriteRune('[')
-	builder.WriteString(e.template.ID)
-	if matcher != nil && len(matcher.Name) > 0 {
-		builder.WriteString(":")
-		builder.WriteString(matcher.Name)
-	}
-	builder.WriteString("] [http] ")
-
-	// Escape the URL by replacing all % with %%
-	escapedURL := strings.Replace(URL, "%", "%%", -1)
-	builder.WriteString(escapedURL)
-
-	// If any extractors, write the results
-	if len(extractorResults) > 0 {
-		builder.WriteString(" [")
-		for i, result := range extractorResults {
-			builder.WriteString(result)
-			if i != len(extractorResults)-1 {
-				builder.WriteRune(',')
-			}
-		}
-		builder.WriteString("]")
-	}
-
-	// write meta if any
-	if len(req.Meta) > 0 {
-		builder.WriteString(" [")
-		var metas []string
-		for name, value := range req.Meta {
-			metas = append(metas, name+"="+value.(string))
-		}
-		builder.WriteString(strings.Join(metas, ","))
-		builder.WriteString("]")
-	}
-
-	builder.WriteRune('\n')
-
-	// Write output to screen as well as any output file
-	message := builder.String()
-	gologger.Silentf("%s", message)
-
-	if e.writer != nil {
-		e.outputMutex.Lock()
-		e.writer.WriteString(message)
-		e.outputMutex.Unlock()
-	}
-}

v2/pkg/extractors/compile.go

@@ -7,8 +7,9 @@ import (
 
 // CompileExtractors performs the initial setup operation on a extractor
 func (e *Extractor) CompileExtractors() error {
-	// Setup the matcher type
-	_, ok := ExtractorTypes[e.Type]
+	var ok bool
+	// Setup the extractor type
+	e.extractorType, ok = ExtractorTypes[e.Type]
 	if !ok {
 		return fmt.Errorf("unknown extractor type specified: %s", e.Type)
 	}
@@ -19,6 +20,7 @@ func (e *Extractor) CompileExtractors() error {
 		if err != nil {
 			return fmt.Errorf("could not compile regex: %s", regex)
 		}
+
 		e.regexCompiled = append(e.regexCompiled, compiled)
 	}
 
@@ -31,5 +33,6 @@ func (e *Extractor) CompileExtractors() error {
 	} else {
 		e.part = BodyPart
 	}
+
 	return nil
 }

v2/pkg/extractors/extract.go

@@ -1,36 +1,95 @@
 package extractors
 
+import (
+	"net/http"
+
+	"github.com/miekg/dns"
+)
+
 // Extract extracts response from the parts of request using a regex
-func (e *Extractor) Extract(body, headers string) map[string]struct{} {
-	// Match the parts as required for regex check
-	if e.part == BodyPart {
-		return e.extractRegex(body)
-	} else if e.part == HeaderPart {
-		return e.extractRegex(headers)
-	} else {
-		matches := e.extractRegex(headers)
+func (e *Extractor) Extract(resp *http.Response, body, headers string) map[string]struct{} {
+	switch e.extractorType {
+	case RegexExtractor:
+		if e.part == BodyPart {
+			return e.extractRegex(body)
+		} else if e.part == HeaderPart {
+			return e.extractRegex(headers)
+		} else {
+			matches := e.extractRegex(headers)
+			if len(matches) > 0 {
+				return matches
+			}
+			return e.extractRegex(body)
+		}
+	case KValExtractor:
+		if e.part == HeaderPart {
+			return e.extractKVal(resp)
+		}
+
+		matches := e.extractKVal(resp)
+
 		if len(matches) > 0 {
 			return matches
 		}
-		return e.extractRegex(body)
+
+		return e.extractCookieKVal(resp)
 	}
+
+	return nil
 }
 
 // ExtractDNS extracts response from dns message using a regex
-func (e *Extractor) ExtractDNS(msg string) map[string]struct{} {
-	// Match the parts as required for regex check
-	return e.extractRegex(msg)
+// nolint:interfacer // dns.Msg is out of current scope
+func (e *Extractor) ExtractDNS(msg *dns.Msg) map[string]struct{} {
+	switch e.extractorType {
+	case RegexExtractor:
+		return e.extractRegex(msg.String())
+	case KValExtractor:
+	}
+
+	return nil
 }
 
 // extractRegex extracts text from a corpus and returns it
 func (e *Extractor) extractRegex(corpus string) map[string]struct{} {
 	results := make(map[string]struct{})
 
+	groupPlusOne := e.RegexGroup + 1
 	for _, regex := range e.regexCompiled {
-		matches := regex.FindAllString(corpus, -1)
+		matches := regex.FindAllStringSubmatch(corpus, -1)
 		for _, match := range matches {
-			results[match] = struct{}{}
+			if len(match) >= groupPlusOne {
+				results[match[e.RegexGroup]] = struct{}{}
+			}
 		}
 	}
 	return results
 }
+
+// extractKVal extracts text from http response
+func (e *Extractor) extractKVal(r *http.Response) map[string]struct{} {
+	results := make(map[string]struct{})
+
+	for _, k := range e.KVal {
+		for _, v := range r.Header.Values(k) {
+			results[v] = struct{}{}
+		}
+	}
+
+	return results
+}
+
+// extractCookieKVal extracts text from cookies
+func (e *Extractor) extractCookieKVal(r *http.Response) map[string]struct{} {
+	results := make(map[string]struct{})
+
+	for _, k := range e.KVal {
+		for _, cookie := range r.Cookies() {
+			if cookie.Name == k {
+				results[cookie.Value] = struct{}{}
+			}
+		}
+	}
+
+	return results
+}

v2/pkg/extractors/extractors.go

@@ -4,20 +4,31 @@ import "regexp"
 
 // Extractor is used to extract part of response using a regex.
 type Extractor struct {
-	// Type is the type of the matcher
+	// Name is the extractor's name
+	Name string `yaml:"name,omitempty"`
+	// Type is the type of the extractor
 	Type string `yaml:"type"`
+	// extractorType is the internal type of the extractor
+	extractorType ExtractorType
 
 	// Regex are the regex pattern required to be present in the response
 	Regex []string `yaml:"regex"`
+	// RegexGroup specifies a group to extract from the regex
+	RegexGroup int `yaml:"group"`
 	// regexCompiled is the compiled variant
 	regexCompiled []*regexp.Regexp
 
+	// KVal are the kval to be present in the response headers/cookies
+	KVal []string `yaml:"kval,omitempty"`
+
 	// Part is the part of the request to match
 	//
 	// By default, matching is performed in request body.
 	Part string `yaml:"part,omitempty"`
 	// part is the part of the request to match
 	part Part
+	// Internal defines if this is used internally
+	Internal bool `yaml:"internal,omitempty"`
 }
 
 // ExtractorType is the type of the extractor specified
@@ -26,11 +37,14 @@ type ExtractorType = int
 const (
 	// RegexExtractor extracts responses with regexes
 	RegexExtractor ExtractorType = iota + 1
+	// KValExtractor extracts responses with key:value
+	KValExtractor
 )
 
 // ExtractorTypes is an table for conversion of extractor type from string.
 var ExtractorTypes = map[string]ExtractorType{
 	"regex": RegexExtractor,
+	"kval":  KValExtractor,
 }
 
 // Part is the part of the request to match

v2/pkg/generators/clusterbomb.go

@@ -8,8 +8,11 @@ func ClusterbombGenerator(payloads map[string][]string) (out chan map[string]int
 	// generator
 	go func() {
 		defer close(out)
+
 		var order []string
+
 		var parts [][]string
+
 		for name, wordlist := range payloads {
 			order = append(order, name)
 			parts = append(parts, wordlist)

v2/pkg/generators/dsl.go

@@ -6,106 +6,291 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
+	"fmt"
 	"html"
+	"math"
+	"math/rand"
 	"net/url"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/Knetic/govaluate"
+	"github.com/projectdiscovery/nuclei/v2/pkg/collaborator"
+	"github.com/spaolacci/murmur3"
 )
 
+const (
+	withCutSetArgsSize   = 2
+	withMaxRandArgsSize  = withCutSetArgsSize
+	withBaseRandArgsSize = 3
+)
+
+var letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+var numbers = "1234567890"
+
 // HelperFunctions contains the dsl functions
 func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 	functions = make(map[string]govaluate.ExpressionFunction)
+
 	// strings
 	functions["len"] = func(args ...interface{}) (interface{}, error) {
 		length := len(args[0].(string))
-		return (float64)(length), nil
+
+		return float64(length), nil
 	}
+
 	functions["toupper"] = func(args ...interface{}) (interface{}, error) {
 		return strings.ToUpper(args[0].(string)), nil
 	}
+
 	functions["tolower"] = func(args ...interface{}) (interface{}, error) {
 		return strings.ToLower(args[0].(string)), nil
 	}
+
 	functions["replace"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Replace(args[0].(string), args[1].(string), args[2].(string), -1), nil
+		return strings.ReplaceAll(args[0].(string), args[1].(string), args[2].(string)), nil
 	}
+
+	functions["replace_regex"] = func(args ...interface{}) (interface{}, error) {
+		compiled, err := regexp.Compile(args[1].(string))
+		if err != nil {
+			return nil, err
+		}
+		return compiled.ReplaceAllString(args[0].(string), args[2].(string)), nil
+	}
+
 	functions["trim"] = func(args ...interface{}) (interface{}, error) {
 		return strings.Trim(args[0].(string), args[2].(string)), nil
 	}
+
 	functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
 		return strings.TrimLeft(args[0].(string), args[1].(string)), nil
 	}
+
 	functions["trimright"] = func(args ...interface{}) (interface{}, error) {
 		return strings.TrimRight(args[0].(string), args[1].(string)), nil
 	}
+
 	functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
 		return strings.TrimSpace(args[0].(string)), nil
 	}
+
 	functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
 		return strings.TrimPrefix(args[0].(string), args[1].(string)), nil
 	}
+
 	functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
 		return strings.TrimSuffix(args[0].(string), args[1].(string)), nil
 	}
+
 	functions["reverse"] = func(args ...interface{}) (interface{}, error) {
 		return reverseString(args[0].(string)), nil
 	}
+
 	// encoding
 	functions["base64"] = func(args ...interface{}) (interface{}, error) {
 		sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
+
 		return sEnc, nil
 	}
-	functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
+
+	// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
+	functions["base64_py"] = func(args ...interface{}) (interface{}, error) {
 		sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
-		return sEnc, nil
+
+		return insertInto(sEnc, 76, '\n'), nil
 	}
+
+	functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
+		return base64.StdEncoding.DecodeString(args[0].(string))
+	}
+
 	functions["url_encode"] = func(args ...interface{}) (interface{}, error) {
 		return url.PathEscape(args[0].(string)), nil
 	}
+
 	functions["url_decode"] = func(args ...interface{}) (interface{}, error) {
 		return url.PathUnescape(args[0].(string))
 	}
+
 	functions["hex_encode"] = func(args ...interface{}) (interface{}, error) {
 		return hex.EncodeToString([]byte(args[0].(string))), nil
 	}
+
 	functions["hex_decode"] = func(args ...interface{}) (interface{}, error) {
 		hx, _ := hex.DecodeString(args[0].(string))
 		return string(hx), nil
 	}
+
 	functions["html_escape"] = func(args ...interface{}) (interface{}, error) {
 		return html.EscapeString(args[0].(string)), nil
 	}
+
 	functions["html_unescape"] = func(args ...interface{}) (interface{}, error) {
 		return html.UnescapeString(args[0].(string)), nil
 	}
+
 	// hashing
 	functions["md5"] = func(args ...interface{}) (interface{}, error) {
 		hash := md5.Sum([]byte(args[0].(string)))
+
 		return hex.EncodeToString(hash[:]), nil
 	}
+
 	functions["sha256"] = func(args ...interface{}) (interface{}, error) {
 		h := sha256.New()
-		h.Write([]byte(args[0].(string)))
+		_, err := h.Write([]byte(args[0].(string)))
+
+		if err != nil {
+			return nil, err
+		}
+
 		return hex.EncodeToString(h.Sum(nil)), nil
 	}
+
 	functions["sha1"] = func(args ...interface{}) (interface{}, error) {
 		h := sha1.New()
-		h.Write([]byte(args[0].(string)))
+		_, err := h.Write([]byte(args[0].(string)))
+
+		if err != nil {
+			return nil, err
+		}
+
 		return hex.EncodeToString(h.Sum(nil)), nil
 	}
+
+	functions["mmh3"] = func(args ...interface{}) (interface{}, error) {
+		return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(args[0].(string)), 0))), nil
+	}
+
 	// search
 	functions["contains"] = func(args ...interface{}) (interface{}, error) {
 		return strings.Contains(args[0].(string), args[1].(string)), nil
 	}
+
 	functions["regex"] = func(args ...interface{}) (interface{}, error) {
 		compiled, err := regexp.Compile(args[0].(string))
 		if err != nil {
 			return nil, err
 		}
+
 		return compiled.MatchString(args[1].(string)), nil
 	}
 
-	return
+	// random generators
+	functions["rand_char"] = func(args ...interface{}) (interface{}, error) {
+		chars := letters + numbers
+		bad := ""
+		if len(args) >= 1 {
+			chars = args[0].(string)
+		}
+		if len(args) >= withCutSetArgsSize {
+			bad = args[1].(string)
+		}
+
+		chars = TrimAll(chars, bad)
+
+		return chars[rand.Intn(len(chars))], nil
+	}
+
+	functions["rand_base"] = func(args ...interface{}) (interface{}, error) {
+		l := 0
+		bad := ""
+		base := letters + numbers
+
+		if len(args) >= 1 {
+			l = args[0].(int)
+		}
+		if len(args) >= withCutSetArgsSize {
+			bad = args[1].(string)
+		}
+		if len(args) >= withBaseRandArgsSize {
+			base = args[2].(string)
+		}
+
+		base = TrimAll(base, bad)
+
+		return RandSeq(base, l), nil
+	}
+
+	functions["rand_text_alphanumeric"] = func(args ...interface{}) (interface{}, error) {
+		l := 0
+		bad := ""
+		chars := letters + numbers
+
+		if len(args) >= 1 {
+			l = args[0].(int)
+		}
+		if len(args) >= withCutSetArgsSize {
+			bad = args[1].(string)
+		}
+
+		chars = TrimAll(chars, bad)
+
+		return RandSeq(chars, l), nil
+	}
+
+	functions["rand_text_alpha"] = func(args ...interface{}) (interface{}, error) {
+		l := 0
+		bad := ""
+		chars := letters
+
+		if len(args) >= 1 {
+			l = args[0].(int)
+		}
+		if len(args) >= withCutSetArgsSize {
+			bad = args[1].(string)
+		}
+
+		chars = TrimAll(chars, bad)
+
+		return RandSeq(chars, l), nil
+	}
+
+	functions["rand_text_numeric"] = func(args ...interface{}) (interface{}, error) {
+		l := 0
+		bad := ""
+		chars := numbers
+
+		if len(args) >= 1 {
+			l = args[0].(int)
+		}
+		if len(args) >= withCutSetArgsSize {
+			bad = args[1].(string)
+		}
+
+		chars = TrimAll(chars, bad)
+
+		return RandSeq(chars, l), nil
+	}
+
+	functions["rand_int"] = func(args ...interface{}) (interface{}, error) {
+		min := 0
+		max := math.MaxInt32
+
+		if len(args) >= 1 {
+			min = args[0].(int)
+		}
+		if len(args) >= withMaxRandArgsSize {
+			max = args[1].(int)
+		}
+
+		return rand.Intn(max-min) + min, nil
+	}
+
+	// Time Functions
+	functions["waitfor"] = func(args ...interface{}) (interface{}, error) {
+		seconds := args[0].(float64)
+		time.Sleep(time.Duration(seconds) * time.Second)
+		return true, nil
+	}
+
+	// Collaborator
+	functions["collab"] = func(args ...interface{}) (interface{}, error) {
+		// check if collaborator contains a specific pattern
+		return collaborator.DefaultCollaborator.Has(args[0].(string)), nil
+	}
+
+	return functions
 }

v2/pkg/generators/pitchfork.go

@@ -14,7 +14,7 @@ func PitchforkGenerator(payloads map[string][]string) (out chan map[string]inter
 		}
 
 		if len(wordlist) != size {
-			//set size = 0 and exit the cycle
+			// set size = 0 and exit the cycle
 			size = 0
 			break
 		}

v2/pkg/generators/util.go

@@ -2,19 +2,42 @@ package generators
 
 import (
 	"bufio"
+	"bytes"
+	"fmt"
+	"math/rand"
 	"os"
 	"strings"
 )
 
-// LoadWordlists creating proper data structure
-func LoadWordlists(payloads map[string]string) map[string][]string {
-	wordlists := make(map[string][]string)
+const two = 2
+
+// LoadPayloads creating proper data structure
+func LoadPayloads(payloads map[string]interface{}) map[string][]string {
+	loadedPayloads := make(map[string][]string)
 	// load all wordlists
-	for name, filepath := range payloads {
-		wordlists[name] = LoadFile(filepath)
+	for name, payload := range payloads {
+		switch pt := payload.(type) {
+		case string:
+			elements := strings.Split(pt, "\n")
+			if len(elements) >= two {
+				loadedPayloads[name] = elements
+			} else {
+				loadedPayloads[name] = LoadFile(pt)
+			}
+		case []interface{}, interface{}:
+			vv := payload.([]interface{})
+
+			var v []string
+
+			for _, vvv := range vv {
+				v = append(v, fmt.Sprintf("%v", vvv))
+			}
+
+			loadedPayloads[name] = v
+		}
 	}
 
-	return wordlists
+	return loadedPayloads
 }
 
 // LoadFile into slice of strings
@@ -32,7 +55,9 @@ func StreamFile(filepath string) (content chan string) {
 
 	go func() {
 		defer close(content)
+
 		file, err := os.Open(filepath)
+
 		if err != nil {
 			return
 		}
@@ -55,14 +80,30 @@ func StreamFile(filepath string) (content chan string) {
 // MergeMaps into a new one
 func MergeMaps(m1, m2 map[string]interface{}) (m map[string]interface{}) {
 	m = make(map[string]interface{})
+
 	for k, v := range m1 {
 		m[k] = v
 	}
+
 	for k, v := range m2 {
 		m[k] = v
 	}
 
-	return m
+	return
+}
+
+// MergeMapsWithStrings into a new string one
+func MergeMapsWithStrings(m1, m2 map[string]string) (m map[string]string) {
+	m = make(map[string]string)
+	for k, v := range m1 {
+		m[k] = v
+	}
+
+	for k, v := range m2 {
+		m[k] = v
+	}
+
+	return
 }
 
 func reverseString(s string) string {
@@ -70,6 +111,7 @@ func reverseString(s string) string {
 	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
 		runes[i], runes[j] = runes[j], runes[i]
 	}
+
 	return string(runes)
 }
 
@@ -79,6 +121,7 @@ func CopyMap(originalMap map[string]interface{}) map[string]interface{} {
 	for key, value := range originalMap {
 		newMap[key] = value
 	}
+
 	return newMap
 }
 
@@ -88,6 +131,7 @@ func CopyMapWithDefaultValue(originalMap map[string][]string, defaultValue inter
 	for key := range originalMap {
 		newMap[key] = defaultValue
 	}
+
 	return newMap
 }
 
@@ -113,5 +157,45 @@ func FileExists(filename string) bool {
 	if os.IsNotExist(err) {
 		return false
 	}
+
 	return !info.IsDir()
 }
+
+// TrimDelimiters removes trailing brackets
+func SliceContins(s []string, k string) bool {
+	for _, a := range s {
+		if a == k {
+			return true
+		}
+	}
+	return false
+}
+
+func TrimAll(s, cutset string) string {
+	for _, c := range cutset {
+		s = strings.ReplaceAll(s, string(c), "")
+	}
+	return s
+}
+
+func RandSeq(base string, n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = rune(base[rand.Intn(len(base))])
+	}
+	return string(b)
+}
+
+func insertInto(s string, interval int, sep rune) string {
+	var buffer bytes.Buffer
+	before := interval - 1
+	last := len(s) - 1
+	for i, char := range s {
+		buffer.WriteRune(char)
+		if i%interval == before && i != last {
+			buffer.WriteRune(sep)
+		}
+	}
+	buffer.WriteRune(sep)
+	return buffer.String()
+}

v2/pkg/matchers/compile.go

@@ -57,5 +57,6 @@ func (m *Matcher) CompileMatchers() error {
 	} else {
 		m.part = BodyPart
 	}
+
 	return nil
 }

v2/pkg/matchers/match.go

@@ -4,57 +4,51 @@ import (
 	"encoding/hex"
 	"net/http"
 	"strings"
+	"time"
 
 	"github.com/miekg/dns"
+	"github.com/projectdiscovery/nuclei/v2/pkg/generators"
 )
 
 // Match matches a http response again a given matcher
-func (m *Matcher) Match(resp *http.Response, body, headers string) bool {
+func (m *Matcher) Match(resp *http.Response, body, headers string, duration time.Duration, data map[string]interface{}) bool {
 	switch m.matcherType {
 	case StatusMatcher:
-		return m.matchStatusCode(resp.StatusCode)
+		return m.isNegative(m.matchStatusCode(resp.StatusCode))
 	case SizeMatcher:
-		return m.matchSizeCode(len(body))
+		return m.isNegative(m.matchSizeCode(len(body)))
 	case WordsMatcher:
 		// Match the parts as required for word check
 		if m.part == BodyPart {
-			return m.matchWords(body)
+			return m.isNegative(m.matchWords(body))
 		} else if m.part == HeaderPart {
-			return m.matchWords(headers)
+			return m.isNegative(m.matchWords(headers))
 		} else {
-			if !m.matchWords(headers) {
-				return false
-			}
-			return m.matchWords(body)
+			return m.isNegative(m.matchWords(headers) || m.matchWords(body))
 		}
 	case RegexMatcher:
 		// Match the parts as required for regex check
 		if m.part == BodyPart {
-			return m.matchRegex(body)
+			return m.isNegative(m.matchRegex(body))
 		} else if m.part == HeaderPart {
-			return m.matchRegex(headers)
+			return m.isNegative(m.matchRegex(headers))
 		} else {
-			if m.matchRegex(headers) {
-				return true
-			}
-			return m.matchRegex(body)
+			return m.isNegative(m.matchRegex(headers) || m.matchRegex(body))
 		}
 	case BinaryMatcher:
 		// Match the parts as required for binary characters check
 		if m.part == BodyPart {
-			return m.matchBinary(body)
+			return m.isNegative(m.matchBinary(body))
 		} else if m.part == HeaderPart {
-			return m.matchBinary(headers)
+			return m.isNegative(m.matchBinary(headers))
 		} else {
-			if !m.matchBinary(headers) {
-				return false
-			}
-			return m.matchBinary(body)
+			return m.isNegative(m.matchBinary(headers) || m.matchBinary(body))
 		}
 	case DSLMatcher:
 		// Match complex query
-		return m.matchDSL(httpToMap(resp, body, headers))
+		return m.isNegative(m.matchDSL(generators.MergeMaps(HTTPToMap(resp, body, headers, duration, ""), data)))
 	}
+
 	return false
 }
 
@@ -75,8 +69,9 @@ func (m *Matcher) MatchDNS(msg *dns.Msg) bool {
 		return m.matchBinary(msg.String())
 	case DSLMatcher:
 		// Match complex query
-		return m.matchDSL(dnsToMap(msg))
+		return m.matchDSL(DNSToMap(msg, ""))
 	}
+
 	return false
 }
 
@@ -93,6 +88,7 @@ func (m *Matcher) matchStatusCode(statusCode int) bool {
 		// Return on the first match.
 		return true
 	}
+
 	return false
 }
 
@@ -109,6 +105,7 @@ func (m *Matcher) matchSizeCode(length int) bool {
 		// Return on the first match.
 		return true
 	}
+
 	return false
 }
 
@@ -117,7 +114,7 @@ func (m *Matcher) matchWords(corpus string) bool {
 	// Iterate over all the words accepted as valid
 	for i, word := range m.Words {
 		// Continue if the word doesn't match
-		if strings.Index(corpus, word) == -1 {
+		if !strings.Contains(corpus, word) {
 			// If we are in an AND request and a match failed,
 			// return false as the AND condition fails on any single mismatch.
 			if m.condition == ANDCondition {
@@ -137,6 +134,7 @@ func (m *Matcher) matchWords(corpus string) bool {
 			return true
 		}
 	}
+
 	return false
 }
 
@@ -165,16 +163,15 @@ func (m *Matcher) matchRegex(corpus string) bool {
 			return true
 		}
 	}
+
 	return false
 }
 
 // matchWords matches a word check against an HTTP Response/Headers.
 func (m *Matcher) matchBinary(corpus string) bool {
-
 	// Iterate over all the words accepted as valid
 	for i, binary := range m.Binary {
 		// Continue if the word doesn't match
-
 		hexa, _ := hex.DecodeString(binary)
 		if !strings.Contains(corpus, string(hexa)) {
 			// If we are in an AND request and a match failed,
@@ -196,6 +193,7 @@ func (m *Matcher) matchBinary(corpus string) bool {
 			return true
 		}
 	}
+
 	return false
 }
 
@@ -207,6 +205,7 @@ func (m *Matcher) matchDSL(mp map[string]interface{}) bool {
 		if err != nil {
 			continue
 		}
+
 		var bResult bool
 		bResult, ok := result.(bool)
 
@@ -231,5 +230,6 @@ func (m *Matcher) matchDSL(mp map[string]interface{}) bool {
 			return true
 		}
 	}
+
 	return false
 }

v2/pkg/matchers/matchers.go

@@ -13,7 +13,7 @@ type Matcher struct {
 	// matcherType is the internal type of the matcher
 	matcherType MatcherType
 
-	// Name is matcher Name to be displayed in result output.
+	// Name is matcher Name
 	Name string `yaml:"name,omitempty"`
 	// Status are the acceptable status codes for the response
 	Status []int `yaml:"status,omitempty"`
@@ -45,6 +45,10 @@ type Matcher struct {
 	Part string `yaml:"part,omitempty"`
 	// part is the part of the request to match
 	part Part
+
+	// Negative specifies if the match should be reversed
+	// It will only match if the condition is not true.
+	Negative bool `yaml:"negative,omitempty"`
 }
 
 // MatcherType is the type of the matcher specified
@@ -114,3 +118,13 @@ var PartTypes = map[string]Part{
 func (m *Matcher) GetPart() Part {
 	return m.part
 }
+
+// isNegative reverts the results of the match if the matcher
+// is of type negative.
+func (m *Matcher) isNegative(data bool) bool {
+	if m.Negative {
+		return !data
+	}
+
+	return data
+}

v2/pkg/matchers/util.go

@@ -5,58 +5,81 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"strings"
+	"time"
 
 	"github.com/miekg/dns"
 )
 
-func httpToMap(resp *http.Response, body, headers string) (m map[string]interface{}) {
+const defaultFormat = "%s"
+
+// HTTPToMap Converts HTTP to Matcher Map
+func HTTPToMap(resp *http.Response, body, headers string, duration time.Duration, format string) (m map[string]interface{}) {
 	m = make(map[string]interface{})
 
-	m["content_length"] = resp.ContentLength
-	m["status_code"] = resp.StatusCode
-	for k, v := range resp.Header {
-		k = strings.ToLower(strings.TrimSpace(strings.Replace(k, "-", "_", -1)))
-		m[k] = strings.Join(v, " ")
+	if format == "" {
+		format = defaultFormat
 	}
-	m["all_headers"] = headers
 
-	m["body"] = body
-	if r, err := httputil.DumpResponse(resp, true); err == nil {
-		m["raw"] = string(r)
+	m[fmt.Sprintf(format, "content_length")] = resp.ContentLength
+	m[fmt.Sprintf(format, "status_code")] = resp.StatusCode
+
+	for k, v := range resp.Header {
+		k = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(k, "-", "_")))
+		m[fmt.Sprintf(format, k)] = strings.Join(v, " ")
 	}
 
+	m[fmt.Sprintf(format, "all_headers")] = headers
+	m[fmt.Sprintf(format, "body")] = body
+
+	if r, err := httputil.DumpResponse(resp, true); err == nil {
+		m[fmt.Sprintf(format, "raw")] = string(r)
+	}
+
+	// Converts duration to seconds (floating point) for DSL syntax
+	m[fmt.Sprintf(format, "duration")] = duration.Seconds()
+
 	return m
 }
 
-func dnsToMap(msg *dns.Msg) (m map[string]interface{}) {
+// DNSToMap Converts DNS to Matcher Map
+func DNSToMap(msg *dns.Msg, format string) (m map[string]interface{}) {
 	m = make(map[string]interface{})
 
-	m["rcode"] = msg.Rcode
+	if format == "" {
+		format = defaultFormat
+	}
+
+	m[fmt.Sprintf(format, "rcode")] = msg.Rcode
+
 	var qs string
+
 	for _, question := range msg.Question {
 		qs += fmt.Sprintln(question.String())
 	}
-	m["question"] = qs
+
+	m[fmt.Sprintf(format, "question")] = qs
 
 	var exs string
 	for _, extra := range msg.Extra {
 		exs += fmt.Sprintln(extra.String())
 	}
-	m["extra"] = exs
+
+	m[fmt.Sprintf(format, "extra")] = exs
 
 	var ans string
 	for _, answer := range msg.Answer {
 		ans += fmt.Sprintln(answer.String())
 	}
-	m["answer"] = ans
+
+	m[fmt.Sprintf(format, "answer")] = ans
 
 	var nss string
 	for _, ns := range msg.Ns {
 		nss += fmt.Sprintln(ns.String())
 	}
-	m["ns"] = nss
 
-	m["raw"] = msg.String()
+	m[fmt.Sprintf(format, "ns")] = nss
+	m[fmt.Sprintf(format, "raw")] = msg.String()
 
 	return m
 }

v2/pkg/projectfile/httputil.go

@@ -0,0 +1,141 @@
+package projectfile
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/gob"
+	"encoding/hex"
+	"io"
+	"io/ioutil"
+	"net/http"
+)
+
+func hash(v interface{}) (string, error) {
+	data, err := marshal(v)
+	if err != nil {
+		return "", err
+	}
+
+	sh := sha256.New()
+
+	_, err = io.WriteString(sh, string(data))
+	if err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(sh.Sum(nil)), nil
+}
+
+func marshal(data interface{}) ([]byte, error) {
+	var b bytes.Buffer
+	enc := gob.NewEncoder(&b)
+	err := enc.Encode(data)
+	if err != nil {
+		return nil, err
+	}
+
+	return b.Bytes(), nil
+}
+
+func unmarshal(data []byte, obj interface{}) error {
+	dec := gob.NewDecoder(bytes.NewBuffer(data))
+	err := dec.Decode(obj)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type HTTPRecord struct {
+	Request  []byte
+	Response *InternalResponse
+}
+
+type InternalRequest struct {
+	Target    string
+	HTTPMajor int
+	HTTPMinor int
+	Method    string
+	Headers   map[string][]string
+	Body      []byte
+}
+
+type InternalResponse struct {
+	HTTPMajor    int
+	HTTPMinor    int
+	StatusCode   int
+	StatusReason string
+	Headers      map[string][]string
+	Body         []byte
+}
+
+// Unused
+// func newInternalRequest() *InternalRequest {
+// 	return &InternalRequest{
+// 		Headers: make(map[string][]string),
+// 	}
+// }
+
+func newInternalResponse() *InternalResponse {
+	return &InternalResponse{
+		Headers: make(map[string][]string),
+	}
+}
+
+// Unused
+// func toInternalRequest(req *http.Request, target string, body []byte) *InternalRequest {
+// 	intReq := newInternalRquest()
+
+// 	intReq.Target = target
+// 	intReq.HTTPMajor = req.ProtoMajor
+// 	intReq.HTTPMinor = req.ProtoMinor
+// 	for k, v := range req.Header {
+// 		intReq.Headers[k] = v
+// 	}
+// 	intReq.Headers = req.Header
+// 	intReq.Method = req.Method
+// 	intReq.Body = body
+
+// 	return intReq
+// }
+
+func toInternalResponse(resp *http.Response, body []byte) *InternalResponse {
+	intResp := newInternalResponse()
+
+	intResp.HTTPMajor = resp.ProtoMajor
+	intResp.HTTPMinor = resp.ProtoMinor
+	intResp.StatusCode = resp.StatusCode
+	intResp.StatusReason = resp.Status
+	for k, v := range resp.Header {
+		intResp.Headers[k] = v
+	}
+	intResp.Body = body
+	return intResp
+}
+
+func fromInternalResponse(intResp *InternalResponse) *http.Response {
+	var contentLength int64
+	if intResp.Body != nil {
+		contentLength = int64(len(intResp.Body))
+	}
+	return &http.Response{
+		ProtoMinor:    intResp.HTTPMinor,
+		ProtoMajor:    intResp.HTTPMajor,
+		Status:        intResp.StatusReason,
+		StatusCode:    intResp.StatusCode,
+		Header:        intResp.Headers,
+		ContentLength: contentLength,
+		Body:          ioutil.NopCloser(bytes.NewReader(intResp.Body)),
+	}
+}
+
+// Unused
+// func fromInternalRequest(intReq *InternalRequest) *http.Request {
+// 	return &http.Request{
+// 		ProtoMinor:    intReq.HTTPMinor,
+// 		ProtoMajor:    intReq.HTTPMajor,
+// 		Header:        intReq.Headers,
+// 		ContentLength: int64(len(intReq.Body)),
+// 		Body:          ioutil.NopCloser(bytes.NewReader(intReq.Body)),
+// 	}
+// }

v2/pkg/projectfile/project.go

@@ -0,0 +1,74 @@
+package projectfile
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/projectdiscovery/hmap/store/hybrid"
+)
+
+type Options struct {
+	Path    string
+	Cleanup bool
+}
+
+type ProjectFile struct {
+	Path string
+	hm   *hybrid.HybridMap
+}
+
+func New(options *Options) (*ProjectFile, error) {
+	var p ProjectFile
+	hOptions := hybrid.DefaultDiskOptions
+	hOptions.Path = options.Path
+	hOptions.Cleanup = options.Cleanup
+	var err error
+	p.hm, err = hybrid.New(hOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	return &p, nil
+}
+
+func (pf *ProjectFile) Get(req []byte) (*http.Response, error) {
+	reqHash, err := hash(req)
+	if err != nil {
+		return nil, err
+	}
+
+	data, ok := pf.hm.Get(reqHash)
+	if !ok {
+		return nil, fmt.Errorf("not found")
+	}
+
+	var httprecord HTTPRecord
+	httprecord.Response = newInternalResponse()
+	err = unmarshal(data, &httprecord)
+	if err != nil {
+		return nil, err
+	}
+
+	return fromInternalResponse(httprecord.Response), nil
+}
+
+func (pf *ProjectFile) Set(req []byte, resp *http.Response, data []byte) error {
+	reqHash, err := hash(req)
+	if err != nil {
+		return err
+	}
+
+	var httprecord HTTPRecord
+	httprecord.Request = req
+	httprecord.Response = toInternalResponse(resp, data)
+	data, err = marshal(httprecord)
+	if err != nil {
+		return err
+	}
+
+	return pf.hm.Set(reqHash, data)
+}
+
+func (pf *ProjectFile) Close() {
+	pf.hm.Close()
+}

v2/pkg/requests/bulk-http-request.go

@@ -0,0 +1,484 @@
+package requests
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/Knetic/govaluate"
+	"github.com/projectdiscovery/nuclei/v2/pkg/extractors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/syncedreadcloser"
+	"github.com/projectdiscovery/rawhttp"
+	retryablehttp "github.com/projectdiscovery/retryablehttp-go"
+)
+
+const (
+	two   = 2
+	three = 3
+)
+
+var urlWithPortRgx = regexp.MustCompile(`{{BaseURL}}:(\d+)`)
+
+// BulkHTTPRequest contains a request to be made from a template
+type BulkHTTPRequest struct {
+	// Path contains the path/s for the request
+	Path []string `yaml:"path"`
+	// Matchers contains the detection mechanism for the request to identify
+	// whether the request was successful
+	Matchers []*matchers.Matcher `yaml:"matchers,omitempty"`
+	// Extractors contains the extraction mechanism for the request to identify
+	// and extract parts of the response.
+	Extractors []*extractors.Extractor `yaml:"extractors,omitempty"`
+	// Raw contains raw requests
+	Raw  []string `yaml:"raw,omitempty"`
+	Name string   `yaml:"Name,omitempty"`
+	// AttackType is the attack type
+	// Sniper, PitchFork and ClusterBomb. Default is Sniper
+	AttackType string `yaml:"attack,omitempty"`
+	// Method is the request method, whether GET, POST, PUT, etc
+	Method string `yaml:"method"`
+	// Body is an optional parameter which contains the request body for POST methods, etc
+	Body string `yaml:"body,omitempty"`
+	// MatchersCondition is the condition of the matchers
+	// whether to use AND or OR. Default is OR.
+	MatchersCondition string `yaml:"matchers-condition,omitempty"`
+	// attackType is internal attack type
+	attackType generators.Type
+	// Path contains the path/s for the request variables
+	Payloads map[string]interface{} `yaml:"payloads,omitempty"`
+	// Headers contains headers to send with the request
+	Headers map[string]string `yaml:"headers,omitempty"`
+	// matchersCondition is internal condition for the matchers.
+	matchersCondition matchers.ConditionType
+	// MaxRedirects is the maximum number of redirects that should be followed.
+	MaxRedirects                  int `yaml:"max-redirects,omitempty"`
+	PipelineConcurrentConnections int `yaml:"pipeline-concurrent-connections,omitempty"`
+	PipelineRequestsPerConnection int `yaml:"pipeline-requests-per-connection,omitempty"`
+	Threads                       int `yaml:"threads,omitempty"`
+	// Internal Finite State Machine keeping track of scan process
+	gsfm *GeneratorFSM
+	// CookieReuse is an optional setting that makes cookies shared within requests
+	CookieReuse bool `yaml:"cookie-reuse,omitempty"`
+	// Redirects specifies whether redirects should be followed.
+	Redirects bool `yaml:"redirects,omitempty"`
+	// Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining (race conditions/billions requests)
+	// All requests must be indempotent (GET/POST)
+	Pipeline bool `yaml:"pipeline,omitempty"`
+	// Specify in order to skip request RFC normalization
+	Unsafe bool `yaml:"unsafe,omitempty"`
+	// DisableAutoHostname Enable/Disable Host header for unsafe raw requests
+	DisableAutoHostname bool `yaml:"disable-automatic-host-header,omitempty"`
+	// DisableAutoContentLength Enable/Disable Content-Length header for unsafe raw requests
+	DisableAutoContentLength bool `yaml:"disable-automatic-content-length-header,omitempty"`
+	// Race determines if all the request have to be attempted at the same time
+	// The minimum number fof requests is determined by threads
+	Race bool `yaml:"race,omitempty"`
+	// Number of same request to send in race condition attack
+	RaceNumberRequests int `yaml:"race_count,omitempty"`
+}
+
+// GetMatchersCondition returns the condition for the matcher
+func (r *BulkHTTPRequest) GetMatchersCondition() matchers.ConditionType {
+	return r.matchersCondition
+}
+
+// SetMatchersCondition sets the condition for the matcher
+func (r *BulkHTTPRequest) SetMatchersCondition(condition matchers.ConditionType) {
+	r.matchersCondition = condition
+}
+
+// GetAttackType returns the attack
+func (r *BulkHTTPRequest) GetAttackType() generators.Type {
+	return r.attackType
+}
+
+// SetAttackType sets the attack
+func (r *BulkHTTPRequest) SetAttackType(attack generators.Type) {
+	r.attackType = attack
+}
+
+// GetRequestCount returns the total number of requests the YAML rule will perform
+func (r *BulkHTTPRequest) GetRequestCount() int64 {
+	return int64(r.gsfm.Total())
+}
+
+// MakeHTTPRequest makes the HTTP request
+func (r *BulkHTTPRequest) MakeHTTPRequest(baseURL string, dynamicValues map[string]interface{}, data string) (*HTTPRequest, error) {
+	ctx := context.Background()
+
+	parsed, err := url.Parse(baseURL)
+	if err != nil {
+		return nil, err
+	}
+
+	hostname := parsed.Host
+
+	values := generators.MergeMaps(dynamicValues, map[string]interface{}{
+		"BaseURL":  baseURLWithTemplatePrefs(data, parsed),
+		"Hostname": hostname,
+	})
+
+	// if data contains \n it's a raw request
+	if strings.Contains(data, "\n") {
+		return r.makeHTTPRequestFromRaw(ctx, baseURL, data, values)
+	}
+
+	return r.makeHTTPRequestFromModel(ctx, data, values)
+}
+
+// MakeHTTPRequestFromModel creates a *http.Request from a request template
+func (r *BulkHTTPRequest) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}) (*HTTPRequest, error) {
+	replacer := newReplacer(values)
+	URL := replacer.Replace(data)
+
+	// Build a request on the specified URL
+	req, err := http.NewRequestWithContext(ctx, r.Method, URL, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	request, err := r.fillRequest(req, values)
+	if err != nil {
+		return nil, err
+	}
+
+	return &HTTPRequest{Request: request}, nil
+}
+
+// InitGenerator initializes the generator
+func (r *BulkHTTPRequest) InitGenerator() {
+	r.gsfm = NewGeneratorFSM(r.attackType, r.Payloads, r.Path, r.Raw)
+}
+
+// CreateGenerator creates the generator
+func (r *BulkHTTPRequest) CreateGenerator(reqURL string) {
+	r.gsfm.Add(reqURL)
+}
+
+// HasGenerator check if an URL has a generator
+func (r *BulkHTTPRequest) HasGenerator(reqURL string) bool {
+	return r.gsfm.Has(reqURL)
+}
+
+// ReadOne reads and return a generator by URL
+func (r *BulkHTTPRequest) ReadOne(reqURL string) {
+	r.gsfm.ReadOne(reqURL)
+}
+
+// makeHTTPRequestFromRaw creates a *http.Request from a raw request
+func (r *BulkHTTPRequest) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values map[string]interface{}) (*HTTPRequest, error) {
+	// Add trailing line
+	data += "\n"
+
+	if len(r.Payloads) > 0 {
+		r.gsfm.InitOrSkip(baseURL)
+		r.ReadOne(baseURL)
+
+		return r.handleRawWithPaylods(ctx, data, baseURL, values, r.gsfm.Value(baseURL))
+	}
+
+	// otherwise continue with normal flow
+	return r.handleRawWithPaylods(ctx, data, baseURL, values, nil)
+}
+
+func (r *BulkHTTPRequest) handleRawWithPaylods(ctx context.Context, raw, baseURL string, values, genValues map[string]interface{}) (*HTTPRequest, error) {
+	baseValues := generators.CopyMap(values)
+	finValues := generators.MergeMaps(baseValues, genValues)
+
+	replacer := newReplacer(finValues)
+
+	// Replace the dynamic variables in the URL if any
+	raw = replacer.Replace(raw)
+
+	dynamicValues := make(map[string]interface{})
+	// find all potentials tokens between {{}}
+	var re = regexp.MustCompile(`(?m)\{\{.+}}`)
+	for _, match := range re.FindAllString(raw, -1) {
+		// check if the match contains a dynamic variable
+		expr := generators.TrimDelimiters(match)
+		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, generators.HelperFunctions())
+
+		if err != nil {
+			return nil, err
+		}
+
+		result, err := compiled.Evaluate(finValues)
+		if err != nil {
+			return nil, err
+		}
+
+		dynamicValues[expr] = result
+	}
+
+	// replace dynamic values
+	dynamicReplacer := newReplacer(dynamicValues)
+	raw = dynamicReplacer.Replace(raw)
+
+	rawRequest, err := r.parseRawRequest(raw, baseURL)
+	if err != nil {
+		return nil, err
+	}
+
+	// rawhttp
+	if r.Unsafe {
+		unsafeReq := &HTTPRequest{
+			RawRequest:                   rawRequest,
+			Meta:                         genValues,
+			AutomaticHostHeader:          !r.DisableAutoHostname,
+			AutomaticContentLengthHeader: !r.DisableAutoContentLength,
+			Unsafe:                       true,
+			FollowRedirects:              r.Redirects,
+		}
+		return unsafeReq, nil
+	}
+
+	// retryablehttp
+	var body io.ReadCloser
+	body = ioutil.NopCloser(strings.NewReader(rawRequest.Data))
+	if r.Race {
+		// More or less this ensures that all requests hit the endpoint at the same approximated time
+		// Todo: sync internally upon writing latest request byte
+		body = syncedreadcloser.NewOpenGateWithTimeout(body, time.Duration(two)*time.Second)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, rawRequest.Method, rawRequest.FullURL, body)
+	if err != nil {
+		return nil, err
+	}
+
+	// copy headers
+	for key, value := range rawRequest.Headers {
+		req.Header[key] = []string{value}
+	}
+
+	request, err := r.fillRequest(req, values)
+	if err != nil {
+		return nil, err
+	}
+
+	return &HTTPRequest{Request: request, Meta: genValues}, nil
+}
+
+func (r *BulkHTTPRequest) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
+	replacer := newReplacer(values)
+	// Set the header values requested
+	for header, value := range r.Headers {
+		req.Header[header] = []string{replacer.Replace(value)}
+	}
+
+	// In case of multiple threads the underlying connection should remain open to allow reuse
+	if r.Threads <= 0 && req.Header.Get("Connection") == "" {
+		req.Close = true
+	}
+
+	// Check if the user requested a request body
+	if r.Body != "" {
+		req.Body = ioutil.NopCloser(strings.NewReader(r.Body))
+	}
+
+	setHeader(req, "User-Agent", "Nuclei - Open-source project (github.com/projectdiscovery/nuclei)")
+
+	// raw requests are left untouched
+	if len(r.Raw) > 0 {
+		return retryablehttp.FromRequest(req)
+	}
+
+	setHeader(req, "Accept", "*/*")
+	setHeader(req, "Accept-Language", "en")
+
+	return retryablehttp.FromRequest(req)
+}
+
+// HTTPRequest is the basic HTTP request
+type HTTPRequest struct {
+	Request    *retryablehttp.Request
+	RawRequest *RawRequest
+	Meta       map[string]interface{}
+
+	// flags
+	Unsafe                       bool
+	Pipeline                     bool
+	AutomaticHostHeader          bool
+	AutomaticContentLengthHeader bool
+	AutomaticConnectionHeader    bool
+	FollowRedirects              bool
+	Rawclient                    *rawhttp.Client
+	Httpclient                   *retryablehttp.Client
+	PipelineClient               *rawhttp.PipelineClient
+}
+
+func setHeader(req *http.Request, name, value string) {
+	// Set some headers only if the header wasn't supplied by the user
+	if _, ok := req.Header[name]; !ok {
+		req.Header.Set(name, value)
+	}
+}
+
+// baseURLWithTemplatePrefs returns the url for BaseURL keeping
+// the template port and path preference
+func baseURLWithTemplatePrefs(data string, parsedURL *url.URL) string {
+	// template port preference over input URL port
+	// template has port
+	hasPort := len(urlWithPortRgx.FindStringSubmatch(data)) > 0
+	if hasPort {
+		// check if also the input contains port, in this case extracts the url
+		if hostname, _, err := net.SplitHostPort(parsedURL.Host); err == nil {
+			parsedURL.Host = hostname
+		}
+	}
+
+	return parsedURL.String()
+}
+
+// CustomHeaders valid for all requests
+type CustomHeaders []string
+
+// String returns just a label
+func (c *CustomHeaders) String() string {
+	return "Custom Global Headers"
+}
+
+// Set a new global header
+func (c *CustomHeaders) Set(value string) error {
+	*c = append(*c, value)
+	return nil
+}
+
+// RawRequest defines a basic HTTP raw request
+type RawRequest struct {
+	FullURL string
+	Method  string
+	Path    string
+	Data    string
+	Headers map[string]string
+}
+
+// parseRawRequest parses the raw request as supplied by the user
+func (r *BulkHTTPRequest) parseRawRequest(request, baseURL string) (*RawRequest, error) {
+	reader := bufio.NewReader(strings.NewReader(request))
+
+	rawRequest := RawRequest{
+		Headers: make(map[string]string),
+	}
+
+	s, err := reader.ReadString('\n')
+	if err != nil {
+		return nil, fmt.Errorf("could not read request: %s", err)
+	}
+
+	parts := strings.Split(s, " ")
+
+	if len(parts) < three {
+		return nil, fmt.Errorf("malformed request supplied")
+	}
+	// Set the request Method
+	rawRequest.Method = parts[0]
+
+	// Accepts all malformed headers
+	var key, value string
+	for {
+		line, readErr := reader.ReadString('\n')
+		line = strings.TrimSpace(line)
+
+		if readErr != nil || line == "" {
+			break
+		}
+
+		p := strings.SplitN(line, ":", two)
+		key = p[0]
+		if len(p) > 1 {
+			value = p[1]
+		}
+
+		rawRequest.Headers[key] = value
+	}
+
+	// Handle case with the full http url in path. In that case,
+	// ignore any host header that we encounter and use the path as request URL
+	if !r.Unsafe && strings.HasPrefix(parts[1], "http") {
+		parsed, parseErr := url.Parse(parts[1])
+		if parseErr != nil {
+			return nil, fmt.Errorf("could not parse request URL: %s", parseErr)
+		}
+
+		rawRequest.Path = parts[1]
+		rawRequest.Headers["Host"] = parsed.Host
+	} else {
+		rawRequest.Path = parts[1]
+	}
+
+	// If raw request doesn't have a Host header and/ path,
+	// this will be generated from the parsed baseURL
+	parsedURL, err := url.Parse(baseURL)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse request URL: %s", err)
+	}
+
+	var hostURL string
+	if rawRequest.Headers["Host"] == "" {
+		hostURL = parsedURL.Host
+	} else {
+		hostURL = rawRequest.Headers["Host"]
+	}
+
+	if rawRequest.Path == "" {
+		rawRequest.Path = parsedURL.Path
+	} else if strings.HasPrefix(rawRequest.Path, "?") {
+		// requests generated from http.ReadRequest have incorrect RequestURI, so they
+		// cannot be used to perform another request directly, we need to generate a new one
+		// with the new target url
+		rawRequest.Path = fmt.Sprintf("%s%s", parsedURL.Path, rawRequest.Path)
+	}
+
+	rawRequest.FullURL = fmt.Sprintf("%s://%s%s", parsedURL.Scheme, strings.TrimSpace(hostURL), rawRequest.Path)
+
+	// Set the request body
+	b, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return nil, fmt.Errorf("could not read request body: %s", err)
+	}
+
+	rawRequest.Data = string(b)
+
+	return &rawRequest, nil
+}
+
+// Next returns the next generator by URL
+func (r *BulkHTTPRequest) Next(reqURL string) bool {
+	return r.gsfm.Next(reqURL)
+}
+
+// Position returns the current generator's position by URL
+func (r *BulkHTTPRequest) Position(reqURL string) int {
+	return r.gsfm.Position(reqURL)
+}
+
+// Reset resets the generator by URL
+func (r *BulkHTTPRequest) Reset(reqURL string) {
+	r.gsfm.Reset(reqURL)
+}
+
+// Current returns the current generator by URL
+func (r *BulkHTTPRequest) Current(reqURL string) string {
+	return r.gsfm.Current(reqURL)
+}
+
+// Total is the total number of requests
+func (r *BulkHTTPRequest) Total() int {
+	return r.gsfm.Total()
+}
+
+// Increment increments the processed request
+func (r *BulkHTTPRequest) Increment(reqURL string) {
+	r.gsfm.Increment(reqURL)
+}

v2/pkg/requests/dns-request.go

@@ -42,6 +42,11 @@ func (r *DNSRequest) SetMatchersCondition(condition matchers.ConditionType) {
 	r.matchersCondition = condition
 }
 
+// Returns the total number of requests the YAML rule will perform
+func (r *DNSRequest) GetRequestCount() int64 {
+	return 1
+}
+
 // MakeDNSRequest creates a *dns.Request from a request template
 func (r *DNSRequest) MakeDNSRequest(domain string) (*dns.Msg, error) {
 	domain = dns.Fqdn(domain)
@@ -87,6 +92,7 @@ func toQType(ttype string) (rtype uint16) {
 	default:
 		rtype = dns.TypeA
 	}
+
 	return
 }
 
@@ -110,5 +116,6 @@ func toQClass(tclass string) (rclass uint16) {
 		// Use INET by default.
 		rclass = dns.ClassINET
 	}
+
 	return
 }

v2/pkg/requests/dump.go

@@ -0,0 +1,21 @@
+package requests
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http/httputil"
+	"strings"
+
+	"github.com/projectdiscovery/rawhttp"
+)
+
+func Dump(req *HTTPRequest, reqURL string) ([]byte, error) {
+	if req.Request != nil {
+		// Create a copy on the fly of the request body - ignore errors
+		bodyBytes, _ := req.Request.BodyBytes()
+		req.Request.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes))
+		return httputil.DumpRequest(req.Request.Request, true)
+	}
+
+	return rawhttp.DumpRequestRaw(req.RawRequest.Method, reqURL, req.RawRequest.Path, ExpandMapValues(req.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(req.RawRequest.Data)))
+}

v2/pkg/requests/generator.go

@@ -0,0 +1,274 @@
+package requests
+
+import (
+	"sync"
+	"time"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/generators"
+)
+
+type GeneratorState int
+
+const (
+	fifteen                = 15
+	initial GeneratorState = iota
+	running
+	done
+)
+
+type Generator struct {
+	sync.RWMutex
+	positionPath          int
+	positionRaw           int
+	gchan                 chan map[string]interface{}
+	currentGeneratorValue map[string]interface{}
+	state                 GeneratorState
+}
+
+type GeneratorFSM struct {
+	sync.RWMutex
+	payloads     map[string]interface{}
+	basePayloads map[string][]string
+	generator    func(payloads map[string][]string) (out chan map[string]interface{})
+	Generators   map[string]*Generator
+	Type         generators.Type
+	Paths        []string
+	Raws         []string
+}
+
+func NewGeneratorFSM(typ generators.Type, payloads map[string]interface{}, paths, raws []string) *GeneratorFSM {
+	var gsfm GeneratorFSM
+	gsfm.payloads = payloads
+	gsfm.Paths = paths
+	gsfm.Raws = raws
+	gsfm.Type = typ
+
+	if len(gsfm.payloads) > 0 {
+		// load payloads if not already done
+		if gsfm.basePayloads == nil {
+			gsfm.basePayloads = generators.LoadPayloads(gsfm.payloads)
+		}
+
+		generatorFunc := generators.SniperGenerator
+
+		switch typ {
+		case generators.PitchFork:
+			generatorFunc = generators.PitchforkGenerator
+		case generators.ClusterBomb:
+			generatorFunc = generators.ClusterbombGenerator
+		case generators.Sniper:
+			generatorFunc = generators.SniperGenerator
+		}
+
+		gsfm.generator = generatorFunc
+	}
+
+	gsfm.Generators = make(map[string]*Generator)
+
+	return &gsfm
+}
+
+func (gfsm *GeneratorFSM) Add(key string) {
+	gfsm.Lock()
+	defer gfsm.Unlock()
+
+	if _, ok := gfsm.Generators[key]; !ok {
+		gfsm.Generators[key] = &Generator{state: initial}
+	}
+}
+
+func (gfsm *GeneratorFSM) Has(key string) bool {
+	gfsm.RLock()
+	defer gfsm.RUnlock()
+
+	_, ok := gfsm.Generators[key]
+
+	return ok
+}
+
+func (gfsm *GeneratorFSM) Delete(key string) {
+	gfsm.Lock()
+	defer gfsm.Unlock()
+
+	delete(gfsm.Generators, key)
+}
+
+func (gfsm *GeneratorFSM) ReadOne(key string) {
+	gfsm.RLock()
+	defer gfsm.RUnlock()
+	g, ok := gfsm.Generators[key]
+
+	if !ok {
+		return
+	}
+
+	for afterCh := time.After(fifteen * time.Second); ; {
+		select {
+		// got a value
+		case curGenValue, ok := <-g.gchan:
+			if !ok {
+				g.Lock()
+				g.gchan = nil
+				g.state = done
+				g.currentGeneratorValue = nil
+				g.Unlock()
+
+				return
+			}
+
+			g.currentGeneratorValue = curGenValue
+
+			return
+		// timeout
+		case <-afterCh:
+			g.Lock()
+			g.gchan = nil
+			g.state = done
+			g.Unlock()
+
+			return
+		}
+	}
+}
+
+func (gfsm *GeneratorFSM) InitOrSkip(key string) {
+	gfsm.RLock()
+	defer gfsm.RUnlock()
+
+	g, ok := gfsm.Generators[key]
+	if !ok {
+		return
+	}
+
+	if len(gfsm.payloads) > 0 {
+		g.Lock()
+		defer g.Unlock()
+
+		if g.gchan == nil {
+			g.gchan = gfsm.generator(gfsm.basePayloads)
+			g.state = running
+		}
+	}
+}
+
+func (gfsm *GeneratorFSM) Value(key string) map[string]interface{} {
+	gfsm.RLock()
+	defer gfsm.RUnlock()
+
+	g, ok := gfsm.Generators[key]
+	if !ok {
+		return nil
+	}
+
+	return g.currentGeneratorValue
+}
+
+func (gfsm *GeneratorFSM) Next(key string) bool {
+	gfsm.RLock()
+	defer gfsm.RUnlock()
+
+	g, ok := gfsm.Generators[key]
+	if !ok {
+		return false
+	}
+
+	if g.positionPath+g.positionRaw >= len(gfsm.Paths)+len(gfsm.Raws) {
+		return false
+	}
+
+	return true
+}
+
+func (gfsm *GeneratorFSM) Position(key string) int {
+	gfsm.RLock()
+	defer gfsm.RUnlock()
+
+	g, ok := gfsm.Generators[key]
+	if !ok {
+		return 0
+	}
+
+	return g.positionPath + g.positionRaw
+}
+
+func (gfsm *GeneratorFSM) Reset(key string) {
+	gfsm.Lock()
+	defer gfsm.Unlock()
+
+	if !gfsm.Has(key) {
+		gfsm.Add(key)
+	}
+
+	g, ok := gfsm.Generators[key]
+	if !ok {
+		return
+	}
+
+	g.positionPath = 0
+	g.positionRaw = 0
+}
+
+func (gfsm *GeneratorFSM) Current(key string) string {
+	gfsm.RLock()
+	defer gfsm.RUnlock()
+
+	g, ok := gfsm.Generators[key]
+	if !ok {
+		return ""
+	}
+
+	if g.positionPath < len(gfsm.Paths) && len(gfsm.Paths) != 0 {
+		return gfsm.Paths[g.positionPath]
+	}
+
+	return gfsm.Raws[g.positionRaw]
+}
+func (gfsm *GeneratorFSM) Total() int {
+	estimatedRequestsWithPayload := 0
+	if len(gfsm.basePayloads) > 0 {
+		switch gfsm.Type {
+		case generators.Sniper:
+			for _, kv := range gfsm.basePayloads {
+				estimatedRequestsWithPayload += len(kv)
+			}
+		case generators.PitchFork:
+			// Positional so it's equal to the length of one list
+			for _, kv := range gfsm.basePayloads {
+				estimatedRequestsWithPayload += len(kv)
+				break
+			}
+		case generators.ClusterBomb:
+			// Total of combinations => rule of product
+			prod := 1
+			for _, kv := range gfsm.basePayloads {
+				prod *= len(kv)
+			}
+			estimatedRequestsWithPayload += prod
+		}
+	}
+
+	return len(gfsm.Paths) + len(gfsm.Raws) + estimatedRequestsWithPayload
+}
+
+func (gfsm *GeneratorFSM) Increment(key string) {
+	gfsm.Lock()
+	defer gfsm.Unlock()
+
+	g, ok := gfsm.Generators[key]
+	if !ok {
+		return
+	}
+
+	if len(gfsm.Paths) > 0 && g.positionPath < len(gfsm.Paths) {
+		g.positionPath++
+		return
+	}
+
+	if len(gfsm.Raws) > 0 && g.positionRaw < len(gfsm.Raws) {
+		// if we have payloads increment only when the generators are done
+		if g.gchan == nil {
+			g.state = done
+			g.positionRaw++
+		}
+	}
+}

v2/pkg/requests/http-request.go

@@ -1,395 +0,0 @@
-package requests
-
-import (
-	"bufio"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"regexp"
-	"strings"
-
-	"github.com/Knetic/govaluate"
-	"github.com/projectdiscovery/nuclei/v2/pkg/extractors"
-	"github.com/projectdiscovery/nuclei/v2/pkg/generators"
-	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
-	retryablehttp "github.com/projectdiscovery/retryablehttp-go"
-)
-
-// HTTPRequest contains a request to be made from a template
-type HTTPRequest struct {
-	// AttackType is the attack type
-	// Sniper, PitchFork and ClusterBomb. Default is Sniper
-	AttackType string `yaml:"attack,omitempty"`
-	// attackType is internal attack type
-	attackType generators.Type
-	// Path contains the path/s for the request variables
-	Payloads map[string]string `yaml:"payloads,omitempty"`
-	// Method is the request method, whether GET, POST, PUT, etc
-	Method string `yaml:"method"`
-	// Path contains the path/s for the request
-	Path []string `yaml:"path"`
-	// Headers contains headers to send with the request
-	Headers map[string]string `yaml:"headers,omitempty"`
-	// Body is an optional parameter which contains the request body for POST methods, etc
-	Body string `yaml:"body,omitempty"`
-	// Matchers contains the detection mechanism for the request to identify
-	// whether the request was successful
-	Matchers []*matchers.Matcher `yaml:"matchers,omitempty"`
-	// MatchersCondition is the condition of the matchers
-	// whether to use AND or OR. Default is OR.
-	MatchersCondition string `yaml:"matchers-condition,omitempty"`
-	// matchersCondition is internal condition for the matchers.
-	matchersCondition matchers.ConditionType
-	// Extractors contains the extraction mechanism for the request to identify
-	// and extract parts of the response.
-	Extractors []*extractors.Extractor `yaml:"extractors,omitempty"`
-	// Redirects specifies whether redirects should be followed.
-	Redirects bool `yaml:"redirects,omitempty"`
-	// MaxRedirects is the maximum number of redirects that should be followed.
-	MaxRedirects int `yaml:"max-redirects,omitempty"`
-	// Raw contains raw requests
-	Raw []string `yaml:"raw,omitempty"`
-}
-
-// GetMatchersCondition returns the condition for the matcher
-func (r *HTTPRequest) GetMatchersCondition() matchers.ConditionType {
-	return r.matchersCondition
-}
-
-// SetMatchersCondition sets the condition for the matcher
-func (r *HTTPRequest) SetMatchersCondition(condition matchers.ConditionType) {
-	r.matchersCondition = condition
-}
-
-// GetAttackType returns the attack
-func (r *HTTPRequest) GetAttackType() generators.Type {
-	return r.attackType
-}
-
-// SetAttackType sets the attack
-func (r *HTTPRequest) SetAttackType(attack generators.Type) {
-	r.attackType = attack
-}
-
-// MakeHTTPRequest creates a *http.Request from a request configuration
-func (r *HTTPRequest) MakeHTTPRequest(baseURL string) (chan *CompiledHTTP, error) {
-	parsed, err := url.Parse(baseURL)
-	if err != nil {
-		return nil, err
-	}
-	hostname := parsed.Hostname()
-
-	values := map[string]interface{}{
-		"BaseURL":  baseURL,
-		"Hostname": hostname,
-	}
-
-	if len(r.Raw) > 0 {
-		return r.makeHTTPRequestFromRaw(baseURL, values)
-	}
-
-	return r.makeHTTPRequestFromModel(baseURL, values)
-}
-
-// MakeHTTPRequestFromModel creates a *http.Request from a request template
-func (r *HTTPRequest) makeHTTPRequestFromModel(baseURL string, values map[string]interface{}) (requests chan *CompiledHTTP, err error) {
-	requests = make(chan *CompiledHTTP)
-
-	// request generator
-	go func() {
-		defer close(requests)
-		for _, path := range r.Path {
-			// process base request
-			replacer := newReplacer(values)
-
-			// Replace the dynamic variables in the URL if any
-			URL := replacer.Replace(path)
-
-			// Build a request on the specified URL
-			req, err := http.NewRequest(r.Method, URL, nil)
-			if err != nil {
-				requests <- &CompiledHTTP{Request: nil, Error: err, Meta: nil}
-				return
-			}
-
-			request, err := r.fillRequest(req, values)
-			if err != nil {
-				requests <- &CompiledHTTP{Request: nil, Error: err, Meta: nil}
-				return
-			}
-
-			requests <- &CompiledHTTP{Request: request, Error: nil, Meta: nil}
-		}
-	}()
-
-	return
-}
-
-// makeHTTPRequestFromRaw creates a *http.Request from a raw request
-func (r *HTTPRequest) makeHTTPRequestFromRaw(baseURL string, values map[string]interface{}) (requests chan *CompiledHTTP, err error) {
-	requests = make(chan *CompiledHTTP)
-	// request generator
-	go func() {
-		defer close(requests)
-
-		for _, raw := range r.Raw {
-			// Add trailing line
-			raw += "\n"
-
-			if len(r.Payloads) > 0 {
-				basePayloads := generators.LoadWordlists(r.Payloads)
-				generatorFunc := generators.SniperGenerator
-				switch r.attackType {
-				case generators.PitchFork:
-					generatorFunc = generators.PitchforkGenerator
-				case generators.ClusterBomb:
-					generatorFunc = generators.ClusterbombGenerator
-				}
-
-				for genValues := range generatorFunc(basePayloads) {
-					compiledHTTP := r.handleRawWithPaylods(raw, baseURL, values, genValues)
-					requests <- compiledHTTP
-					if compiledHTTP.Error != nil {
-						return
-					}
-				}
-			} else {
-				// otherwise continue with normal flow
-				compiledHTTP := r.handleSimpleRaw(raw, baseURL, values)
-				requests <- compiledHTTP
-				if compiledHTTP.Error != nil {
-					return
-				}
-			}
-		}
-	}()
-
-	return requests, nil
-}
-
-func (r *HTTPRequest) handleSimpleRaw(raw string, baseURL string, values map[string]interface{}) *CompiledHTTP {
-	// base request
-	replacer := newReplacer(values)
-	// Replace the dynamic variables in the request if any
-	raw = replacer.Replace(raw)
-
-	compiledRequest, err := r.parseRawRequest(raw)
-	if err != nil {
-		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
-	}
-
-	// requests generated from http.ReadRequest have incorrect RequestURI, so they
-	// cannot be used to perform another request directly, we need to generate a new one
-	// with the new target url
-	var finalURL string
-	if compiledRequest.Path != "?" {
-		finalURL = fmt.Sprintf("%s%s", baseURL, compiledRequest.Path)
-	} else {
-		finalURL = baseURL
-	}
-	req, err := http.NewRequest(compiledRequest.Method, finalURL, strings.NewReader(compiledRequest.Data))
-	if err != nil {
-		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
-	}
-
-	// copy headers
-	for key, value := range compiledRequest.Headers {
-		req.Header.Set(key, value)
-	}
-
-	request, err := r.fillRequest(req, values)
-	if err != nil {
-		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
-	}
-
-	return &CompiledHTTP{Request: request, Error: nil, Meta: nil}
-}
-
-func (r *HTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, genValues map[string]interface{}) *CompiledHTTP {
-	baseValues := generators.CopyMap(values)
-	finValues := generators.MergeMaps(baseValues, genValues)
-
-	replacer := newReplacer(finValues)
-
-	// Replace the dynamic variables in the URL if any
-	raw = replacer.Replace(raw)
-
-	dynamicValues := make(map[string]interface{})
-	// find all potentials tokens between {{}}
-	var re = regexp.MustCompile(`(?m)\{\{.+}}`)
-	for _, match := range re.FindAllString(raw, -1) {
-		// check if the match contains a dynamic variable
-		if generators.StringContainsAnyMapItem(finValues, match) {
-			expr := generators.TrimDelimiters(match)
-			compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, generators.HelperFunctions())
-			if err != nil {
-				return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
-			}
-			result, err := compiled.Evaluate(finValues)
-			if err != nil {
-				return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
-			}
-			dynamicValues[expr] = result
-		}
-	}
-
-	// replace dynamic values
-	dynamicReplacer := newReplacer(dynamicValues)
-	raw = dynamicReplacer.Replace(raw)
-
-	compiledRequest, err := r.parseRawRequest(raw)
-	if err != nil {
-		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
-	}
-
-	// requests generated from http.ReadRequest have incorrect RequestURI, so they
-	// cannot be used to perform another request directly, we need to generate a new one
-	// with the new target url
-	var finalURL string
-	if compiledRequest.Path != "?" {
-		finalURL = fmt.Sprintf("%s%s", baseURL, compiledRequest.Path)
-	} else {
-		finalURL = baseURL
-	}
-	req, err := http.NewRequest(compiledRequest.Method, finalURL, strings.NewReader(compiledRequest.Data))
-	if err != nil {
-		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
-	}
-
-	// copy headers
-	for key, value := range compiledRequest.Headers {
-		req.Header.Set(key, value)
-	}
-
-	request, err := r.fillRequest(req, values)
-	if err != nil {
-		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
-	}
-
-	return &CompiledHTTP{Request: request, Error: nil, Meta: genValues}
-}
-
-func (r *HTTPRequest) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
-	req.Header.Set("Connection", "close")
-	req.Close = true
-
-	// raw requests are left untouched
-	if len(r.Raw) > 0 {
-		return retryablehttp.FromRequest(req)
-	}
-
-	replacer := newReplacer(values)
-	// Check if the user requested a request body
-	if r.Body != "" {
-		req.Body = ioutil.NopCloser(strings.NewReader(r.Body))
-	}
-
-	// Set the header values requested
-	for header, value := range r.Headers {
-		req.Header.Set(header, replacer.Replace(value))
-	}
-
-	// Set some headers only if the header wasn't supplied by the user
-	if _, ok := req.Header["User-Agent"]; !ok {
-		req.Header.Set("User-Agent", "Nuclei - Open-source project (github.com/projectdiscovery/nuclei)")
-	}
-
-	if _, ok := req.Header["Accept"]; !ok {
-		req.Header.Set("Accept", "*/*")
-	}
-	if _, ok := req.Header["Accept-Language"]; !ok {
-		req.Header.Set("Accept-Language", "en")
-	}
-
-	return retryablehttp.FromRequest(req)
-}
-
-// CompiledHTTP contains Generated HTTP Request or error
-type CompiledHTTP struct {
-	Request *retryablehttp.Request
-	Error   error
-	Meta    map[string]interface{}
-}
-
-// CustomHeaders valid for all requests
-type CustomHeaders []string
-
-// String returns just a label
-func (c *CustomHeaders) String() string {
-	return "Custom Global Headers"
-}
-
-// Set a new global header
-func (c *CustomHeaders) Set(value string) error {
-	*c = append(*c, value)
-	return nil
-}
-
-type compiledRawRequest struct {
-	Method  string
-	Path    string
-	Data    string
-	Headers map[string]string
-}
-
-// parseRawRequest parses the raw request as supplied by the user
-func (r *HTTPRequest) parseRawRequest(request string) (*compiledRawRequest, error) {
-	reader := bufio.NewReader(strings.NewReader(request))
-
-	rawRequest := compiledRawRequest{
-		Headers: make(map[string]string),
-	}
-
-	s, err := reader.ReadString('\n')
-	if err != nil {
-		return nil, fmt.Errorf("could not read request: %s", err)
-	}
-	parts := strings.Split(s, " ")
-	if len(parts) < 3 {
-		return nil, fmt.Errorf("malformed request supplied")
-	}
-	// Set the request Method
-	rawRequest.Method = parts[0]
-
-	for {
-		line, err := reader.ReadString('\n')
-		line = strings.TrimSpace(line)
-
-		if err != nil || line == "" {
-			break
-		}
-
-		p := strings.SplitN(line, ":", 2)
-		if len(p) != 2 {
-			continue
-		}
-
-		if strings.EqualFold(p[0], "content-length") {
-			continue
-		}
-
-		rawRequest.Headers[strings.TrimSpace(p[0])] = strings.TrimSpace(p[1])
-	}
-
-	// Handle case with the full http url in path. In that case,
-	// ignore any host header that we encounter and use the path as request URL
-	if strings.HasPrefix(parts[1], "http") {
-		parsed, err := url.Parse(parts[1])
-		if err != nil {
-			return nil, fmt.Errorf("could not parse request URL: %s", err)
-		}
-		rawRequest.Path = parts[1]
-		rawRequest.Headers["Host"] = parsed.Host
-	} else {
-		rawRequest.Path = parts[1]
-	}
-
-	// Set the request body
-	b, err := ioutil.ReadAll(reader)
-	if err != nil {
-		return nil, fmt.Errorf("could not read request body: %s", err)
-	}
-	rawRequest.Data = string(b)
-	return &rawRequest, nil
-}

v2/pkg/requests/util.go

@@ -6,24 +6,37 @@ import (
 	"fmt"
 	"io/ioutil"
 	"strings"
+)
 
-	"github.com/projectdiscovery/retryablehttp-go"
+const (
+	markerParenthesisOpen  = "{{"
+	markerParenthesisClose = "}}"
+	markerGeneral          = "§"
 )
 
 func newReplacer(values map[string]interface{}) *strings.Replacer {
 	var replacerItems []string
-	for k, v := range values {
-		replacerItems = append(replacerItems, fmt.Sprintf("{{%s}}", k))
-		replacerItems = append(replacerItems, fmt.Sprintf("%s", v))
+	for key, val := range values {
+		replacerItems = append(
+			replacerItems,
+			fmt.Sprintf("%s%s%s", markerParenthesisOpen, key, markerParenthesisClose),
+			fmt.Sprintf("%s", val),
+			fmt.Sprintf("%s%s%s", markerGeneral, key, markerGeneral),
+			fmt.Sprintf("%s", val),
+		)
 	}
 
 	return strings.NewReplacer(replacerItems...)
 }
 
 // HandleDecompression if the user specified a custom encoding (as golang transport doesn't do this automatically)
-func HandleDecompression(r *retryablehttp.Request, bodyOrig []byte) (bodyDec []byte, err error) {
-	encodingHeader := strings.ToLower(r.Header.Get("Accept-Encoding"))
-	if encodingHeader == "gzip" {
+func HandleDecompression(r *HTTPRequest, bodyOrig []byte) (bodyDec []byte, err error) {
+	if r.Request == nil {
+		return bodyOrig, nil
+	}
+
+	encodingHeader := strings.TrimSpace(strings.ToLower(r.Request.Header.Get("Accept-Encoding")))
+	if encodingHeader == "gzip" || encodingHeader == "gzip, deflate" {
 		gzipreader, err := gzip.NewReader(bytes.NewReader(bodyOrig))
 		if err != nil {
 			return bodyDec, err
@@ -40,3 +53,21 @@ func HandleDecompression(r *retryablehttp.Request, bodyOrig []byte) (bodyDec []b
 
 	return bodyOrig, nil
 }
+
+// ZipMapValues converts values from strings slices to flat string
+func ZipMapValues(m map[string][]string) (m1 map[string]string) {
+	m1 = make(map[string]string)
+	for k, v := range m {
+		m1[k] = strings.Join(v, "")
+	}
+	return
+}
+
+// ExpandMapValues converts values from flat string to strings slice
+func ExpandMapValues(m map[string]string) (m1 map[string][]string) {
+	m1 = make(map[string][]string)
+	for k, v := range m {
+		m1[k] = []string{v}
+	}
+	return
+}

v2/pkg/syncedreadcloser/syncedreadcloser.go

@@ -0,0 +1,97 @@
+package syncedreadcloser
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"time"
+)
+
+// compatible with ReadSeeker
+type SyncedReadCloser struct {
+	data           []byte
+	p              int64
+	length         int64
+	opengate       chan struct{}
+	enableBlocking bool
+}
+
+func New(r io.ReadCloser) *SyncedReadCloser {
+	var (
+		s   SyncedReadCloser
+		err error
+	)
+	s.data, err = ioutil.ReadAll(r)
+	if err != nil {
+		return nil
+	}
+	r.Close()
+	s.length = int64(len(s.data))
+	s.opengate = make(chan struct{})
+	s.enableBlocking = true
+
+	return &s
+}
+
+func NewOpenGateWithTimeout(r io.ReadCloser, d time.Duration) *SyncedReadCloser {
+	s := New(r)
+	s.OpenGateAfter(d)
+
+	return s
+}
+
+func (s *SyncedReadCloser) SetOpenGate(status bool) {
+	s.enableBlocking = status
+}
+
+func (s *SyncedReadCloser) OpenGate() {
+	s.opengate <- struct{}{}
+}
+
+func (s *SyncedReadCloser) OpenGateAfter(d time.Duration) {
+	time.AfterFunc(d, func() {
+		s.opengate <- struct{}{}
+	})
+}
+
+func (s *SyncedReadCloser) Seek(offset int64, whence int) (int64, error) {
+	var err error
+	switch whence {
+	case io.SeekStart:
+		s.p = 0
+	case io.SeekCurrent:
+		if s.p+offset < s.length {
+			s.p += offset
+			break
+		}
+		err = fmt.Errorf("offset is too big")
+	case io.SeekEnd:
+		if s.length-offset >= 0 {
+			s.p = s.length - offset
+			break
+		}
+		err = fmt.Errorf("offset is too big")
+	}
+	return s.p, err
+}
+
+func (s *SyncedReadCloser) Read(p []byte) (n int, err error) {
+	// If the data fits in the buffer blocks awaiting the sync instruction
+	if s.p+int64(len(p)) >= s.length && s.enableBlocking {
+		<-s.opengate
+	}
+	n = copy(p, s.data[s.p:])
+	s.p += int64(n)
+	if s.p == s.length {
+		err = io.EOF
+	}
+	return n, err
+}
+
+func (s *SyncedReadCloser) Close() error {
+	return nil
+}
+
+func (s *SyncedReadCloser) Len() int {
+	return int(s.length)
+}

v2/pkg/templates/compile.go

@@ -1,9 +1,10 @@
 package templates
 
 import (
-	"errors"
 	"fmt"
 	"os"
+	"path"
+	"strings"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/generators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
@@ -25,13 +26,15 @@ func Parse(file string) (*Template, error) {
 	}
 	defer f.Close()
 
+	template.path = file
+
 	// If no requests, and it is also not a workflow, return error.
-	if len(template.RequestsHTTP)+len(template.RequestsDNS) <= 0 {
-		return nil, errors.New("No requests defined")
+	if len(template.BulkRequestsHTTP)+len(template.RequestsDNS) <= 0 {
+		return nil, fmt.Errorf("no requests defined for %s", template.ID)
 	}
 
 	// Compile the matchers and the extractors for http requests
-	for _, request := range template.RequestsHTTP {
+	for _, request := range template.BulkRequestsHTTP {
 		// Get the condition between the matchers
 		condition, ok := matchers.ConditionTypes[request.MatchersCondition]
 		if !ok {
@@ -49,23 +52,56 @@ func Parse(file string) (*Template, error) {
 		}
 
 		// Validate the payloads if any
-		for name, wordlist := range request.Payloads {
-			if !generators.FileExists(wordlist) {
-				return nil, fmt.Errorf("The %s file for payload %s does not exist", wordlist, name)
+		for name, payload := range request.Payloads {
+			switch pt := payload.(type) {
+			case string:
+				// check if it's a multiline string list
+				if len(strings.Split(pt, "\n")) <= 1 {
+					// check if it's a worldlist file
+					if !generators.FileExists(pt) {
+						// attempt to load the file by taking the full path, tokezining it and searching the template in such paths
+						changed := false
+						pathTokens := strings.Split(template.path, "/")
+
+						for i := range pathTokens {
+							tpath := path.Join(strings.Join(pathTokens[:i], "/"), pt)
+							if generators.FileExists(tpath) {
+								request.Payloads[name] = tpath
+								changed = true
+
+								break
+							}
+						}
+
+						if !changed {
+							return nil, fmt.Errorf("the %s file for payload %s does not exist or does not contain enough elements", pt, name)
+						}
+					}
+				}
+			case []string, []interface{}:
+				if len(payload.([]interface{})) == 0 {
+					return nil, fmt.Errorf("the payload %s does not contain enough elements", name)
+				}
+			default:
+				return nil, fmt.Errorf("the payload %s has invalid type", name)
 			}
 		}
 
 		for _, matcher := range request.Matchers {
-			if err = matcher.CompileMatchers(); err != nil {
-				return nil, err
+			matchErr := matcher.CompileMatchers()
+			if matchErr != nil {
+				return nil, matchErr
 			}
 		}
 
 		for _, extractor := range request.Extractors {
-			if err := extractor.CompileExtractors(); err != nil {
-				return nil, err
+			extractErr := extractor.CompileExtractors()
+			if extractErr != nil {
+				return nil, extractErr
 			}
 		}
+
+		request.InitGenerator()
 	}
 
 	// Compile the matchers and the extractors for dns requests
@@ -79,13 +115,15 @@ func Parse(file string) (*Template, error) {
 		}
 
 		for _, matcher := range request.Matchers {
-			if err = matcher.CompileMatchers(); err != nil {
+			err = matcher.CompileMatchers()
+			if err != nil {
 				return nil, err
 			}
 		}
 
 		for _, extractor := range request.Extractors {
-			if err := extractor.CompileExtractors(); err != nil {
+			err := extractor.CompileExtractors()
+			if err != nil {
 				return nil, err
 			}
 		}

v2/pkg/templates/templates.go

@@ -9,19 +9,33 @@ type Template struct {
 	// ID is the unique id for the template
 	ID string `yaml:"id"`
 	// Info contains information about the template
-	Info Info `yaml:"info"`
-	// RequestHTTP contains the http request to make in the template
-	RequestsHTTP []*requests.HTTPRequest `yaml:"requests,omitempty"`
-	// RequestDNS contains the dns request to make in the template
+	Info map[string]string `yaml:"info"`
+	// BulkRequestsHTTP contains the http request to make in the template
+	BulkRequestsHTTP []*requests.BulkHTTPRequest `yaml:"requests,omitempty"`
+	// RequestsDNS contains the dns request to make in the template
 	RequestsDNS []*requests.DNSRequest `yaml:"dns,omitempty"`
+	path        string
 }
 
-// Info contains information about the request template
-type Info struct {
-	// Name is the name of the template
-	Name string `yaml:"name"`
-	// Author is the name of the author of the template
-	Author string `yaml:"author"`
-	// Severity optionally describes the severity of the template
-	Severity string `yaml:"severity,omitempty"`
+// GetPath of the workflow
+func (t *Template) GetPath() string {
+	return t.path
+}
+
+func (t *Template) GetHTTPRequestCount() int64 {
+	var count int64 = 0
+	for _, request := range t.BulkRequestsHTTP {
+		count += request.GetRequestCount()
+	}
+
+	return count
+}
+
+func (t *Template) GetDNSRequestCount() int64 {
+	var count int64 = 0
+	for _, request := range t.RequestsDNS {
+		count += request.GetRequestCount()
+	}
+
+	return count
 }

v2/pkg/workflows/compile.go

@@ -1,9 +1,9 @@
 package workflows
 
 import (
-	"errors"
 	"os"
 
+	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 )
 
@@ -22,9 +22,14 @@ func Parse(file string) (*Workflow, error) {
 		return nil, err
 	}
 
-	if workflow.Logic == "" {
-		return nil, errors.New("No logic provided")
+	if len(workflow.Workflows) > 0 {
+		if err := workflow.generateLogicFromWorkflows(); err != nil {
+			return nil, errors.Wrap(err, "could not generate workflow")
+		}
 	}
-
+	if workflow.Logic == "" {
+		return nil, errors.New("no logic provided")
+	}
+	workflow.path = file
 	return workflow, nil
 }

v2/pkg/workflows/generator.go

@@ -0,0 +1,78 @@
+package workflows
+
+import (
+	"errors"
+	"strings"
+
+	"github.com/segmentio/ksuid"
+)
+
+// generateLogicFromWorkflows generates a workflow logic using the
+// yaml based workflow declaration.
+//
+// The implementation is very basic and contains a simple yaml->tengo
+// convertor that implements basic required features.
+func (w *Workflow) generateLogicFromWorkflows() error {
+	w.Variables = make(map[string]string)
+
+	workflowBuilder := &strings.Builder{}
+	for _, template := range w.Workflows {
+		if err := w.generateTemplateFunc(template, workflowBuilder); err != nil {
+			return err
+		}
+	}
+	w.Logic = workflowBuilder.String()
+	return nil
+}
+
+func (w *Workflow) generateTemplateFunc(template *WorkflowTemplate, workflowBuilder *strings.Builder) error {
+	builder := &strings.Builder{}
+	builder.WriteString("var_")
+	builder.WriteString(ksuid.New().String())
+	ID := builder.String()
+	w.Variables[ID] = template.Template
+
+	if len(template.Subtemplates) > 0 && len(template.Matchers) > 0 {
+		return errors.New("subtemplates and matchers cannot be present together")
+	}
+	workflowBuilder.WriteRune('\n')
+	if len(template.Matchers) > 0 {
+		workflowBuilder.WriteString(ID)
+		workflowBuilder.WriteString("()\n")
+
+		for _, matcher := range template.Matchers {
+			if len(matcher.Subtemplates) == 0 {
+				return errors.New("no subtemplates present for matcher")
+			}
+			workflowBuilder.WriteString("\nif ")
+			workflowBuilder.WriteString(ID)
+			workflowBuilder.WriteString("[\"")
+			workflowBuilder.WriteString(matcher.Name)
+			workflowBuilder.WriteString("\"] {")
+
+			for _, subtemplate := range matcher.Subtemplates {
+				if err := w.generateTemplateFunc(subtemplate, workflowBuilder); err != nil {
+					return err
+				}
+			}
+			workflowBuilder.WriteString("\n}")
+		}
+	}
+	if len(template.Subtemplates) > 0 {
+		workflowBuilder.WriteString("if ")
+		workflowBuilder.WriteString(ID)
+		workflowBuilder.WriteString("() {")
+
+		for _, subtemplate := range template.Subtemplates {
+			if err := w.generateTemplateFunc(subtemplate, workflowBuilder); err != nil {
+				return err
+			}
+		}
+		workflowBuilder.WriteString("\n}")
+	}
+	if len(template.Matchers) == 0 && len(template.Subtemplates) == 0 {
+		workflowBuilder.WriteString(ID)
+		workflowBuilder.WriteString("();")
+	}
+	return nil
+}

v2/pkg/workflows/var.go

@@ -1,22 +1,34 @@
 package workflows
 
 import (
-	"github.com/d5/tengo/v2"
+	"sync"
+
+	tengo "github.com/d5/tengo/v2"
+	"github.com/logrusorgru/aurora"
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/executor"
+	"github.com/projectdiscovery/nuclei/v2/internal/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
+	"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/executer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/generators"
 )
 
+const two = 2
+
 // NucleiVar within the scripting engine
 type NucleiVar struct {
 	tengo.ObjectImpl
-	Templates []*Template
-	URL       string
+	Templates    []*Template
+	URL          string
+	InternalVars map[string]interface{}
+	sync.RWMutex
 }
 
 // Template contains HTTPOptions and DNSOptions for a single template
 type Template struct {
-	HTTPOptions *executor.HTTPOptions
-	DNSOptions  *executor.DNSOptions
+	HTTPOptions *executer.HTTPOptions
+	DNSOptions  *executer.DNSOptions
+	Progress    *progress.Progress
 }
 
 // TypeName of the variable
@@ -29,47 +41,194 @@ func (n *NucleiVar) CanCall() bool {
 	return true
 }
 
-// Call logic - actually it doesn't require arguments
+// Call logic - args[0]=headers, args[1]=payloads
 func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
-	var gotResult bool
+	n.InternalVars = make(map[string]interface{})
+	headers := make(map[string]string)
+	externalVars := make(map[string]interface{})
+
+	// if external variables are specified and matches the template ones, these gets overwritten
+	if len(args) >= 1 {
+		headers = iterableToMapString(args[0])
+	}
+
+	// if external variables are specified and matches the template ones, these gets overwritten
+	if len(args) >= two {
+		externalVars = iterableToMap(args[1])
+	}
+
+	var gotResult atomicboolean.AtomBool
 
 	for _, template := range n.Templates {
+		p := template.Progress
+
 		if template.HTTPOptions != nil {
-			for _, request := range template.HTTPOptions.Template.RequestsHTTP {
-				template.HTTPOptions.HTTPRequest = request
-				httpExecutor, err := executor.NewHTTPExecutor(template.HTTPOptions)
+			p.AddToTotal(template.HTTPOptions.Template.GetHTTPRequestCount())
+
+			for _, request := range template.HTTPOptions.Template.BulkRequestsHTTP {
+				// apply externally supplied payloads if any
+				request.Headers = generators.MergeMapsWithStrings(request.Headers, headers)
+				// apply externally supplied payloads if any
+				request.Payloads = generators.MergeMaps(request.Payloads, externalVars)
+
+				template.HTTPOptions.BulkHTTPRequest = request
+
+				if template.HTTPOptions.Colorizer == nil {
+					template.HTTPOptions.Colorizer = colorizer.NewNucleiColorizer(aurora.NewAurora(true))
+				}
+
+				httpExecuter, err := executer.NewHTTPExecuter(template.HTTPOptions)
+
 				if err != nil {
+					p.Drop(request.GetRequestCount())
 					gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, err)
+
 					continue
 				}
-				err = httpExecutor.ExecuteHTTP(n.URL)
-				if err != nil {
-					gologger.Warningf("Could not send request for template '%s': %s\n", template.HTTPOptions.Template.ID, err)
+
+				result := httpExecuter.ExecuteHTTP(p, n.URL)
+
+				if result.Error != nil {
+					gologger.Warningf("Could not send request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error)
 					continue
 				}
-				if httpExecutor.GotResults() {
-					gotResult = true
+
+				if result.GotResults {
+					gotResult.Or(result.GotResults)
+					n.addResults(result)
 				}
 			}
 		}
 
 		if template.DNSOptions != nil {
+			p.AddToTotal(template.DNSOptions.Template.GetDNSRequestCount())
+
 			for _, request := range template.DNSOptions.Template.RequestsDNS {
 				template.DNSOptions.DNSRequest = request
-				dnsExecutor := executor.NewDNSExecutor(template.DNSOptions)
-				err = dnsExecutor.ExecuteDNS(n.URL)
-				if err != nil {
-					gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, err)
+				dnsExecuter := executer.NewDNSExecuter(template.DNSOptions)
+				result := dnsExecuter.ExecuteDNS(p, n.URL)
+
+				if result.Error != nil {
+					gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error)
 					continue
 				}
-				if dnsExecutor.GotResults() {
-					gotResult = true
+
+				if result.GotResults {
+					gotResult.Or(result.GotResults)
+					n.addResults(result)
 				}
 			}
 		}
 	}
-	if gotResult {
+
+	if gotResult.Get() {
 		return tengo.TrueValue, nil
 	}
+
 	return tengo.FalseValue, nil
 }
+
+func (n *NucleiVar) IsFalsy() bool {
+	n.RLock()
+	defer n.RUnlock()
+
+	return len(n.InternalVars) == 0
+}
+
+func (n *NucleiVar) addResults(r *executer.Result) {
+	n.RLock()
+	defer n.RUnlock()
+
+	// add payload values as first, they will be accessible if not overwritter through
+	// payload_name (from template) => value
+	for k, v := range r.Meta {
+		n.InternalVars[k] = v
+	}
+
+	for k := range r.Matches {
+		n.InternalVars[k] = true
+	}
+
+	for k, v := range r.Extractions {
+		n.InternalVars[k] = v
+	}
+}
+
+// IndexGet returns the value for the given key.
+func (n *NucleiVar) IndexGet(index tengo.Object) (res tengo.Object, err error) {
+	strIdx, ok := tengo.ToString(index)
+	if !ok {
+		err = tengo.ErrInvalidIndexType
+		return
+	}
+
+	r, ok := n.InternalVars[strIdx]
+	if !ok {
+		return tengo.UndefinedValue, nil
+	}
+
+	switch rt := r.(type) {
+	case bool:
+		if rt {
+			res = tengo.TrueValue
+		} else {
+			res = tengo.FalseValue
+		}
+	case string:
+		res = &tengo.String{Value: rt}
+	case []string:
+		rr, ok := r.([]string)
+		if !ok {
+			break
+		}
+
+		var resA []tengo.Object
+
+		for _, rrr := range rr {
+			resA = append(resA, &tengo.String{Value: rrr})
+		}
+
+		res = &tengo.Array{Value: resA}
+	}
+
+	return res, nil
+}
+
+func iterableToMap(t tengo.Object) map[string]interface{} {
+	m := make(map[string]interface{})
+
+	if t.CanIterate() {
+		i := t.Iterate()
+		for i.Next() {
+			key, ok := tengo.ToString(i.Key())
+			if !ok {
+				continue
+			}
+
+			value := tengo.ToInterface(i.Value())
+			m[key] = value
+		}
+	}
+
+	return m
+}
+
+func iterableToMapString(t tengo.Object) map[string]string {
+	m := make(map[string]string)
+
+	if t.CanIterate() {
+		i := t.Iterate()
+		for i.Next() {
+			key, ok := tengo.ToString(i.Key())
+			if !ok {
+				continue
+			}
+
+			if value, ok := tengo.ToString(i.Value()); ok {
+				m[key] = value
+			}
+		}
+	}
+
+	return m
+}

v2/pkg/workflows/workflows.go

@@ -5,19 +5,32 @@ type Workflow struct {
 	// ID is the unique id for the template
 	ID string `yaml:"id"`
 	// Info contains information about the template
-	Info Info `yaml:"info"`
+	Info map[string]string `yaml:"info"`
+	// CookieReuse makes all cookies shared by templates within the workflow
+	CookieReuse bool `yaml:"cookie-reuse,omitempty"`
 	// Variables contains the variables accessible to the pseudo-code
 	Variables map[string]string `yaml:"variables"`
 	// Logic contains the workflow pseudo-code
 	Logic string `yaml:"logic"`
+	// Workflows is a yaml based workflow declaration code.
+	Workflows []*WorkflowTemplate `yaml:"workflows"`
+	path      string
 }
 
-// Info contains information about workflow
-type Info struct {
-	// Name is the name of the workflow
-	Name string `yaml:"name"`
-	// Author is the name of the author of the workflow
-	Author string `yaml:"author"`
-	// Severity optionally describes the severity of the template
-	Severity string `yaml:"severity,omitempty"`
+// WorkflowTemplate is a template to be ran as part of a workflow
+type WorkflowTemplate struct {
+	Template     string              `yaml:"template"`
+	Matchers     []*Matcher          `yaml:"matchers"`
+	Subtemplates []*WorkflowTemplate `yaml:"subtemplates"`
+}
+
+// Matcher performs conditional matching on the workflow template results.
+type Matcher struct {
+	Name         string              `yaml:"name"`
+	Subtemplates []*WorkflowTemplate `yaml:"subtemplates"`
+}
+
+// GetPath of the workflow
+func (w *Workflow) GetPath() string {
+	return w.path
 }