Merge pull request #691 from projectdiscovery/dev

Minor bugfix release

.github/workflows/build.yaml

@@ -13,7 +13,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v2.5.2
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
           version: v1.33

README.md

@@ -22,8 +22,8 @@
   <a href="#for-developers-and-organisations">For Developers</a> •
   <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">Documentation</a> •
   <a href="#credits">Credits</a> •
-  <a href="#license">License</a> •
-  <a href="https://discord.gg/KECAGdH">Join Discord</a>
+  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">FAQs</a> •
+  <a href="https://discord.gg/projectdiscovery">Join Discord</a>
 </p>
 
 ---

v2/Makefile

@@ -0,0 +1,14 @@
+# Go parameters
+GOCMD=go
+GOBUILD=$(GOCMD) build
+GOMOD=$(GOCMD) mod
+GOTEST=$(GOCMD) test
+GOGET=$(GOCMD) get
+    
+all: build
+build:
+		$(GOBUILD) -v -ldflags="-extldflags=-static" -o "nuclei" cmd/nuclei/main.go
+test: 
+		$(GOTEST) -v ./...
+tidy:
+		$(GOMOD) tidy

v2/cmd/nuclei/main.go

@@ -52,7 +52,6 @@ based on templates offering massive extensibility and ease of use.`)
 	set.BoolVarP(&options.NoColor, "no-color", "nc", false, "Disable colors in output")
 	set.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout")
 	set.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
-	set.BoolVarP(&options.RandomAgent, "random-agent", "ra", false, "Use randomly selected HTTP User-Agent header value")
 	set.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "Custom Header.")
 	set.BoolVar(&options.Debug, "debug", false, "Debugging request and responses")
 	set.BoolVar(&options.DebugRequests, "debug-req", false, "Debugging request")

v2/internal/runner/banner.go

@@ -7,11 +7,11 @@ const banner = `
      ____  __  _______/ /__  (_)
     / __ \/ / / / ___/ / _ \/ /
    / / / / /_/ / /__/ /  __/ /
-  /_/ /_/\__,_/\___/_/\___/_/   v2.3.2
+  /_/ /_/\__,_/\___/_/\___/_/   v2.3.5
 `
 
 // Version is the current version of nuclei
-const Version = `2.3.2`
+const Version = `2.3.5`
 
 // showBanner is used to show the banner to the user
 func showBanner() {

v2/internal/runner/config.go

@@ -1,14 +1,14 @@
 package runner
 
 import (
-	"bufio"
 	"os"
 	"path"
 	"regexp"
-	"strings"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
+	"github.com/projectdiscovery/gologger"
+	"gopkg.in/yaml.v2"
 )
 
 // nucleiConfig contains some configuration options for nuclei
@@ -83,25 +83,27 @@ func (r *Runner) writeConfiguration(config *nucleiConfig) error {
 
 const nucleiIgnoreFile = ".nuclei-ignore"
 
+type ignoreFile struct {
+	Tags  []string `yaml:"tags"`
+	Files []string `yaml:"files"`
+}
+
 // readNucleiIgnoreFile reads the nuclei ignore file marking it in map
 func (r *Runner) readNucleiIgnoreFile() {
 	file, err := os.Open(r.getIgnoreFilePath())
 	if err != nil {
+		gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err)
 		return
 	}
 	defer file.Close()
 
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		text := scanner.Text()
-		if text == "" {
-			continue
-		}
-		if strings.HasPrefix(text, "#") {
-			continue
-		}
-		r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, text)
+	ignore := &ignoreFile{}
+	if err := yaml.NewDecoder(file).Decode(ignore); err != nil {
+		gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err)
+		return
 	}
+	r.options.ExcludeTags = append(r.options.ExcludeTags, ignore.Tags...)
+	r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, ignore.Files...)
 }
 
 // getIgnoreFilePath returns the ignore file path for the runner
@@ -114,17 +116,12 @@ func (r *Runner) getIgnoreFilePath() string {
 		_ = os.MkdirAll(configDir, os.ModePerm)
 
 		defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile)
+		return defIgnoreFilePath
 	}
-
 	cwd, err := os.Getwd()
 	if err != nil {
 		return defIgnoreFilePath
 	}
 	cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile)
-
-	cwdIfpInfo, err := os.Stat(cwdIgnoreFilePath)
-	if os.IsNotExist(err) || cwdIfpInfo.IsDir() {
-		return defIgnoreFilePath
-	}
 	return cwdIgnoreFilePath
 }

v2/internal/runner/options.go

@@ -25,7 +25,6 @@ func ParseOptions(options *types.Options) {
 	// Show the user the banner
 	showBanner()
 
-	options.ExcludeTags = append(options.ExcludeTags, "dos")
 	if options.Version {
 		gologger.Info().Msgf("Current Version: %s\n", Version)
 		os.Exit(0)

v2/internal/runner/runner.go

@@ -18,6 +18,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
@@ -67,6 +68,7 @@ func New(options *types.Options) (*Runner, error) {
 		runner.readNucleiIgnoreFile()
 	}
 	runner.catalog = catalog.New(runner.options.TemplatesDirectory)
+	runner.catalog.AppendIgnore(runner.templatesConfig.IgnorePaths)
 
 	var reportingOptions *reporting.Options
 	if options.ReportingConfig != "" {
@@ -217,6 +219,7 @@ func (r *Runner) Close() {
 	if r.projectFile != nil {
 		r.projectFile.Close()
 	}
+	protocolinit.Close()
 }
 
 // RunEnumeration sets up the input layer for giving input nuclei.
@@ -235,8 +238,8 @@ func (r *Runner) RunEnumeration() {
 		}
 		r.options.Templates = append(r.options.Templates, templatesLoaded...)
 	}
-	includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates)
-	excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates)
+	includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates, false)
+	excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates, true)
 	// defaults to all templates
 	allTemplates := includedTemplates
 
@@ -260,7 +263,7 @@ func (r *Runner) RunEnumeration() {
 	// pre-parse all the templates, apply filters
 	finalTemplates := []*templates.Template{}
 
-	workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows)
+	workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows, false)
 	availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false)
 	availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true)
 

v2/internal/runner/update.go

@@ -44,7 +44,7 @@ func (r *Runner) updateTemplates() error {
 	configDir := path.Join(home, "/.config", "/nuclei")
 	_ = os.MkdirAll(configDir, os.ModePerm)
 
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
+	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
 	if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
 		config, readErr := readConfiguration()
 		if err != nil {
@@ -65,6 +65,7 @@ func (r *Runner) updateTemplates() error {
 		}
 		r.templatesConfig = currentConfig
 	}
+
 	// Check if last checked for nuclei-ignore is more than 1 hours.
 	// and if true, run the check.
 	if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
@@ -316,7 +317,7 @@ func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults
 		paths := strings.Split(directory, "/")
 		finalPath := strings.Join(paths[1:], "/")
 
-		if (!strings.EqualFold(name, ".nuclei-ignore") && strings.HasPrefix(name, ".")) || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") {
+		if strings.HasPrefix(name, ".") || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") {
 			continue
 		}
 		results.totalCount++

v2/internal/testutils/testutils.go

@@ -19,7 +19,6 @@ func Init(options *types.Options) {
 
 // DefaultOptions is the default options structure for nuclei during mocking.
 var DefaultOptions = &types.Options{
-	RandomAgent:          false,
 	Metrics:              false,
 	Debug:                false,
 	DebugRequests:        false,

v2/pkg/catalog/catalogue.go

@@ -9,6 +9,10 @@ type Catalog struct {
 // New creates a new Catalog structure using provided input items
 func New(directory string) *Catalog {
 	catalog := &Catalog{templatesDirectory: directory}
-	catalog.readNucleiIgnoreFile()
 	return catalog
 }
+
+// AppendIgnore appends to the catalog store ignore list.
+func (c *Catalog) AppendIgnore(list []string) {
+	c.ignoreFiles = append(c.ignoreFiles, list...)
+}

v2/pkg/catalog/find.go

@@ -12,7 +12,7 @@ import (
 )
 
 // GetTemplatesPath returns a list of absolute paths for the provided template list.
-func (c *Catalog) GetTemplatesPath(definitions []string) []string {
+func (c *Catalog) GetTemplatesPath(definitions []string, noCheckIgnore bool) []string {
 	// keeps track of processed dirs and files
 	processed := make(map[string]bool)
 	allTemplates := []string{}
@@ -23,6 +23,9 @@ func (c *Catalog) GetTemplatesPath(definitions []string) []string {
 			gologger.Error().Msgf("Could not find template '%s': %s\n", t, err)
 		}
 		for _, path := range paths {
+			if !noCheckIgnore && c.checkIfInNucleiIgnore(path) {
+				continue
+			}
 			if _, ok := processed[path]; !ok {
 				processed[path] = true
 				allTemplates = append(allTemplates, path)
@@ -139,10 +142,6 @@ func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]stru
 		},
 		Callback: func(path string, d *godirwalk.Dirent) error {
 			if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
-				if c.checkIfInNucleiIgnore(path) {
-					return nil
-				}
-
 				if _, ok := processed[path]; !ok {
 					results = append(results, path)
 					processed[path] = struct{}{}

v2/pkg/catalog/ignore.go

@@ -1,37 +1,11 @@
 package catalog
 
 import (
-	"bufio"
-	"os"
-	"path"
 	"strings"
 
 	"github.com/projectdiscovery/gologger"
 )
 
-const nucleiIgnoreFile = ".nuclei-ignore"
-
-// readNucleiIgnoreFile reads the nuclei ignore file marking it in map
-func (c *Catalog) readNucleiIgnoreFile() {
-	file, err := os.Open(path.Join(c.templatesDirectory, nucleiIgnoreFile))
-	if err != nil {
-		return
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		text := scanner.Text()
-		if text == "" {
-			continue
-		}
-		if strings.HasPrefix(text, "#") {
-			continue
-		}
-		c.ignoreFiles = append(c.ignoreFiles, text)
-	}
-}
-
 // checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules.
 func (c *Catalog) checkIfInNucleiIgnore(item string) bool {
 	if c.templatesDirectory == "" {
@@ -51,7 +25,7 @@ func (c *Catalog) checkIfInNucleiIgnore(item string) bool {
 		}
 	}
 	if matched {
-		gologger.Error().Msgf("Excluding %s due to nuclei-ignore filter", item)
+		gologger.Warning().Msgf("Excluding %s due to nuclei-ignore filter", item)
 		return true
 	}
 	return false

v2/pkg/protocols/common/protocolinit/init.go

@@ -1,6 +1,8 @@
 package protocolinit
 
 import (
+	"github.com/corpix/uarand"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
@@ -9,6 +11,11 @@ import (
 
 // Init initializes the client pools for the protocols
 func Init(options *types.Options) error {
+	uarand.Default = uarand.NewWithCustomList(userAgents)
+
+	if err := protocolstate.Init(options); err != nil {
+		return err
+	}
 	if err := dnsclientpool.Init(options); err != nil {
 		return err
 	}
@@ -20,3 +27,42 @@ func Init(options *types.Options) error {
 	}
 	return nil
 }
+
+var userAgents = []string{
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36",
+	"Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2919.83 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2866.71 Safari/537.36",
+	"Mozilla/5.0 (X11; Ubuntu; Linux i686 on x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2820.59 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2762.73 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2656.18 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/44.0.2403.155 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36",
+	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
+	"Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36",
+	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F",
+}
+
+func Close() {
+	protocolstate.Dialer.Close()
+}

v2/pkg/protocols/common/protocolstate/state.go

@@ -0,0 +1,31 @@
+package protocolstate
+
+import (
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+var Dialer *fastdialer.Dialer
+
+func Init(options *types.Options) error {
+	opts := fastdialer.DefaultOptions
+	if options.SystemResolvers {
+		opts.EnableFallback = true
+	}
+	if options.ResolversFile != "" {
+		opts.BaseResolvers = options.InternalResolversList
+	}
+	dialer, err := fastdialer.NewDialer(opts)
+	if err != nil {
+		return errors.Wrap(err, "could not create dialer")
+	}
+	Dialer = dialer
+	return nil
+}
+
+func Close() {
+	if Dialer != nil {
+		Dialer.Close()
+	}
+}

v2/pkg/protocols/headless/engine/engine.go

@@ -73,13 +73,10 @@ func New(options *types.Options) (*Browser, error) {
 			customAgent = parts[1]
 		}
 	}
-	if options.RandomAgent {
+	if customAgent == "" {
 		customAgent = uarand.GetRandom()
 	}
-	httpclient, err := newhttpClient(options)
-	if err != nil {
-		return nil, err
-	}
+	httpclient := newhttpClient(options)
 	engine := &Browser{
 		tempDir:     dataStore,
 		customAgent: customAgent,

v2/pkg/protocols/headless/engine/http_client.go

@@ -5,25 +5,13 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
 // newhttpClient creates a new http client for headless communication with a timeout
-func newhttpClient(options *types.Options) (*http.Client, error) {
-	opts := fastdialer.DefaultOptions
-	if options.SystemResolvers {
-		opts.EnableFallback = true
-	}
-	if options.ResolversFile != "" {
-		opts.BaseResolvers = options.InternalResolversList
-	}
-	dialer, err := fastdialer.NewDialer(opts)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not create dialer")
-	}
-
+func newhttpClient(options *types.Options) *http.Client {
+	dialer := protocolstate.Dialer
 	transport := &http.Transport{
 		DialContext:         dialer.Dial,
 		MaxIdleConns:        500,
@@ -34,5 +22,5 @@ func newhttpClient(options *types.Options) (*http.Client, error) {
 			InsecureSkipVerify: true,
 		},
 	}
-	return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second}, nil
+	return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second}
 }

v2/pkg/protocols/headless/engine/page_actions_test.go

@@ -9,11 +9,14 @@ import (
 	"testing"
 	"time"
 
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/stretchr/testify/require"
 )
 
 func TestActionNavigate(t *testing.T) {
+	_ = protocolstate.Init(&types.Options{})
+
 	browser, err := New(&types.Options{ShowBrowser: false})
 	require.Nil(t, err, "could not create browser")
 	defer browser.Close()
@@ -46,6 +49,8 @@ func TestActionNavigate(t *testing.T) {
 }
 
 func TestActionScript(t *testing.T) {
+	_ = protocolstate.Init(&types.Options{})
+
 	browser, err := New(&types.Options{ShowBrowser: false})
 	require.Nil(t, err, "could not create browser")
 	defer browser.Close()
@@ -113,6 +118,8 @@ func TestActionScript(t *testing.T) {
 }
 
 func TestActionClick(t *testing.T) {
+	_ = protocolstate.Init(&types.Options{})
+
 	browser, err := New(&types.Options{ShowBrowser: false})
 	require.Nil(t, err, "could not create browser")
 	defer browser.Close()
@@ -151,6 +158,8 @@ func TestActionClick(t *testing.T) {
 }
 
 func TestActionRightClick(t *testing.T) {
+	_ = protocolstate.Init(&types.Options{})
+
 	browser, err := New(&types.Options{ShowBrowser: false})
 	require.Nil(t, err, "could not create browser")
 	defer browser.Close()
@@ -197,6 +206,8 @@ func TestActionRightClick(t *testing.T) {
 }
 
 func TestActionTextInput(t *testing.T) {
+	_ = protocolstate.Init(&types.Options{})
+
 	browser, err := New(&types.Options{ShowBrowser: false})
 	require.Nil(t, err, "could not create browser")
 	defer browser.Close()
@@ -236,6 +247,8 @@ func TestActionTextInput(t *testing.T) {
 }
 
 func TestActionHeadersChange(t *testing.T) {
+	_ = protocolstate.Init(&types.Options{})
+
 	browser, err := New(&types.Options{ShowBrowser: false})
 	require.Nil(t, err, "could not create browser")
 	defer browser.Close()

v2/pkg/protocols/http/build_request.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/corpix/uarand"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
@@ -113,11 +114,11 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st
 
 // makeHTTPRequestFromRaw creates a *http.Request from a raw request
 func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}) (*generatedRequest, error) {
-	return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads)
+	return r.handleRawWithPayloads(ctx, data, baseURL, values, payloads)
 }
 
-// handleRawWithPaylods handles raw requests along with paylaods
-func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest, baseURL string, values, generatorValues map[string]interface{}) (*generatedRequest, error) {
+// handleRawWithPayloads handles raw requests along with payloads
+func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest, baseURL string, values, generatorValues map[string]interface{}) (*generatedRequest, error) {
 	// Combine the template payloads along with base
 	// request values.
 	finalValues := generators.MergeMaps(generatorValues, values)
@@ -182,7 +183,7 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte
 	if r.request.Body != "" {
 		req.Body = ioutil.NopCloser(strings.NewReader(r.request.Body))
 	}
-	setHeader(req, "User-Agent", "Nuclei - Open-source project (github.com/projectdiscovery/nuclei)")
+	setHeader(req, "User-Agent", uarand.GetRandom())
 
 	// Only set these headers on non raw requests
 	if len(r.request.Raw) == 0 {

v2/pkg/protocols/http/http.go

@@ -3,7 +3,6 @@ package http
 import (
 	"strings"
 
-	"github.com/corpix/uarand"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
@@ -102,10 +101,6 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
 		}
 		r.customHeaders[parts[0]] = strings.TrimSpace(parts[1])
 	}
-	// Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given
-	if r.options.Options.RandomAgent {
-		r.customHeaders["User-Agent"] = uarand.GetRandom()
-	}
 
 	if r.Body != "" && !strings.Contains(r.Body, "\r\n") {
 		r.Body = strings.ReplaceAll(r.Body, "\n", "\r\n")

v2/pkg/protocols/http/httpclientpool/clientpool.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/rawhttp"
 	"github.com/projectdiscovery/retryablehttp-go"
@@ -97,14 +98,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 	var err error
 
 	if Dialer == nil {
-		opts := fastdialer.DefaultOptions
-		if options.SystemResolvers {
-			opts.EnableFallback = true
-		}
-		if options.ResolversFile != "" {
-			opts.BaseResolvers = options.InternalResolversList
-		}
-		Dialer, err = fastdialer.NewDialer(opts)
+		Dialer = protocolstate.Dialer
 	}
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create dialer")

v2/pkg/protocols/http/request.go

@@ -372,6 +372,9 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
 	finalEvent := make(output.InternalEvent)
 
 	outputEvent := r.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, request.meta)
+	if i := strings.LastIndex(hostname, ":"); i != -1 {
+		hostname = hostname[:i]
+	}
 	outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
 	outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse)
 	for k, v := range previous {

v2/pkg/protocols/network/networkclientpool/clientpool.go

@@ -1,8 +1,8 @@
 package networkclientpool
 
 import (
-	"github.com/pkg/errors"
 	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
@@ -16,18 +16,7 @@ func Init(options *types.Options) error {
 	if normalClient != nil {
 		return nil
 	}
-	opts := fastdialer.DefaultOptions
-	if options.SystemResolvers {
-		opts.EnableFallback = true
-	}
-	if options.ResolversFile != "" {
-		opts.BaseResolvers = options.InternalResolversList
-	}
-	dialer, err := fastdialer.NewDialer(opts)
-	if err != nil {
-		return errors.Wrap(err, "could not create dialer")
-	}
-	normalClient = dialer
+	normalClient = protocolstate.Dialer
 	return nil
 }
 

v2/pkg/types/types.go

@@ -73,8 +73,6 @@ type Options struct {
 	ShowBrowser bool
 	// SytemResolvers enables override of nuclei's DNS client opting to use system resolver stack.
 	SystemResolvers bool
-	// RandomAgent generates random User-Agent
-	RandomAgent bool
 	// Metrics enables display of metrics via an http endpoint
 	Metrics bool
 	// Debug mode allows debugging request/responses for the engine