Merge remote-tracking branch 'origin'

Signed-off-by: Dwi Siswanto <[email protected]>

lib/config.go

@@ -463,6 +463,14 @@ func EnablePassiveMode() NucleiSDKOptions {
 	}
 }
 
+// EnableMatcherStatus allows enabling matcher status
+func EnableMatcherStatus() NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		e.opts.MatcherStatus = true
+		return nil
+	}
+}
+
 // WithAuthProvider allows setting a custom authprovider implementation
 func WithAuthProvider(provider authprovider.AuthProvider) NucleiSDKOptions {
 	return func(e *NucleiEngine) error {

pkg/authprovider/authx/dynamic.go

@@ -52,14 +52,19 @@ func (d *Dynamic) GetDomainAndDomainRegex() ([]string, []string) {
 }
 
 func (d *Dynamic) UnmarshalJSON(data []byte) error {
-	if err := json.Unmarshal(data, &d); err != nil {
+	if d == nil {
+		return errorutil.New("cannot unmarshal into nil Dynamic struct")
+	}
+
+	// Use an alias type (auxiliary) to avoid a recursive call in this method.
+	type Alias Dynamic
+
+	// If d.Secret was nil, json.Unmarshal will allocate a new Secret object
+	// and populate it from the top level JSON fields.
+	if err := json.Unmarshal(data, (*Alias)(d)); err != nil {
 		return err
 	}
-	var s Secret
-	if err := json.Unmarshal(data, &s); err != nil {
-		return err
-	}
-	d.Secret = &s
+
 	return nil
 }
 

pkg/authprovider/authx/dynamic_test.go

@@ -0,0 +1,125 @@
+package authx
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDynamicUnmarshalJSON(t *testing.T) {
+	t.Run("basic-unmarshal", func(t *testing.T) {
+		data := []byte(`{
+			"template": "test-template.yaml",
+			"variables": [
+				{
+					"key": "username",
+					"value": "testuser"
+				}
+			],
+			"secrets": [
+				{
+					"type": "BasicAuth",
+					"domains": ["example.com"],
+					"username": "user1",
+					"password": "pass1"
+				}
+			],
+			"type": "BasicAuth",
+			"domains": ["test.com"],
+			"username": "testuser",
+			"password": "testpass"
+		}`)
+
+		var d Dynamic
+		err := d.UnmarshalJSON(data)
+		require.NoError(t, err)
+
+		// Secret
+		require.NotNil(t, d.Secret)
+		require.Equal(t, "BasicAuth", d.Secret.Type)
+		require.Equal(t, []string{"test.com"}, d.Secret.Domains)
+		require.Equal(t, "testuser", d.Secret.Username)
+		require.Equal(t, "testpass", d.Secret.Password)
+
+		// Dynamic fields
+		require.Equal(t, "test-template.yaml", d.TemplatePath)
+		require.Len(t, d.Variables, 1)
+		require.Equal(t, "username", d.Variables[0].Key)
+		require.Equal(t, "testuser", d.Variables[0].Value)
+		require.Len(t, d.Secrets, 1)
+		require.Equal(t, "BasicAuth", d.Secrets[0].Type)
+		require.Equal(t, []string{"example.com"}, d.Secrets[0].Domains)
+		require.Equal(t, "user1", d.Secrets[0].Username)
+		require.Equal(t, "pass1", d.Secrets[0].Password)
+	})
+
+	t.Run("complex-unmarshal", func(t *testing.T) {
+		data := []byte(`{
+			"template": "test-template.yaml",
+			"variables": [
+				{
+					"key": "token",
+					"value": "Bearer xyz"
+				}
+			],
+			"secrets": [
+				{
+					"type": "CookiesAuth",
+					"domains": ["example.com"],
+					"cookies": [
+						{
+							"key": "session",
+							"value": "abc123"
+						}
+					]
+				}
+			],
+			"type": "HeadersAuth",
+			"domains": ["api.test.com"],
+			"headers": [
+				{
+					"key": "X-API-Key",
+					"value": "secret-key"
+				}
+			]
+		}`)
+
+		var d Dynamic
+		err := d.UnmarshalJSON(data)
+		require.NoError(t, err)
+
+		// Secret
+		require.NotNil(t, d.Secret)
+		require.Equal(t, "HeadersAuth", d.Secret.Type)
+		require.Equal(t, []string{"api.test.com"}, d.Secret.Domains)
+		require.Len(t, d.Secret.Headers, 1)
+		require.Equal(t, "X-API-Key", d.Secret.Headers[0].Key)
+		require.Equal(t, "secret-key", d.Secret.Headers[0].Value)
+
+		// Dynamic fields
+		require.Equal(t, "test-template.yaml", d.TemplatePath)
+		require.Len(t, d.Variables, 1)
+		require.Equal(t, "token", d.Variables[0].Key)
+		require.Equal(t, "Bearer xyz", d.Variables[0].Value)
+		require.Len(t, d.Secrets, 1)
+		require.Equal(t, "CookiesAuth", d.Secrets[0].Type)
+		require.Equal(t, []string{"example.com"}, d.Secrets[0].Domains)
+		require.Len(t, d.Secrets[0].Cookies, 1)
+		require.Equal(t, "session", d.Secrets[0].Cookies[0].Key)
+		require.Equal(t, "abc123", d.Secrets[0].Cookies[0].Value)
+	})
+
+	t.Run("invalid-json", func(t *testing.T) {
+		data := []byte(`{invalid json}`)
+		var d Dynamic
+		err := d.UnmarshalJSON(data)
+		require.Error(t, err)
+	})
+
+	t.Run("empty-json", func(t *testing.T) {
+		data := []byte(`{}`)
+		var d Dynamic
+		err := d.UnmarshalJSON(data)
+		require.NoError(t, err)
+	})
+}

pkg/catalog/config/constants.go

@@ -31,7 +31,7 @@ const (
 	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
-	Version = `v3.4.4`
+	Version = `v3.4.5`
 	// Directory Names of custom templates
 	CustomS3TemplatesDirName     = "s3"
 	CustomGitHubTemplatesDirName = "github"

pkg/core/executors.go

@@ -38,7 +38,7 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
 				match, err = template.Executer.Execute(ctx)
 			}
 			if err != nil {
-				gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
+				gologger.Warning().Msgf("[%s] Could not execute step (self-contained): %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
 			}
 			results.CompareAndSwap(false, match)
 		}(v)
@@ -140,7 +140,7 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ
 				}
 			}
 			if err != nil {
-				gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
+				gologger.Warning().Msgf("[%s] Could not execute step on %s: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), value.Input, err)
 			}
 			results.CompareAndSwap(false, match)
 		}(index, skip, scannedValue)
@@ -206,7 +206,7 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t
 				}
 			}
 			if err != nil {
-				gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
+				gologger.Warning().Msgf("[%s] Could not execute step on %s: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), value.Input, err)
 			}
 			results.CompareAndSwap(false, match)
 		}(tpl, target, sg)

pkg/protocols/dns/dns.go

@@ -185,6 +185,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadata map[string]interface{}) (*retryabledns.Client, error) {
 	dnsClientOptions := &dnsclientpool.Configuration{
 		Retries: request.Retries,
+		Proxy:   options.Options.AliveSocksProxy,
 	}
 	if len(request.Resolvers) > 0 {
 		if len(request.Resolvers) > 0 {

pkg/protocols/dns/dnsclientpool/clientpool.go

@@ -51,6 +51,8 @@ type Configuration struct {
 	Retries int
 	// Resolvers contains the specific per request resolvers
 	Resolvers []string
+	// Proxy contains the proxy to use for the dns client
+	Proxy string
 }
 
 // Hash returns the hash of the configuration to allow client pooling
@@ -60,6 +62,8 @@ func (c *Configuration) Hash() string {
 	builder.WriteString(strconv.Itoa(c.Retries))
 	builder.WriteString("l")
 	builder.WriteString(strings.Join(c.Resolvers, ""))
+	builder.WriteString("p")
+	builder.WriteString(c.Proxy)
 	hash := builder.String()
 	return hash
 }
@@ -83,7 +87,11 @@ func Get(options *types.Options, configuration *Configuration) (*retryabledns.Cl
 	} else if len(configuration.Resolvers) > 0 {
 		resolvers = configuration.Resolvers
 	}
-	client, err := retryabledns.New(resolvers, configuration.Retries)
+	client, err := retryabledns.NewWithOptions(retryabledns.Options{
+		BaseResolvers: resolvers,
+		MaxRetries:    configuration.Retries,
+		Proxy:         options.AliveSocksProxy,
+	})
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create dns client")
 	}

pkg/protocols/headless/request.go

@@ -223,10 +223,15 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
 }
 
 func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecutorOptions, responseBody string, input string) {
-	cliOptions := requestOptions.Options
-	if cliOptions.Debug || cliOptions.DebugResponse {
-		highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, responseBody, cliOptions.NoColor, false)
-		gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n\n%s", requestOptions.TemplateID, input, highlightedResponse)
+	if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse {
+		msg := fmt.Sprintf("[%s] Dumped Headless response for %s\n\n", requestOptions.TemplateID, input)
+		if requestOptions.Options.Debug || requestOptions.Options.DebugResponse {
+			resp := responsehighlighter.Highlight(event.OperatorsResult, responseBody, requestOptions.Options.NoColor, false)
+			gologger.Debug().Msgf("%s%s", msg, resp)
+		}
+		if requestOptions.Options.StoreResponse {
+			requestOptions.Output.WriteStoreDebugData(input, requestOptions.TemplateID, "headless", fmt.Sprintf("%s%s", msg, responseBody))
+		}
 	}
 }
 

pkg/protocols/http/http.go

@@ -11,6 +11,7 @@ import (
 	json "github.com/json-iterator/go"
 	"github.com/pkg/errors"
 
+	"github.com/projectdiscovery/fastdialer/fastdialer"
 	_ "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/analyzers/time"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/fuzz"
@@ -22,6 +23,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
 	httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
 	"github.com/projectdiscovery/rawhttp"
@@ -144,6 +146,7 @@ type Request struct {
 	generator         *generators.PayloadGenerator // optional, only enabled when using payloads
 	httpClient        *retryablehttp.Client
 	rawhttpClient     *rawhttp.Client
+	dialer            *fastdialer.Dialer
 
 	// description: |
 	//   SelfContained specifies if the request is self-contained.
@@ -348,6 +351,15 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 	}
 	request.customHeaders = make(map[string]string)
 	request.httpClient = client
+
+	dialer, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{
+		CustomDialer: options.CustomFastdialer,
+	})
+	if err != nil {
+		return errors.Wrap(err, "could not get dialer")
+	}
+	request.dialer = dialer
+
 	request.options = options
 	for _, option := range request.options.Options.CustomHeaders {
 		parts := strings.SplitN(option, ":", 2)

pkg/protocols/http/request.go

@@ -841,7 +841,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		if input.MetaInput.CustomIP != "" {
 			outputEvent["ip"] = input.MetaInput.CustomIP
 		} else {
-			outputEvent["ip"] = protocolstate.Dialer.GetDialedIP(hostname)
+			outputEvent["ip"] = request.dialer.GetDialedIP(hostname)
 			// try getting cname
 			request.addCNameIfAvailable(hostname, outputEvent)
 		}
@@ -1085,11 +1085,11 @@ func (request *Request) validateNFixEvent(input *contextargs.Context, gr *genera
 
 // addCNameIfAvailable adds the cname to the event if available
 func (request *Request) addCNameIfAvailable(hostname string, outputEvent map[string]interface{}) {
-	if protocolstate.Dialer == nil {
+	if request.dialer == nil {
 		return
 	}
 
-	data, err := protocolstate.Dialer.GetDNSData(hostname)
+	data, err := request.dialer.GetDNSData(hostname)
 	if err == nil {
 		switch len(data.CNAME) {
 		case 0:

pkg/protocols/network/network.go

@@ -237,7 +237,9 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 	}
 
 	// Create a client for the class
-	client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{})
+	client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{
+		CustomDialer: options.CustomFastdialer,
+	})
 	if err != nil {
 		return errors.Wrap(err, "could not get network client")
 	}
@@ -259,7 +261,3 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 func (request *Request) Requests() int {
 	return len(request.Address)
 }
-
-func (request *Request) SetDialer(dialer *fastdialer.Dialer) {
-	request.dialer = dialer
-}

pkg/protocols/network/networkclientpool/clientpool.go

@@ -21,7 +21,9 @@ func Init(options *types.Options) error {
 }
 
 // Configuration contains the custom configuration options for a client
-type Configuration struct{}
+type Configuration struct {
+	CustomDialer *fastdialer.Dialer
+}
 
 // Hash returns the hash of the configuration to allow client pooling
 func (c *Configuration) Hash() string {
@@ -30,5 +32,10 @@ func (c *Configuration) Hash() string {
 
 // Get creates or gets a client for the protocol based on custom configuration
 func Get(options *types.Options, configuration *Configuration /*TODO review unused parameters*/) (*fastdialer.Dialer, error) {
+
+	if configuration != nil && configuration.CustomDialer != nil {
+		return configuration.CustomDialer, nil
+	}
+
 	return normalClient, nil
 }

pkg/protocols/network/request.go

@@ -25,9 +25,9 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
 	protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	errorutil "github.com/projectdiscovery/utils/errors"
@@ -64,7 +64,11 @@ func (request *Request) getOpenPorts(target *contextargs.Context) ([]string, err
 			errs = append(errs, err)
 			continue
 		}
-		conn, err := protocolstate.Dialer.Dial(target.Context(), "tcp", addr)
+		if request.dialer == nil {
+			request.dialer, _ = networkclientpool.Get(request.options.Options, &networkclientpool.Configuration{})
+		}
+
+		conn, err := request.dialer.Dial(target.Context(), "tcp", addr)
 		if err != nil {
 			errs = append(errs, err)
 			continue

pkg/protocols/protocols.go

@@ -5,6 +5,7 @@ import (
 	"encoding/base64"
 	"sync/atomic"
 
+	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/ratelimit"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	stringsutil "github.com/projectdiscovery/utils/strings"
@@ -132,6 +133,8 @@ type ExecutorOptions struct {
 	ExportReqURLPattern bool
 	// GlobalMatchers is the storage for global matchers with http passive templates
 	GlobalMatchers *globalmatchers.Storage
+	// CustomFastdialer is a fastdialer dialer instance
+	CustomFastdialer *fastdialer.Dialer
 }
 
 // todo: centralizing components is not feasible with current clogged architecture

pkg/protocols/ssl/ssl.go

@@ -115,7 +115,9 @@ func (request *Request) IsClusterable() bool {
 func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 	request.options = options
 
-	client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{})
+	client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{
+		CustomDialer: options.CustomFastdialer,
+	})
 	if err != nil {
 		return errorutil.NewWithTag("ssl", "could not get network client").Wrap(err)
 	}

pkg/protocols/websocket/websocket.go

@@ -100,7 +100,9 @@ const (
 func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 	request.options = options
 
-	client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{})
+	client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{
+		CustomDialer: options.CustomFastdialer,
+	})
 	if err != nil {
 		return errors.Wrap(err, "could not get network client")
 	}

pkg/tmplexec/generic/exec.go

@@ -1,13 +1,13 @@
 package generic
 
 import (
-	"strings"
 	"sync/atomic"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
+	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/utils"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 )
 
@@ -64,17 +64,9 @@ func (g *Generic) ExecuteWithResults(ctx *scan.ScanContext) error {
 				// ideally this should never happen since protocol exits on error and callback is not called
 				return
 			}
-			ID := req.GetID()
-			if ID != "" {
-				builder := &strings.Builder{}
-				for k, v := range event.InternalEvent {
-					builder.WriteString(ID)
-					builder.WriteString("_")
-					builder.WriteString(k)
-					_ = previous.Set(builder.String(), v)
-					builder.Reset()
-				}
-			}
+
+			utils.FillPreviousEvent(req.GetID(), event, previous)
+
 			if event.HasOperatorResult() {
 				g.results.CompareAndSwap(false, true)
 			}

pkg/tmplexec/multiproto/multi.go

@@ -2,7 +2,6 @@ package multiproto
 
 import (
 	"strconv"
-	"strings"
 	"sync/atomic"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
@@ -10,6 +9,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/utils"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
@@ -90,17 +90,7 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
 				return
 			}
 
-			ID := req.GetID()
-			if ID != "" {
-				builder := &strings.Builder{}
-				for k, v := range event.InternalEvent {
-					builder.WriteString(ID)
-					builder.WriteString("_")
-					builder.WriteString(k)
-					_ = previous.Set(builder.String(), v)
-					builder.Reset()
-				}
-			}
+			utils.FillPreviousEvent(req.GetID(), event, previous)
 
 			// log event and generate result for the event
 			ctx.LogEvent(event)

pkg/tmplexec/utils/utils.go

@@ -0,0 +1,34 @@
+package utils
+
+import (
+	"strings"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	mapsutil "github.com/projectdiscovery/utils/maps"
+)
+
+// FillPreviousEvent is a helper function to get the previous event from the event
+// without leading to duplicate prefixes
+func FillPreviousEvent(reqID string, event *output.InternalWrappedEvent, previous *mapsutil.SyncLockMap[string, any]) {
+	if reqID == "" {
+		return
+	}
+
+	for k, v := range event.InternalEvent {
+		if _, ok := previous.Get(k); ok {
+			continue
+		}
+
+		if strings.HasPrefix(k, reqID+"_") {
+			continue
+		}
+
+		var builder strings.Builder
+
+		builder.WriteString(reqID)
+		builder.WriteString("_")
+		builder.WriteString(k)
+
+		_ = previous.Set(builder.String(), v)
+	}
+}