Merge branch 'dev' into issue-3386-improve-error-logging

cmd/nuclei/main.go

@@ -352,6 +352,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.EnablePprof, "enable-pprof", "ep", false, "enable pprof debugging server"),
 		flagSet.CallbackVarP(printTemplateVersion, "templates-version", "tv", "shows the version of the installed nuclei-templates"),
 		flagSet.BoolVarP(&options.HealthCheck, "health-check", "hc", false, "run diagnostic check up"),
+		flagSet.DynamicVarP(&options.ErrorLabels, "error-label", "elabel", []string{"UNV"}, "enable logging for specific error labels (comma-separated) (labels: UNV"),
 	)
 
 	flagSet.CreateGroup("update", "Update",

internal/runner/options.go

@@ -17,6 +17,7 @@ import (
 	"github.com/projectdiscovery/gologger/formatter"
 	"github.com/projectdiscovery/gologger/levels"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+	elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
@@ -26,6 +27,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
 	fileutil "github.com/projectdiscovery/utils/file"
 	"github.com/projectdiscovery/utils/generic"
@@ -87,6 +89,8 @@ func ParseOptions(options *types.Options) {
 	// Load the resolvers if user asked for them
 	loadResolvers(options)
 
+	options.ErrorLabels = parseErrorLables(options.ErrorLabels)
+
 	err := protocolinit.Init(options)
 	if err != nil {
 		gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err)
@@ -430,3 +434,21 @@ func getBoolEnvValue(key string) bool {
 	value := os.Getenv(key)
 	return strings.EqualFold(value, "true")
 }
+
+func parseErrorLables(errorLabels []string) []string {
+	// Make error label entries in stats
+	for _, v := range elabel.ErrorLableMap {
+		stats.NewEntry(v.Name, v.Description)
+	}
+	if len(errorLabels) == 0 {
+		return errorLabels
+	}
+	var errLabels []string
+	for _, label := range errorLabels {
+		label = strings.ToLower(label)
+		if v, ok := elabel.ErrorLableMap[label]; ok {
+			errLabels = append(errLabels, v.Name)
+		}
+	}
+	return errLabels
+}

internal/runner/runner.go

@@ -17,6 +17,7 @@ import (
 	uncoverlib "github.com/projectdiscovery/uncover"
 	"github.com/projectdiscovery/utils/env"
 	permissionutil "github.com/projectdiscovery/utils/permission"
+	stringsutil "github.com/projectdiscovery/utils/strings"
 	updateutils "github.com/projectdiscovery/utils/update"
 
 	"github.com/logrusorgru/aurora"
@@ -40,6 +41,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
@@ -492,6 +494,7 @@ func (r *Runner) RunEnumeration() error {
 		}
 	}
 	r.progress.Stop()
+	r.displayErrorInfo()
 
 	if executorOpts.InputHelper != nil {
 		_ = executorOpts.InputHelper.Close()
@@ -615,6 +618,17 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
 	}
 }
 
+func (r *Runner) displayErrorInfo() {
+	if !r.options.Verbose {
+		return
+	}
+	errLables := r.options.ErrorLabels
+	// if error label is not provided, display the stats for specific labels
+	if !stringsutil.ContainsAny(elabel.UnresolvedVariablesErrorLabel, errLables...) {
+		stats.Display(elabel.UnresolvedVariablesErrorLabel)
+	}
+}
+
 // SaveResumeConfig to file
 func (r *Runner) SaveResumeConfig(path string) error {
 	resumeCfgClone := r.resumeCfg.Clone()

pkg/protocols/common/errors/label/error_label.go

@@ -0,0 +1,35 @@
+package label
+
+import (
+	"strings"
+)
+
+const (
+	UnresolvedVariablesErrorLabel = "unresolved-variables-error"
+)
+
+var (
+	ErrorLabels = []string{
+		UnresolvedVariablesErrorLabel,
+	}
+
+	ErrorLableMap = map[string]struct {
+		Name        string
+		Description string
+	}{
+		"unv": {
+			Name:        UnresolvedVariablesErrorLabel,
+			Description: "Failed %v requests due to unresolved variables. Use -elabel=UNV to enable unresolved variables logs.",
+		},
+	}
+)
+
+// Contains checks if the error string contains any of the provided labels, if yes returns matched label and true
+func Contains(errStr string, lables []string) (string, bool) {
+	for _, label := range lables {
+		if strings.Contains(errStr, label) {
+			return label, true
+		}
+	}
+	return "", false
+}

pkg/protocols/common/expressions/variables.go

@@ -7,6 +7,8 @@ import (
 
 	"github.com/Knetic/govaluate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
+	elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
+	errorutil "github.com/projectdiscovery/utils/errors"
 )
 
 var (
@@ -38,7 +40,7 @@ func ContainsUnresolvedVariables(items ...string) error {
 			unresolvedVariables = append(unresolvedVariables, match[1])
 		}
 		if len(unresolvedVariables) > 0 {
-			return errors.New("unresolved variables found: " + strings.Join(unresolvedVariables, ","))
+			return errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, "unresolved variables found: "+strings.Join(unresolvedVariables, ","))
 		}
 	}
 

pkg/protocols/common/expressions/variables_test.go

@@ -4,6 +4,8 @@ import (
 	"errors"
 	"testing"
 
+	elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
+	errorutil "github.com/projectdiscovery/utils/errors"
 	"github.com/stretchr/testify/require"
 )
 
@@ -12,11 +14,11 @@ func TestUnresolvedVariablesCheck(t *testing.T) {
 		data string
 		err  error
 	}{
-		{"{{test}}", errors.New("unresolved variables found: test")},
-		{"{{test}}/{{another}}", errors.New("unresolved variables found: test,another")},
+		{"{{test}}", errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, "unresolved variables found: test")},
+		{"{{test}}/{{another}}", errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, "unresolved variables found: test,another")},
 		{"test", nil},
-		{"%7b%7btest%7d%7d", errors.New("unresolved variables found: test")},
-		{"%7B%7Bfirst%2Asecond%7D%7D", errors.New("unresolved variables found: first%2Asecond")},
+		{"%7b%7btest%7d%7d", errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, "unresolved variables found: test")},
+		{"%7B%7Bfirst%2Asecond%7D%7D", errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, errors.New("unresolved variables found: first%2Asecond").Error())},
 		{"{{7*7}}", nil},
 		{"{{'a'+'b'}}", nil},
 		{"{{'a'}}", nil},

pkg/protocols/http/build_request.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
@@ -34,7 +35,7 @@ import (
 // ErrEvalExpression
 var (
 	ErrEvalExpression = errorutil.NewWithTag("expr", "could not evaluate helper expressions")
-	ErrUnresolvedVars = errorutil.NewWithFmt("unresolved variables `%v` found in request")
+	UnresolvedErrFmt  = "unresolved variables `%v` found in request"
 )
 
 // generatedRequest is a single generated request wrapped for a template request
@@ -208,7 +209,7 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
 		}
 
 		if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil {
-			return nil, ErrUnresolvedVars.Msgf(parts[1])
+			return nil, errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, UnresolvedErrFmt, parts[1])
 		}
 
 		parsed, err := urlutil.ParseURL(parts[1], true)
@@ -229,7 +230,7 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
 	if err := expressions.ContainsUnresolvedVariables(data); err != nil {
 		// early exit: if there are any unresolved variables in `path` after evaluation
 		// then return early since this will definitely fail
-		return nil, ErrUnresolvedVars.Msgf(data)
+		return nil, errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, UnresolvedErrFmt, data)
 	}
 
 	urlx, err := urlutil.ParseURL(data, true)
@@ -289,6 +290,10 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
 		return unsafeReq, nil
 	}
 
+	if err := expressions.ContainsUnresolvedVariables(rawRequestData.FullURL); err != nil {
+		return nil, errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, UnresolvedErrFmt, rawRequestData.FullURL)
+	}
+
 	urlx, err := urlutil.ParseURL(rawRequestData.FullURL, true)
 	if err != nil {
 		return nil, errorutil.NewWithErr(err).Msgf("failed to create request with url %v got %v", rawRequestData.FullURL, err).WithTag("raw")

pkg/protocols/http/request.go

@@ -24,6 +24,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/fuzz"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
@@ -36,7 +37,9 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
 	"github.com/projectdiscovery/rawhttp"
+	errorutil "github.com/projectdiscovery/utils/errors"
 	"github.com/projectdiscovery/utils/reader"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
@@ -368,6 +371,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 					return true, nil
 				}
 				request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
+				if lb, ok := elabel.Contains(err.Error(), elabel.ErrorLabels); ok {
+					stats.Increment(lb)
+				}
 				return true, err
 			}
 
@@ -462,7 +468,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 
 const drainReqSize = int64(8 * 1024)
 
-var errStopExecution = errors.New("stop execution due to unresolved variables")
+var errStopExecution = errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, "stop execution due to unresolved variables")
 
 // executeRequest executes the actual generated request and returns error if occurred
 func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, callback protocols.OutputEventCallback, requestCount int) error {
@@ -538,7 +544,10 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			}
 		} else { // Check if are there any unresolved variables. If yes, skip unless overridden by user.
 			if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
-				gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.MetaInput.Input, varErr)
+				stats.Increment(elabel.UnresolvedVariablesErrorLabel)
+				if _, ok := elabel.Contains(varErr.Error(), request.options.Options.ErrorLabels); ok {
+					gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.MetaInput.Input, varErr)
+				}
 				return errStopExecution
 			}
 		}

pkg/protocols/network/request.go

@@ -18,6 +18,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
@@ -28,6 +29,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
 	protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	"github.com/projectdiscovery/utils/reader"
@@ -245,7 +247,11 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 		reqBuilder.Write(finalData)
 
 		if err := expressions.ContainsUnresolvedVariables(string(finalData)); err != nil {
-			gologger.Warning().Msgf("[%s] Could not make network request for %s: %v\n", request.options.TemplateID, actualAddress, err)
+			request.options.Progress.IncrementFailedRequestsBy(1)
+			stats.Increment(elabel.UnresolvedVariablesErrorLabel)
+			if _, ok := elabel.Contains(err.Error(), request.options.Options.ErrorLabels); ok {
+				gologger.Warning().Msgf("[%s] Could not make network request for %s: %v\n", request.options.TemplateID, actualAddress, err)
+			}
 			return nil
 		}
 

pkg/tmplexec/generic/exec.go

@@ -7,6 +7,7 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
+	elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
 )
 
@@ -79,7 +80,13 @@ func (g *Generic) ExecuteWithResults(ctx *scan.ScanContext) error {
 			if g.options.HostErrorsCache != nil {
 				g.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.ID(), err)
 			}
-			gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, ctx.Input.MetaInput.PrettyPrint(), err)
+			if _, ok := elabel.Contains(err.Error(), g.options.Options.ErrorLabels); ok {
+				gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, ctx.Input.MetaInput.PrettyPrint(), err)
+			}
+			// TODO: remove logging for otherthan unresolved variables
+			if _, ok := elabel.Contains(err.Error(), []string{elabel.UnresolvedVariablesErrorLabel}); !ok {
+				gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, ctx.Input.MetaInput.PrettyPrint(), err)
+			}
 		}
 		// If a match was found and stop at first match is set, break out of the loop and return
 		if g.results.Load() && (g.options.StopAtFirstMatch || g.options.Options.StopAtFirstMatch) {

pkg/types/types.go

@@ -96,6 +96,8 @@ type Options struct {
 	TraceLogFile string
 	// ErrorLogFile specifies a file to write with the errors of all requests
 	ErrorLogFile string
+	// ErrorLabels is the list of error labels to match and enable logging
+	ErrorLabels	[]string
 	// ReportingDB is the db for report storage as well as deduplication
 	ReportingDB string
 	// ReportingConfig is the config file for nuclei reporting module