Merge branch 'dev' into disable_non_default_templates

cmd/integration-test/code.go

@@ -99,7 +99,7 @@ type codePreCondition struct{}
 
 // Execute executes a test case and returns an error if occurred
 func (h *codePreCondition) Execute(filePath string) error {
-	results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input", "-code")
+	results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input", "-code", "-var", "foo=bar")
 	if err != nil {
 		return err
 	}

cmd/integration-test/http.go

@@ -952,7 +952,7 @@ func (h *httpRequestSelfContained) Execute(filePath string) error {
 	}()
 	defer server.Close()
 
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug)
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-var foo=bar")
 	if err != nil {
 		return err
 	}
@@ -988,7 +988,7 @@ func (h *httpRequestSelfContainedWithParams) Execute(filePath string) error {
 	}()
 	defer server.Close()
 
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug)
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-var foo=bar")
 	if err != nil {
 		return err
 	}

cmd/integration-test/network.go

@@ -119,7 +119,7 @@ func (h *networkRequestSelContained) Execute(filePath string) error {
 		_, _ = conn.Write([]byte("Authentication successful"))
 	})
 	defer ts.Close()
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug)
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-var foo=bar")
 	if err != nil {
 		return err
 	}

go.mod

@@ -214,6 +214,7 @@ require (
 	github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
 	github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
 	github.com/refraction-networking/utls v1.6.7 // indirect
+	github.com/samber/lo v1.44.0 // indirect
 	github.com/sashabaranov/go-openai v1.15.3 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect

go.sum

@@ -959,6 +959,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
+github.com/samber/lo v1.44.0 h1:5il56KxRE+GHsm1IR+sZ/6J42NODigFiqCWpSc2dybA=
+github.com/samber/lo v1.44.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/sashabaranov/go-openai v1.15.3 h1:rzoNK9n+Cak+PM6OQ9puxDmFllxfnVea9StlmhglXqA=
 github.com/sashabaranov/go-openai v1.15.3/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=

internal/runner/runner.go

@@ -536,6 +536,7 @@ func (r *Runner) RunEnumeration() error {
 		// if input type is not list (implicitly enable fuzzing)
 		r.options.DAST = true
 	}
+
 	store, err := loader.New(loaderConfig)
 	if err != nil {
 		return errors.Wrap(err, "Could not create loader.")
@@ -744,6 +745,8 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
 	stats.DisplayAsWarning(httpProtocol.SetThreadToCountZero)
 	stats.ForceDisplayWarning(templates.SkippedUnsignedStats)
 	stats.ForceDisplayWarning(templates.SkippedRequestSignatureStats)
+	stats.ForceDisplayWarning(templates.SkippedSelfContainedStats)
+	stats.ForceDisplayWarning(templates.SkippedFileStats)
 
 	cfg := config.DefaultConfig
 

pkg/catalog/loader/loader.go

@@ -5,10 +5,12 @@ import (
 	"io"
 	"net/url"
 	"os"
+	"regexp"
 	"sort"
 	"strings"
 	"sync"
 
+	"github.com/Knetic/govaluate"
 	"github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
@@ -17,6 +19,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter"
 	"github.com/projectdiscovery/nuclei/v3/pkg/keys"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
@@ -25,9 +28,11 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/workflows"
 	"github.com/projectdiscovery/retryablehttp-go"
 	errorutil "github.com/projectdiscovery/utils/errors"
+	fileutil "github.com/projectdiscovery/utils/file"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 	urlutil "github.com/projectdiscovery/utils/url"
+	"github.com/samber/lo"
 )
 
 const (
@@ -489,6 +494,22 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
 						stats.Increment(templates.SkippedUnsignedStats)
 						return
 					}
+
+					if parsed.SelfContained &&
+						store.config.ExecutorOptions.Options.Vars.IsEmpty() && !store.config.ExecutorOptions.Options.EnvironmentVariables &&
+						templateContainsUnresolvedVariables(templatePath) {
+						stats.Increment(templates.SkippedSelfContainedStats)
+						return
+					}
+
+					if parsed.HasFileProtocol() &&
+						lo.NoneBy(store.config.ExecutorOptions.Options.Targets, func(target string) bool {
+							return fileutil.FileOrFolderExists(target)
+						}) {
+						stats.Increment(templates.SkippedFileStats)
+						return
+					}
+
 					// if template has request signature like aws then only signed and verified templates are allowed
 					if parsed.UsesRequestSignature() && !parsed.Verified {
 						stats.Increment(templates.SkippedRequestSignatureStats)
@@ -553,6 +574,55 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
 	return loadedTemplates.Slice
 }
 
+var (
+	numericalExpressionRegex = regexp.MustCompile(`^[0-9+\-/\W]+$`)
+	unresolvedVariablesRegex = regexp.MustCompile(`(?:%7[B|b]|\{){2}([^}]+)(?:%7[D|d]|\}){2}["'\)\}]*`)
+)
+
+// copy of the original function from pkg/protocols/common/expressions/variables.go:ContainsUnresolvedVariables
+func templateContainsUnresolvedVariables(templatePath string) bool {
+	data, err := os.ReadFile(templatePath)
+	if err != nil {
+		return false
+	}
+
+	matches := unresolvedVariablesRegex.FindAllStringSubmatch(string(data), -1)
+	if len(matches) == 0 {
+		return false
+	}
+
+	var unresolvedVariables []string
+	for _, match := range matches {
+		if len(match) < 2 {
+			continue
+		}
+
+		// Skip if the match is an expression
+		if numericalExpressionRegex.MatchString(match[1]) {
+			continue
+		}
+		// or if it contains only literals (can be solved from expression engine)
+		if hasLiteralsOnly(match[1]) {
+			continue
+		}
+		unresolvedVariables = append(unresolvedVariables, match[1])
+	}
+
+	return len(unresolvedVariables) > 0
+}
+
+func hasLiteralsOnly(data string) bool {
+	expr, err := govaluate.NewEvaluableExpressionWithFunctions(data, dsl.HelperFunctions)
+	if err != nil {
+		return false
+	}
+	if expr != nil {
+		_, err = expr.Evaluate(nil)
+		return err == nil
+	}
+	return true
+}
+
 // IsHTTPBasedProtocolUsed returns true if http/headless protocol is being used for
 // any templates.
 func IsHTTPBasedProtocolUsed(store *Store) bool {

pkg/templates/parser_stats.go

@@ -11,4 +11,6 @@ const (
 	ExludedDastTmplStats         = "fuzz-flag-missing-warnings"
 	SkippedUnsignedStats         = "skipped-unsigned-stats" // tracks loading of unsigned templates
 	SkippedRequestSignatureStats = "skipped-request-signature-stats"
+	SkippedSelfContainedStats    = "skipped-self-contained-stats"
+	SkippedFileStats             = "skipped-file-stats"
 )

pkg/templates/stats.go

@@ -13,4 +13,6 @@ func init() {
 	stats.NewEntry(ExludedDastTmplStats, "Excluded %d dast template[s] (disabled as default), use -dast option to run dast templates.")
 	stats.NewEntry(SkippedUnsignedStats, "Skipping %d unsigned template[s]")
 	stats.NewEntry(SkippedRequestSignatureStats, "Skipping %d templates, HTTP Request signatures can only be used in Signed & Verified templates.")
+	stats.NewEntry(SkippedSelfContainedStats, "Skipping %d self-contained template[s], use  -var or -env-vars flag to run them")
+	stats.NewEntry(SkippedFileStats, "Skipping %d file template[s], use file or directory as an input to run file templates")
 }

pkg/templates/templates.go

@@ -227,6 +227,11 @@ func (template *Template) HasCodeProtocol() bool {
 	return len(template.RequestsCode) > 0
 }
 
+// HasFileProtocol returns true if the template has a file protocol section
+func (template *Template) HasFileProtocol() bool {
+	return len(template.RequestsFile) > 0
+}
+
 // validateAllRequestIDs check if that protocol already has given id if not
 // then is is manually set to proto_index
 func (template *Template) validateAllRequestIDs() {