Merge remote-tracking branch 'origin/dev' into feat-performance-improvements

# Conflicts:
#	pkg/operators/cache/cache.go
#	pkg/operators/matchers/match.go
#	pkg/output/multi_writer.go
#	pkg/protocols/common/expressions/expressions.go
#	pkg/protocols/common/expressions/variables.go

pkg/catalog/loader/loader.go

@@ -727,28 +727,75 @@ func (store *Store) LoadTemplates(templatesList []string) ([]*templates.Template
 	return store.LoadTemplatesWithTags(templatesList, nil)
 }
 
-// LoadWorkflows takes a list of workflows and returns paths for them
+// LoadWorkflows takes a list of workflows and returns paths for them.
+//
+// Workflows are loaded concurrently using TemplateLoadingConcurrency (mirrors
+// the concurrency model of LoadTemplatesWithTags). Order is preserved by
+// pre-allocating the result slice and writing into the original index, so
+// downstream consumers see the same order as the input list.
 func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
 	includedWorkflows, errs := store.config.Catalog.GetTemplatesPath(workflowsList)
 	store.logErroredTemplates(errs)
 
-	loadedWorkflows := make([]*templates.Template, 0, len(includedWorkflows))
-	for _, workflowPath := range includedWorkflows {
-		loaded, err := store.config.ExecutorOptions.Parser.LoadWorkflow(workflowPath, store.config.Catalog)
-		if err != nil {
-			store.logger.Warning().Msgf("Could not load workflow %s: %s\n", workflowPath, err)
-		}
-
-		if loaded {
-			parsed, err := templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions)
-			if err != nil {
-				store.logger.Warning().Msgf("Could not parse workflow %s: %s\n", workflowPath, err)
-			} else if parsed != nil {
-				loadedWorkflows = append(loadedWorkflows, parsed)
-			}
-		}
+	if len(includedWorkflows) == 0 {
+		return nil
 	}
 
+	concurrency := store.config.ExecutorOptions.Options.TemplateLoadingConcurrency
+	if concurrency <= 0 {
+		concurrency = types.DefaultTemplateLoadingConcurrency
+	}
+	wg, errWg := syncutil.New(syncutil.WithSize(concurrency))
+	if errWg != nil {
+		// Fall back to sequential on the rare error path.
+		loadedWorkflows := make([]*templates.Template, 0, len(includedWorkflows))
+		for _, workflowPath := range includedWorkflows {
+			loaded, err := store.config.ExecutorOptions.Parser.LoadWorkflow(workflowPath, store.config.Catalog)
+			if err != nil {
+				store.logger.Warning().Msgf("Could not load workflow %s: %s\n", workflowPath, err)
+			}
+			if loaded {
+				parsed, err := templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions)
+				if err != nil {
+					store.logger.Warning().Msgf("Could not parse workflow %s: %s\n", workflowPath, err)
+				} else if parsed != nil {
+					loadedWorkflows = append(loadedWorkflows, parsed)
+				}
+			}
+		}
+		return loadedWorkflows
+	}
+
+	parsedSlots := make([]*templates.Template, len(includedWorkflows))
+	for i, workflowPath := range includedWorkflows {
+		wg.Add()
+		go func(idx int, path string) {
+			defer wg.Done()
+			loaded, err := store.config.ExecutorOptions.Parser.LoadWorkflow(path, store.config.Catalog)
+			if err != nil {
+				store.logger.Warning().Msgf("Could not load workflow %s: %s\n", path, err)
+			}
+			if !loaded {
+				return
+			}
+			parsed, err := templates.Parse(path, store.preprocessor, store.config.ExecutorOptions)
+			if err != nil {
+				store.logger.Warning().Msgf("Could not parse workflow %s: %s\n", path, err)
+				return
+			}
+			if parsed != nil {
+				parsedSlots[idx] = parsed
+			}
+		}(i, workflowPath)
+	}
+	wg.Wait()
+
+	loadedWorkflows := make([]*templates.Template, 0, len(includedWorkflows))
+	for _, p := range parsedSlots {
+		if p != nil {
+			loadedWorkflows = append(loadedWorkflows, p)
+		}
+	}
 	return loadedWorkflows
 }
 

pkg/core/executors.go

@@ -90,7 +90,14 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ
 		value *contextargs.MetaInput
 	}
 
-	tasks := make(chan task)
+	// Buffered to workerCount so the producer (target.Iterate) does not block
+	// on every send when workers are momentarily busy. This keeps the input
+	// iterator advancing and reduces tail latency on slow targets.
+	taskBuf := workerCount
+	if taskBuf < 1 {
+		taskBuf = 1
+	}
+	tasks := make(chan task, taskBuf)
 	var workersWg sync.WaitGroup
 	workersWg.Add(workerCount)
 	for i := 0; i < workerCount; i++ {

pkg/operators/cache/cache.go

@@ -4,6 +4,7 @@ import (
 	"regexp"
 	"sync"
 
+	"github.com/itchyny/gojq"
 	"github.com/projectdiscovery/gcache"
 	"github.com/projectdiscovery/govaluate"
 )
@@ -14,15 +15,18 @@ var (
 
 	regexCap = 4096
 	dslCap   = 4096
+	jqCap    = 2048
 
 	regexCache gcache.Cache[string, *regexp.Regexp]
 	dslCache   gcache.Cache[string, *govaluate.EvaluableExpression]
+	jqCache    gcache.Cache[string, *gojq.Code]
 )
 
 func initCaches() {
 	initOnce.Do(func() {
 		regexCache = gcache.New[string, *regexp.Regexp](regexCap).LRU().Build()
 		dslCache = gcache.New[string, *govaluate.EvaluableExpression](dslCap).LRU().Build()
+		jqCache = gcache.New[string, *gojq.Code](jqCap).LRU().Build()
 	})
 }
 
@@ -60,3 +64,13 @@ func DSL() gcache.Cache[string, *govaluate.EvaluableExpression] {
 	defer mu.RUnlock()
 	return dslCache
 }
+
+// JQ returns the shared LRU cache for compiled gojq programs. Sharing one
+// cache across all extractors means that thousands of templates referencing
+// the same JQ query (e.g. `.cves[]`) compile it once.
+func JQ() gcache.Cache[string, *gojq.Code] {
+	initCaches()
+	mu.RLock()
+	defer mu.RUnlock()
+	return jqCache
+}

pkg/operators/common/bodycache/bodycache.go

@@ -0,0 +1,139 @@
+// Package bodycache memoizes per-response HTML, XML and JSON parses so that
+// multiple matchers and extractors against the same response share a single
+// parse. Without this, each MatchHTML/MatchXML/ExtractHTML/ExtractXML/ExtractJSON
+// call re-parses the body independently, which is a substantial slice of the
+// per-response CPU budget for clustered or matcher-heavy templates.
+//
+// The cache is intentionally tiny: a single most-recent entry per parser
+// type. The vast majority of matchers in one operator block target the same
+// corpus (typically the response body), so a one-slot LRU achieves near-100%
+// hit rate without map allocation or hashing of large body strings.
+package bodycache
+
+import (
+	"strings"
+	"sync"
+
+	"github.com/antchfx/htmlquery"
+	"github.com/antchfx/xmlquery"
+	"golang.org/x/net/html"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
+)
+
+// Key is the reserved entry on output.InternalEvent / data maps that holds
+// a *Cache. Underscored prefix avoids collisions with template variables and
+// DSL identifiers.
+const Key = "__nuclei_bodycache"
+
+// Cache stores at most one parsed form per parser type. Matchers/extractors
+// typically run sequentially against one response on one goroutine, but the
+// same cache can outlive that goroutine when it is stored on an event map
+// that is later re-evaluated (e.g. interactsh OOB matching). The mutex keeps
+// the memoized parses safe if the cache is ever touched concurrently; it is
+// uncontended on the common single-goroutine path.
+type Cache struct {
+	mu sync.Mutex
+
+	htmlCorpus string
+	htmlDoc    *html.Node
+	htmlErr    error
+	htmlSet    bool
+
+	xmlCorpus string
+	xmlDoc    *xmlquery.Node
+	xmlErr    error
+	xmlSet    bool
+
+	jsonCorpus string
+	jsonObj    interface{}
+	jsonErr    error
+	jsonSet    bool
+
+	loweredCorpus string
+	loweredOut    string
+	loweredSet    bool
+}
+
+// From returns the Cache associated with data, creating one on demand. Calls
+// from within Match/Extract dispatch share the same instance for the lifetime
+// of a single response event.
+func From(data map[string]interface{}) *Cache {
+	if data == nil {
+		return &Cache{}
+	}
+	if v, ok := data[Key]; ok {
+		if c, ok := v.(*Cache); ok && c != nil {
+			return c
+		}
+	}
+	c := &Cache{}
+	data[Key] = c
+	return c
+}
+
+// HTMLNode returns the parsed HTML tree for corpus. Subsequent calls with
+// the same corpus return the cached parse; calls with a different corpus
+// re-parse and replace the cache entry.
+func (c *Cache) HTMLNode(corpus string) (*html.Node, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.htmlSet && c.htmlCorpus == corpus {
+		return c.htmlDoc, c.htmlErr
+	}
+	doc, err := htmlquery.Parse(strings.NewReader(corpus))
+	c.htmlCorpus = corpus
+	c.htmlDoc = doc
+	c.htmlErr = err
+	c.htmlSet = true
+	return doc, err
+}
+
+// XMLNode returns the parsed XML tree for corpus. Same caching contract as
+// HTMLNode.
+func (c *Cache) XMLNode(corpus string) (*xmlquery.Node, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.xmlSet && c.xmlCorpus == corpus {
+		return c.xmlDoc, c.xmlErr
+	}
+	doc, err := xmlquery.Parse(strings.NewReader(corpus))
+	c.xmlCorpus = corpus
+	c.xmlDoc = doc
+	c.xmlErr = err
+	c.xmlSet = true
+	return doc, err
+}
+
+// JSONObject returns the unmarshaled JSON value for corpus. Same caching
+// contract as HTMLNode.
+func (c *Cache) JSONObject(corpus string) (interface{}, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.jsonSet && c.jsonCorpus == corpus {
+		return c.jsonObj, c.jsonErr
+	}
+	var obj interface{}
+	err := json.Unmarshal([]byte(corpus), &obj)
+	c.jsonCorpus = corpus
+	c.jsonObj = obj
+	c.jsonErr = err
+	c.jsonSet = true
+	return obj, err
+}
+
+// Lowered returns the lower-cased form of corpus. Cached so repeated
+// case-insensitive word matchers against the same body avoid redundant
+// O(n) ToLower allocations.
+func (c *Cache) Lowered(corpus string) string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.loweredSet && c.loweredCorpus == corpus {
+		return c.loweredOut
+	}
+	out := strings.ToLower(corpus)
+	c.loweredCorpus = corpus
+	c.loweredOut = out
+	c.loweredSet = true
+	return out
+}

pkg/operators/common/bodycache/bodycache_test.go

@@ -0,0 +1,84 @@
+package bodycache
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFrom_CreatesAndReuses(t *testing.T) {
+	data := map[string]interface{}{}
+	c1 := From(data)
+	require.NotNil(t, c1)
+	c2 := From(data)
+	require.Same(t, c1, c2, "From must return the cached instance stored on the map")
+
+	// nil map is tolerated and yields a throwaway cache.
+	require.NotNil(t, From(nil))
+}
+
+func TestFrom_IgnoresWrongType(t *testing.T) {
+	data := map[string]interface{}{Key: "not-a-cache"}
+	c := From(data)
+	require.NotNil(t, c)
+	if _, ok := data[Key].(*Cache); !ok {
+		t.Fatalf("From must overwrite a non-*Cache value at Key with a real cache")
+	}
+}
+
+func TestCache_HTMLMemoizesSameCorpus(t *testing.T) {
+	c := &Cache{}
+	const corpus = `<html><body><a href="x">link</a></body></html>`
+	doc1, err1 := c.HTMLNode(corpus)
+	doc2, err2 := c.HTMLNode(corpus)
+	require.NoError(t, err1)
+	require.NoError(t, err2)
+	require.Same(t, doc1, doc2, "same corpus must return the memoized parse")
+}
+
+func TestCache_HTMLReparsesOnCorpusChange(t *testing.T) {
+	c := &Cache{}
+	doc1, _ := c.HTMLNode(`<html><body>a</body></html>`)
+	doc2, _ := c.HTMLNode(`<html><body>b</body></html>`)
+	require.NotSame(t, doc1, doc2, "a different corpus must invalidate and re-parse")
+}
+
+func TestCache_JSONMemoizesAndReportsError(t *testing.T) {
+	c := &Cache{}
+	obj1, err1 := c.JSONObject(`{"a":1}`)
+	obj2, err2 := c.JSONObject(`{"a":1}`)
+	require.NoError(t, err1)
+	require.NoError(t, err2)
+	require.Equal(t, obj1, obj2)
+
+	_, err := c.JSONObject(`{not-json`)
+	require.Error(t, err, "invalid JSON must surface the unmarshal error")
+}
+
+func TestCache_Lowered(t *testing.T) {
+	c := &Cache{}
+	require.Equal(t, "abc", c.Lowered("ABC"))
+	require.Equal(t, "abc", c.Lowered("ABC"), "second call hits the memoized result")
+	require.Equal(t, "xyz", c.Lowered("XYZ"), "different corpus re-lowers")
+}
+
+// TestCache_ConcurrentAccess exercises the mutex: a single cache instance hit
+// from many goroutines (the interactsh re-evaluation shape) must not race or
+// panic. Run with -race to validate.
+func TestCache_ConcurrentAccess(t *testing.T) {
+	c := &Cache{}
+	const corpus = `<html><body><div id="a">x</div></body></html>`
+
+	var wg sync.WaitGroup
+	for i := 0; i < 32; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			_, _ = c.HTMLNode(corpus)
+			_, _ = c.JSONObject(`{"k":"v"}`)
+			_ = c.Lowered("MiXeD")
+		}()
+	}
+	wg.Wait()
+}

pkg/operators/extractors/compile.go

@@ -37,14 +37,19 @@ func (e *Extractor) CompileExtractors() error {
 	}
 
 	for _, query := range e.JSON {
-		query, err := gojq.Parse(query)
+		if cached, err := cache.JQ().GetIFPresent(query); err == nil && cached != nil {
+			e.jsonCompiled = append(e.jsonCompiled, cached)
+			continue
+		}
+		parsed, err := gojq.Parse(query)
 		if err != nil {
 			return fmt.Errorf("could not parse json: %s", query)
 		}
-		compiled, err := gojq.Compile(query)
+		compiled, err := gojq.Compile(parsed)
 		if err != nil {
 			return fmt.Errorf("could not compile json: %s", query)
 		}
+		_ = cache.JQ().Set(query, compiled)
 		e.jsonCompiled = append(e.jsonCompiled, compiled)
 	}
 

pkg/operators/extractors/extract.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/antchfx/htmlquery"
 	"github.com/antchfx/xmlquery"
+	"golang.org/x/net/html"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
@@ -43,20 +44,37 @@ func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} {
 	return results
 }
 
-// ExtractKval extracts key value pairs from a data map
+// ExtractKval extracts key value pairs from a data map.
+//
+// For the case-insensitive variant we previously copied the entire input map
+// and lower-cased every key plus every string value. This was O(event size)
+// per extractor invocation regardless of how many KVal entries we actually
+// queried (typically 1-3 keys against a map that can hold dozens of headers
+// and full body fields). The new implementation scans the source map only
+// for matches against the requested KVal entries via strings.EqualFold,
+// then lower-cases hits in place. No event-wide copy.
 func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{} {
+	results := make(map[string]struct{})
 	if e.CaseInsensitive {
-		inputData := data
-		data = make(map[string]interface{}, len(inputData))
-		for k, v := range inputData {
-			if s, ok := v.(string); ok {
-				v = strings.ToLower(s)
+		for _, k := range e.KVal {
+			for srcKey, srcVal := range data {
+				if !strings.EqualFold(srcKey, k) {
+					continue
+				}
+				itemString := types.ToString(srcVal)
+				if s, ok := srcVal.(string); ok {
+					itemString = strings.ToLower(s)
+				} else {
+					itemString = strings.ToLower(itemString)
+				}
+				if _, present := results[itemString]; !present {
+					results[itemString] = struct{}{}
+				}
 			}
-			data[strings.ToLower(k)] = v
 		}
+		return results
 	}
 
-	results := make(map[string]struct{})
 	for _, k := range e.KVal {
 		item, ok := data[k]
 		if !ok {
@@ -78,12 +96,22 @@ func (e *Extractor) ExtractXPath(corpus string) map[string]struct{} {
 	return e.ExtractHTML(corpus)
 }
 
-// ExtractHTML extracts items from HTML using XPath selectors
+// ExtractHTML extracts items from HTML using XPath selectors. Parses the
+// corpus on every call. Protocol callers with access to the per-response
+// body cache should prefer ExtractHTMLNode.
 func (e *Extractor) ExtractHTML(corpus string) map[string]struct{} {
-	results := make(map[string]struct{})
-
 	doc, err := htmlquery.Parse(strings.NewReader(corpus))
 	if err != nil {
+		return make(map[string]struct{})
+	}
+	return e.ExtractHTMLNode(doc)
+}
+
+// ExtractHTMLNode extracts items from a pre-parsed HTML node using XPath
+// selectors.
+func (e *Extractor) ExtractHTMLNode(doc *html.Node) map[string]struct{} {
+	results := make(map[string]struct{})
+	if doc == nil {
 		return results
 	}
 	for _, k := range e.XPath {
@@ -93,7 +121,6 @@ func (e *Extractor) ExtractHTML(corpus string) map[string]struct{} {
 		}
 		for _, node := range nodes {
 			var value string
-
 			if e.Attribute != "" {
 				value = htmlquery.SelectAttr(node, e.Attribute)
 			} else {
@@ -107,15 +134,24 @@ func (e *Extractor) ExtractHTML(corpus string) map[string]struct{} {
 	return results
 }
 
-// ExtractXML extracts items from XML using XPath selectors
+// ExtractXML extracts items from XML using XPath selectors. Parses the
+// corpus on every call. Protocol callers with access to the per-response
+// body cache should prefer ExtractXMLNode.
 func (e *Extractor) ExtractXML(corpus string) map[string]struct{} {
-	results := make(map[string]struct{})
-
 	doc, err := xmlquery.Parse(strings.NewReader(corpus))
 	if err != nil {
+		return make(map[string]struct{})
+	}
+	return e.ExtractXMLNode(doc)
+}
+
+// ExtractXMLNode extracts items from a pre-parsed XML node using XPath
+// selectors.
+func (e *Extractor) ExtractXMLNode(doc *xmlquery.Node) map[string]struct{} {
+	results := make(map[string]struct{})
+	if doc == nil {
 		return results
 	}
-
 	for _, k := range e.XPath {
 		nodes, err := xmlquery.QueryAll(doc, k)
 		if err != nil {
@@ -123,7 +159,6 @@ func (e *Extractor) ExtractXML(corpus string) map[string]struct{} {
 		}
 		for _, node := range nodes {
 			var value string
-
 			if e.Attribute != "" {
 				value = node.SelectAttr(e.Attribute)
 			} else {
@@ -137,16 +172,22 @@ func (e *Extractor) ExtractXML(corpus string) map[string]struct{} {
 	return results
 }
 
-// ExtractJSON extracts text from a corpus using JQ queries and returns it
+// ExtractJSON extracts text from a corpus using JQ queries. Unmarshals on
+// every call. Protocol callers with access to the per-response body cache
+// should prefer ExtractJSONObject to share the unmarshaled value across
+// multiple json extractors.
 func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} {
-	results := make(map[string]struct{})
-
 	var jsonObj interface{}
-
 	if err := json.Unmarshal([]byte(corpus), &jsonObj); err != nil {
-		return results
+		return make(map[string]struct{})
 	}
+	return e.ExtractJSONObject(jsonObj)
+}
 
+// ExtractJSONObject runs the compiled jq programs against an already
+// unmarshaled JSON value.
+func (e *Extractor) ExtractJSONObject(jsonObj interface{}) map[string]struct{} {
+	results := make(map[string]struct{})
 	for _, k := range e.jsonCompiled {
 		iter := k.Run(jsonObj)
 		for {

pkg/operators/matchers/match.go

@@ -7,8 +7,10 @@ import (
 	"github.com/antchfx/htmlquery"
 	"github.com/antchfx/xmlquery"
 	"github.com/projectdiscovery/govaluate"
+	"golang.org/x/net/html"
 
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/bodycache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
 	stringsutil "github.com/projectdiscovery/utils/strings"
@@ -52,9 +54,22 @@ func (matcher *Matcher) MatchSize(length int) bool {
 }
 
 // MatchWords matches a word check against a corpus.
+//
+// For case-insensitive matchers we look up the lower-cased corpus in the
+// per-response bodycache (when present) so that multiple word matchers
+// against the same body avoid redundant ToLower allocations. Words
+// themselves are lower-cased once at compile time.
 func (matcher *Matcher) MatchWords(corpus string, data map[string]interface{}) (bool, []string) {
 	if matcher.CaseInsensitive {
-		corpus = strings.ToLower(corpus)
+		if data != nil {
+			if bc, ok := data[bodycache.Key].(*bodycache.Cache); ok && bc != nil {
+				corpus = bc.Lowered(corpus)
+			} else {
+				corpus = strings.ToLower(corpus)
+			}
+		} else {
+			corpus = strings.ToLower(corpus)
+		}
 	}
 
 	var matchedWords []string
@@ -253,25 +268,32 @@ func (matcher *Matcher) MatchXPath(corpus string) bool {
 	return matcher.MatchHTML(corpus)
 }
 
-// MatchHTML matches items from HTML using XPath selectors
+// MatchHTML matches items from HTML using XPath selectors. Parses corpus on
+// every call. Callers that already have a parsed node should prefer
+// MatchHTMLNode to avoid redundant parses (see pkg/operators/common/bodycache).
 func (matcher *Matcher) MatchHTML(corpus string) bool {
 	doc, err := htmlquery.Parse(strings.NewReader(corpus))
 	if err != nil {
 		return false
 	}
+	return matcher.MatchHTMLNode(doc)
+}
 
+// MatchHTMLNode matches items from a pre-parsed HTML node using XPath
+// selectors. Used by the protocol Match dispatch in conjunction with the
+// per-response body cache so multiple xpath matchers against the same
+// response share one parse.
+func (matcher *Matcher) MatchHTMLNode(doc *html.Node) bool {
+	if doc == nil {
+		return false
+	}
 	matches := 0
-
 	for _, k := range matcher.XPath {
 		nodes, err := htmlquery.QueryAll(doc, k)
 		if err != nil {
 			continue
 		}
-
-		// Continue if the xpath doesn't return any nodes
 		if len(nodes) == 0 {
-			// If we are in an AND request and a match failed,
-			// return false as the AND condition fails on any single mismatch.
 			switch matcher.condition {
 			case ANDCondition:
 				return false
@@ -279,36 +301,38 @@ func (matcher *Matcher) MatchHTML(corpus string) bool {
 				continue
 			}
 		}
-
-		// If the condition was an OR, return on the first match.
 		if matcher.condition == ORCondition && !matcher.MatchAll {
 			return true
 		}
-
 		matches = matches + len(nodes)
 	}
 	return matches > 0
 }
 
-// MatchXML matches items from XML using XPath selectors
+// MatchXML matches items from XML using XPath selectors. Parses corpus on
+// every call. Callers that already have a parsed node should prefer
+// MatchXMLNode to avoid redundant parses.
 func (matcher *Matcher) MatchXML(corpus string) bool {
 	doc, err := xmlquery.Parse(strings.NewReader(corpus))
 	if err != nil {
 		return false
 	}
+	return matcher.MatchXMLNode(doc)
+}
 
+// MatchXMLNode matches items from a pre-parsed XML node using XPath
+// selectors.
+func (matcher *Matcher) MatchXMLNode(doc *xmlquery.Node) bool {
+	if doc == nil {
+		return false
+	}
 	matches := 0
-
 	for _, k := range matcher.XPath {
 		nodes, err := xmlquery.QueryAll(doc, k)
 		if err != nil {
 			continue
 		}
-
-		// Continue if the xpath doesn't return any nodes
 		if len(nodes) == 0 {
-			// If we are in an AND request and a match failed,
-			// return false as the AND condition fails on any single mismatch.
 			switch matcher.condition {
 			case ANDCondition:
 				return false
@@ -316,14 +340,11 @@ func (matcher *Matcher) MatchXML(corpus string) bool {
 				continue
 			}
 		}
-
-		// If the condition was an OR, return on the first match.
 		if matcher.condition == ORCondition && !matcher.MatchAll {
 			return true
 		}
 		matches = matches + len(nodes)
 	}
-
 	return matches > 0
 }
 

pkg/output/multi_writer.go

@@ -1,6 +1,8 @@
 package output
 
 import (
+	"sync"
+
 	"github.com/logrusorgru/aurora/v4"
 )
 
@@ -30,22 +32,67 @@ func (mw *MultiWriter) Colorizer() *aurora.Aurora {
 	return aurora.New(aurora.WithColors(false))
 }
 
+// Write fans out the event to all underlying writers concurrently. The local
+// file/stdout writer typically completes in microseconds while remote writers
+// (e.g. PDCP upload) can take hundreds of milliseconds; running them serially
+// gates the fast path on the slowest writer. With concurrent fan-out the
+// caller waits at most max(writer latencies) instead of sum.
+//
+// Behavior change vs. previous implementation: errors no longer short-circuit
+// the remaining writers. All writers are attempted; the first observed error
+// is returned. This is intentional: a transient PDCP upload failure should
+// not silently drop the local file/json result.
 func (mw *MultiWriter) Write(event *ResultEvent) error {
-	for _, writer := range mw.writers {
-		if err := writer.Write(event); err != nil {
-			return err
-		}
+	if len(mw.writers) == 1 {
+		return mw.writers[0].Write(event)
 	}
-	return nil
+	var (
+		mu       sync.Mutex
+		firstErr error
+		wg       sync.WaitGroup
+	)
+	wg.Add(len(mw.writers))
+	for _, w := range mw.writers {
+		go func(w Writer) {
+			defer wg.Done()
+			if err := w.Write(event); err != nil {
+				mu.Lock()
+				if firstErr == nil {
+					firstErr = err
+				}
+				mu.Unlock()
+			}
+		}(w)
+	}
+	wg.Wait()
+	return firstErr
 }
 
+// WriteFailure mirrors Write semantics: fan-out and collect the first error.
 func (mw *MultiWriter) WriteFailure(event *InternalWrappedEvent) error {
-	for _, writer := range mw.writers {
-		if err := writer.WriteFailure(event); err != nil {
-			return err
-		}
+	if len(mw.writers) == 1 {
+		return mw.writers[0].WriteFailure(event)
 	}
-	return nil
+	var (
+		mu       sync.Mutex
+		firstErr error
+		wg       sync.WaitGroup
+	)
+	wg.Add(len(mw.writers))
+	for _, w := range mw.writers {
+		go func(w Writer) {
+			defer wg.Done()
+			if err := w.WriteFailure(event); err != nil {
+				mu.Lock()
+				if firstErr == nil {
+					firstErr = err
+				}
+				mu.Unlock()
+			}
+		}(w)
+	}
+	wg.Wait()
+	return firstErr
 }
 
 func (mw *MultiWriter) Request(templateID, url, requestType string, err error) {

pkg/protocols/common/expressions/cache.go

@@ -0,0 +1,99 @@
+package expressions
+
+import (
+	"strings"
+	"sync"
+
+	"github.com/projectdiscovery/gcache"
+	"github.com/projectdiscovery/govaluate"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators/cache"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
+)
+
+// badExprCacheCap caps the negative-result cache (expressions that fail to compile).
+// Keeping it small bounds memory while still preventing repeated parse work for
+// pathological input that appears repeatedly in the same scan.
+const badExprCacheCap = 2048
+
+// bareExprCacheCap caps the cache for govaluate.NewEvaluableExpression (no helper functions).
+// This is hit from isExpression which discriminates whether a {{...}} substring is a valid
+// govaluate expression. Hot in DSL-heavy templates.
+const bareExprCacheCap = 4096
+
+var (
+	badExprOnce  sync.Once
+	badExprCache gcache.Cache[string, struct{}]
+
+	bareExprOnce  sync.Once
+	bareExprCache gcache.Cache[string, *govaluate.EvaluableExpression]
+)
+
+func badExpr() gcache.Cache[string, struct{}] {
+	badExprOnce.Do(func() {
+		badExprCache = gcache.New[string, struct{}](badExprCacheCap).LRU().Build()
+	})
+	return badExprCache
+}
+
+func bareExpr() gcache.Cache[string, *govaluate.EvaluableExpression] {
+	bareExprOnce.Do(func() {
+		bareExprCache = gcache.New[string, *govaluate.EvaluableExpression](bareExprCacheCap).LRU().Build()
+	})
+	return bareExprCache
+}
+
+// compileExprWithFunctions compiles a govaluate expression with the DSL helper
+// function table, caching both successes (in the shared dsl cache) and
+// failures (in a small local negative cache). Callers in this package should
+// use this helper instead of calling govaluate.NewEvaluableExpressionWithFunctions
+// directly so per-request hot paths do not pay parse cost on every invocation.
+func compileExprWithFunctions(src string) (*govaluate.EvaluableExpression, error) {
+	c := cache.DSL()
+	if compiled, err := c.GetIFPresent(src); err == nil && compiled != nil {
+		return compiled, nil
+	}
+	if _, err := badExpr().GetIFPresent(src); err == nil {
+		// Known-bad: re-run the parser to surface the original error message.
+		// This is rare; the cache prevents the common case of repeated parse work.
+		return govaluate.NewEvaluableExpressionWithFunctions(src, dsl.HelperFunctions)
+	}
+	compiled, err := govaluate.NewEvaluableExpressionWithFunctions(src, dsl.HelperFunctions)
+	if err != nil {
+		_ = badExpr().Set(src, struct{}{})
+		return nil, err
+	}
+	_ = c.Set(src, compiled)
+	return compiled, nil
+}
+
+// compileBareExpr compiles a govaluate expression without DSL helpers, used by
+// isExpression to discriminate between bare expressions and helper-function
+// invocations. Cached separately because it has different parse semantics
+// from the with-functions variant (functions are unknown identifiers).
+func compileBareExpr(src string) (*govaluate.EvaluableExpression, error) {
+	c := bareExpr()
+	if compiled, err := c.GetIFPresent(src); err == nil && compiled != nil {
+		return compiled, nil
+	}
+	compiled, err := govaluate.NewEvaluableExpression(src)
+	if err != nil {
+		return nil, err
+	}
+	_ = c.Set(src, compiled)
+	return compiled, nil
+}
+
+// baseHasAnyKey reports whether any key in base appears as a substring in
+// data. Replaces an O(len(base)) []string allocation that getFunctionsNames
+// previously incurred on every isExpression call.
+func baseHasAnyKey(data string, base map[string]interface{}) bool {
+	for k := range base {
+		if k == "" {
+			continue
+		}
+		if strings.Contains(data, k) {
+			return true
+		}
+	}
+	return false
+}

pkg/protocols/common/expressions/expressions.go

@@ -14,7 +14,7 @@ import (
 
 // Eval compiles the given expression and evaluate it with the given values preserving the return type
 func Eval(expression string, values map[string]interface{}) (interface{}, error) {
-	compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions)
+	compiled, err := compileExprWithFunctions(expression)
 	if err != nil {
 		return nil, err
 	}
@@ -58,7 +58,7 @@ func evaluate(data string, base map[string]interface{}) (string, error) {
 		expression = replacer.Replace(expression, base)
 
 		// turns expressions (either helper functions+base values or base values)
-		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions)
+		compiled, err := compileExprWithFunctions(expression)
 		if err != nil {
 			return data, fmt.Errorf("failed to compile expression %q: %w", originalExpression, err)
 		}
@@ -154,15 +154,15 @@ func isExpression(data string, base map[string]interface{}) bool {
 		return false
 	}
 
-	if _, err := govaluate.NewEvaluableExpression(data); err == nil {
-		if stringsutil.ContainsAny(data, getFunctionsNames(base)...) {
+	if _, err := compileBareExpr(data); err == nil {
+		if baseHasAnyKey(data, base) {
 			return true
 		} else if stringsutil.ContainsAny(data, dsl.FunctionNames...) {
 			return true
 		}
 		return false
 	}
-	_, err := govaluate.NewEvaluableExpressionWithFunctions(data, dsl.HelperFunctions)
+	_, err := compileExprWithFunctions(data)
 	return err == nil
 }
 
@@ -196,11 +196,3 @@ func unresolvedVarMarkers(vars []string, base map[string]any) string {
 	}
 	return strings.Join(markers, "")
 }
-
-func getFunctionsNames(m map[string]interface{}) []string {
-	keys := make([]string, 0, len(m))
-	for k := range m {
-		keys = append(keys, k)
-	}
-	return keys
-}

pkg/protocols/common/expressions/variables.go

@@ -4,9 +4,6 @@ import (
 	"errors"
 	"regexp"
 	"strings"
-
-	"github.com/projectdiscovery/govaluate"
-	"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
 )
 
 var (
@@ -116,7 +113,7 @@ func ContainsVariablesWithIgnoreList(skipNames map[string]interface{}, items ...
 }
 
 func hasLiteralsOnly(data string) bool {
-	expr, err := govaluate.NewEvaluableExpressionWithFunctions(data, dsl.HelperFunctions)
+	expr, err := compileExprWithFunctions(data)
 	if err != nil {
 		return false
 	}

pkg/protocols/common/generators/maps.go

@@ -5,43 +5,71 @@ import (
 	"reflect"
 )
 
-// MergeMapsMany merges many maps into a new map
+// MergeMapsMany merges many maps into a new map.
+//
+// Fast paths handle the two map shapes the codebase actually passes:
+// map[string][]string and map[string]interface{}. Other shapes fall back to
+// the reflect-based implementation. This avoids per-key reflect.Value
+// allocation on the HTTP operator-callback hot path
+// (pkg/protocols/http/request.go) which fires on every successful response.
 func MergeMapsMany(maps ...interface{}) map[string][]string {
 	m := make(map[string][]string)
+	appendToSlice := func(key, value string) {
+		m[key] = append(m[key], value)
+	}
 	for _, gotMap := range maps {
-		val := reflect.ValueOf(gotMap)
-		if val.Kind() != reflect.Map {
-			continue
-		}
-		appendToSlice := func(key, value string) {
-			if values, ok := m[key]; !ok {
-				m[key] = []string{value}
-			} else {
-				m[key] = append(values, value)
+		switch typed := gotMap.(type) {
+		case map[string][]string:
+			for k, vs := range typed {
+				m[k] = append(m[k], vs...)
 			}
-		}
-		for _, e := range val.MapKeys() {
-			v := val.MapIndex(e)
-			switch v.Kind() {
-			case reflect.Slice, reflect.Array:
-				for i := 0; i < v.Len(); i++ {
-					appendToSlice(e.String(), v.Index(i).String())
-				}
-			case reflect.String:
-				appendToSlice(e.String(), v.String())
-			case reflect.Interface:
-				switch data := v.Interface().(type) {
+		case map[string]interface{}:
+			for k, v := range typed {
+				switch data := v.(type) {
 				case string:
-					appendToSlice(e.String(), data)
+					appendToSlice(k, data)
 				case []string:
 					for _, value := range data {
-						appendToSlice(e.String(), value)
+						appendToSlice(k, value)
 					}
 				}
 			}
+		case map[string]string:
+			for k, v := range typed {
+				appendToSlice(k, v)
+			}
+		default:
+			mergeMapsManyReflect(m, gotMap, appendToSlice)
+		}
+	}
+	return m
+}
+
+func mergeMapsManyReflect(m map[string][]string, gotMap interface{}, appendToSlice func(string, string)) {
+	val := reflect.ValueOf(gotMap)
+	if val.Kind() != reflect.Map {
+		return
+	}
+	for _, e := range val.MapKeys() {
+		v := val.MapIndex(e)
+		switch v.Kind() {
+		case reflect.Slice, reflect.Array:
+			for i := 0; i < v.Len(); i++ {
+				appendToSlice(e.String(), v.Index(i).String())
+			}
+		case reflect.String:
+			appendToSlice(e.String(), v.String())
+		case reflect.Interface:
+			switch data := v.Interface().(type) {
+			case string:
+				appendToSlice(e.String(), data)
+			case []string:
+				for _, value := range data {
+					appendToSlice(e.String(), value)
+				}
+			}
 		}
 	}
-	return m
 }
 
 // MergeMaps merges multiple maps into a new map.

pkg/protocols/common/replacer/replacer.go

@@ -1,6 +1,7 @@
 package replacer
 
 import (
+	"io"
 	"strings"
 
 	"github.com/projectdiscovery/fasttemplate"
@@ -10,14 +11,29 @@ import (
 )
 
 // Replace replaces placeholders in template with values on the fly.
+//
+// This avoids the previous behavior of eagerly stringifying every value in the
+// map for each call (which allocated a copy of the map on every per-request
+// evaluate path) and skips the second-pass `§§` substitution when the marker
+// is not present in the input.
 func Replace(template string, values map[string]interface{}) string {
-	valuesMap := make(map[string]interface{}, len(values))
-	for k, v := range values {
-		valuesMap[k] = types.ToString(v)
+	// Fast path: no placeholders at all (cheaper than fasttemplate's own check
+	// because it avoids constructing the closure).
+	hasParenOpen := strings.Contains(template, marker.ParenthesisOpen)
+	hasGeneral := strings.Contains(template, marker.General)
+	if !hasParenOpen && !hasGeneral {
+		return template
 	}
-	replaced := fasttemplate.ExecuteStringStd(template, marker.ParenthesisOpen, marker.ParenthesisClose, valuesMap)
-	final := fasttemplate.ExecuteStringStd(replaced, marker.General, marker.General, valuesMap)
-	return final
+
+	if hasParenOpen {
+		tagFn := makeStdTagFunc(values, marker.ParenthesisOpen, marker.ParenthesisClose)
+		template = fasttemplate.ExecuteFuncString(template, marker.ParenthesisOpen, marker.ParenthesisClose, tagFn)
+	}
+	if hasGeneral {
+		gtagFn := makeStdTagFunc(values, marker.General, marker.General)
+		template = fasttemplate.ExecuteFuncString(template, marker.General, marker.General, gtagFn)
+	}
+	return template
 }
 
 // Replace replaces one placeholder in template with one value on the fly.
@@ -30,3 +46,54 @@ func ReplaceOne(template string, key string, value interface{}) string {
 func replaceOneWithMarkers(template, key string, value interface{}, openMarker, closeMarker string) string {
 	return strings.Replace(template, openMarker+key+closeMarker, types.ToString(value), 1)
 }
+
+// makeStdTagFunc returns a fasttemplate.TagFunc that lazily resolves tags
+// against the supplied map and stringifies hits on demand. Unknown tags are
+// re-emitted with their original markers, mirroring fasttemplate's
+// keepUnknownTagFunc behavior. This avoids allocating an intermediate
+// `map[string]interface{}` and avoids eagerly stringifying values that the
+// template never references.
+func makeStdTagFunc(values map[string]interface{}, startTag, endTag string) fasttemplate.TagFunc {
+	return func(w io.Writer, tag string) (int, error) {
+		// Honor fasttemplate's nested startTag handling: if the tag itself
+		// contains another startTag (e.g. "foo{{bar"), emit the literal
+		// "{{" + leading prefix and only resolve the trailing portion. This
+		// matches keepUnknownTagFunc semantics so behavior is preserved.
+		if i := strings.LastIndex(tag, startTag); i >= 0 {
+			if _, err := io.WriteString(w, startTag); err != nil {
+				return 0, err
+			}
+			if _, err := io.WriteString(w, tag[:i]); err != nil {
+				return 0, err
+			}
+			tag = tag[i+len(startTag):]
+		}
+		v, ok := values[tag]
+		if !ok {
+			n, err := io.WriteString(w, startTag)
+			if err != nil {
+				return n, err
+			}
+			n2, err := io.WriteString(w, tag)
+			n += n2
+			if err != nil {
+				return n, err
+			}
+			n3, err := io.WriteString(w, endTag)
+			return n + n3, err
+		}
+		if v == nil {
+			return 0, nil
+		}
+		switch value := v.(type) {
+		case []byte:
+			return w.Write(value)
+		case string:
+			return io.WriteString(w, value)
+		case fasttemplate.TagFunc:
+			return value(w, tag)
+		default:
+			return io.WriteString(w, types.ToString(v))
+		}
+	}
+}

pkg/protocols/http/build_request.go

@@ -116,29 +116,37 @@ func (gr *generatedRequest) setReqURLPattern(reqURLPattern string) {
 }
 
 // ApplyAuth applies the auth provider to the generated request
-func (g *generatedRequest) ApplyAuth(provider authprovider.AuthProvider) {
+// ApplyAuth applies any matching auth strategies onto the generated request
+// and reports whether the request was potentially mutated (i.e. at least one
+// strategy matched). Callers use the return value to decide whether a request
+// dump taken before this call is still accurate.
+func (g *generatedRequest) ApplyAuth(provider authprovider.AuthProvider) bool {
 	if provider == nil {
-		return
+		return false
 	}
+	var applied bool
 	if g.request != nil {
 		authStrategies := provider.LookupURLX(g.request.URL)
 		for _, strategy := range authStrategies {
 			strategy.ApplyOnRR(g.request)
+			applied = true
 		}
 	}
 	if g.rawRequest != nil {
 		parsed, err := urlutil.ParseAbsoluteURL(g.rawRequest.FullURL, true)
 		if err != nil {
 			gologger.Warning().Msgf("[authprovider] Could not parse URL %s: %s\n", g.rawRequest.FullURL, err)
-			return
+			return applied
 		}
 		authStrategies := provider.LookupURLX(parsed)
 		// here we need to apply it custom because we don't have a standard/official
 		// rawhttp request format ( which we probably should have )
 		for _, strategy := range authStrategies {
 			g.rawRequest.ApplyAuthStrategy(strategy)
+			applied = true
 		}
 	}
+	return applied
 }
 
 func (g *generatedRequest) URL() string {

pkg/protocols/http/http.go

@@ -22,6 +22,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
@@ -140,14 +141,16 @@ type Request struct {
 
 	CompiledOperators *operators.Operators `yaml:"-" json:"-"`
 
-	options           *protocols.ExecutorOptions
-	connConfiguration *httpclientpool.Configuration
-	totalRequests     int
-	customHeaders     map[string]string
-	generator         *generators.PayloadGenerator // optional, only enabled when using payloads
-	httpClient        *retryablehttp.Client
-	rawhttpClient     *rawhttp.Client
-	dialer            *fastdialer.Dialer
+	options             *protocols.ExecutorOptions
+	connConfiguration   *httpclientpool.Configuration
+	totalRequests       int
+	customHeaders       map[string]string
+	generator           *generators.PayloadGenerator // optional, only enabled when using payloads
+	httpClient          *retryablehttp.Client
+	rawhttpClient       *rawhttp.Client
+	dialer              *fastdialer.Dialer
+	hasInteractMatchers bool // cached interactsh.HasMatchers(CompiledOperators) computed at Compile time
+	needsRequestEvent   bool // cached hasInteractMatchers && NeedsRequestCondition()
 
 	// description: |
 	//   SelfContained specifies if the request is self-contained.
@@ -398,6 +401,10 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 		}
 		request.CompiledOperators = compiled
 	}
+	// Cache interactsh.HasMatchers / NeedsRequestCondition results so per-request
+	// hot paths skip the linear matcher/extractor scan (interactsh.go ~374-394).
+	request.hasInteractMatchers = interactsh.HasMatchers(request.CompiledOperators)
+	request.needsRequestEvent = request.hasInteractMatchers && request.NeedsRequestCondition()
 
 	// === fuzzing filters ===== //
 

pkg/protocols/http/httputils/spm.go

@@ -40,6 +40,21 @@ type StopAtFirstMatchHandler[T comparable] struct {
 	maxResults  int
 }
 
+// resultChanBuffer returns the result channel buffer for a SPM handler given
+// the pool size. We size to the pool so workers can deposit results without
+// blocking when the consumer goroutine is briefly slow. Bounded above by 64
+// to keep memory predictable for very large pools.
+func resultChanBuffer(size int) int {
+	switch {
+	case size <= 1:
+		return 1
+	case size > 64:
+		return 64
+	default:
+		return size
+	}
+}
+
 // NewBlockingSPMHandler creates a new stop at first match handler
 func NewBlockingSPMHandler[T comparable](ctx context.Context, size int, maxResults int, spm bool) *StopAtFirstMatchHandler[T] {
 	ctx1, cancel := context.WithCancel(ctx)
@@ -47,7 +62,7 @@ func NewBlockingSPMHandler[T comparable](ctx context.Context, size int, maxResul
 	awg, _ := syncutil.New(syncutil.WithSize(size))
 
 	s := &StopAtFirstMatchHandler[T]{
-		ResultChan:  make(chan T, 1),
+		ResultChan:  make(chan T, resultChanBuffer(size)),
 		poolType:    Blocking,
 		sgPool:      awg,
 		internalWg:  &sync.WaitGroup{},
@@ -66,7 +81,7 @@ func NewBlockingSPMHandler[T comparable](ctx context.Context, size int, maxResul
 func NewNonBlockingSPMHandler[T comparable](ctx context.Context, maxResults int, spm bool) *StopAtFirstMatchHandler[T] {
 	ctx1, cancel := context.WithCancel(ctx)
 	s := &StopAtFirstMatchHandler[T]{
-		ResultChan:  make(chan T, 1),
+		ResultChan:  make(chan T, 16),
 		poolType:    NonBlocking,
 		wgPool:      &sync.WaitGroup{},
 		internalWg:  &sync.WaitGroup{},

pkg/protocols/http/operators.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/model"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/bodycache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
@@ -43,7 +44,21 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
 	case matchers.DSLMatcher:
 		return matcher.Result(matcher.MatchDSL(data)), []string{}
 	case matchers.XPathMatcher:
-		return matcher.Result(matcher.MatchXPath(item)), []string{}
+		// Share the parsed HTML/XML tree across all xpath matchers + extractors
+		// targeting the same response via bodycache.
+		cache := bodycache.From(data)
+		if strings.HasPrefix(item, "<?xml") {
+			doc, err := cache.XMLNode(item)
+			if err != nil {
+				return false, []string{}
+			}
+			return matcher.Result(matcher.MatchXMLNode(doc)), []string{}
+		}
+		doc, err := cache.HTMLNode(item)
+		if err != nil {
+			return false, []string{}
+		}
+		return matcher.Result(matcher.MatchHTMLNode(doc)), []string{}
 	}
 	return false, []string{}
 }
@@ -72,9 +87,28 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
 	case extractors.KValExtractor:
 		return extractor.ExtractKval(data)
 	case extractors.XPathExtractor:
-		return extractor.ExtractXPath(item)
+		// Share parsed HTML/XML across xpath extractors and matchers via bodycache.
+		cache := bodycache.From(data)
+		if strings.HasPrefix(item, "<?xml") {
+			doc, err := cache.XMLNode(item)
+			if err != nil {
+				return make(map[string]struct{})
+			}
+			return extractor.ExtractXMLNode(doc)
+		}
+		doc, err := cache.HTMLNode(item)
+		if err != nil {
+			return make(map[string]struct{})
+		}
+		return extractor.ExtractHTMLNode(doc)
 	case extractors.JSONExtractor:
-		return extractor.ExtractJSON(item)
+		// Share unmarshaled JSON value across json extractors via bodycache.
+		cache := bodycache.From(data)
+		obj, err := cache.JSONObject(item)
+		if err != nil {
+			return make(map[string]struct{})
+		}
+		return extractor.ExtractJSONObject(obj)
 	case extractors.DSLExtractor:
 		return extractor.ExtractDSL(data)
 	}

pkg/protocols/http/request.go

@@ -269,8 +269,9 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 					continue
 				}
 				request.options.RateLimitTake()
-				hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
-				needsRequestEvent := hasInteractMatchers && request.NeedsRequestCondition()
+				// Cached at Compile time; avoids walking the matcher/extractor graph per worker iteration.
+				hasInteractMatchers := request.hasInteractMatchers
+				needsRequestEvent := request.needsRequestEvent
 				select {
 				case <-spmHandler.Done():
 					spmHandler.Release()
@@ -531,7 +532,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 	for {
 		// returns two values, error and skip, which skips the execution for the request instance.
 		executeFunc := func(data string, payloads, dynamicValue map[string]interface{}) (bool, error) {
-			hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
+			// Cached at Compile time.
+			hasInteractMatchers := request.hasInteractMatchers
 
 			request.options.RateLimitTake()
 
@@ -569,7 +571,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 			execReqErr := request.executeRequest(input, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
 				// a special case where operators has interactsh matchers and multiple request are made
 				// ex: status_code_2 , interactsh_protocol (from 1st request) etc
-				needsRequestEvent := interactsh.HasMatchers(request.CompiledOperators) && request.NeedsRequestCondition()
+				// Cached at Compile time.
+				needsRequestEvent := request.needsRequestEvent
 				if (hasInteractMarkers || needsRequestEvent) && request.options.Interactsh != nil {
 					requestData := &interactsh.RequestData{
 						MakeResultFunc: request.MakeResultEvent,
@@ -753,8 +756,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 	}
 
 	// === apply auth strategies ===
+	var authApplied bool
 	if generatedRequest.request != nil && !request.SkipSecretFile {
-		generatedRequest.ApplyAuth(request.options.AuthProvider)
+		authApplied = generatedRequest.ApplyAuth(request.options.AuthProvider)
 	}
 
 	var formedURL string
@@ -871,15 +875,23 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 	// converts whitespace and other chars that cannot be printed to url encoded values
 	formedURL = urlutil.URLEncodeWithEscapes(formedURL)
 
-	// Dump the requests containing all headers
-	if !generatedRequest.original.Race {
-		var dumpError error
-		dumpedRequest, dumpError = dump(generatedRequest, input.MetaInput.Input)
+	// Re-dump the request after auth so the value carried into the result
+	// event (and any debug/store output) reflects headers applied by
+	// ApplyAuth(). The pre-flight dump above runs before auth, so we must
+	// refresh dumpedRequest whenever auth actually mutated the request.
+	//
+	// The dump is expensive, so when auth did not touch the request we reuse
+	// the pre-flight dump and only re-run it when a debug/store consumer
+	// needs to emit it.
+	needsDebugDump := request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.StoreResponse
+	if !generatedRequest.original.Race && (authApplied || needsDebugDump) {
+		postAuthDump, dumpError := dump(generatedRequest, input.MetaInput.Input)
 		if dumpError != nil {
 			return dumpError
 		}
-		dumpedRequestString := string(dumpedRequest)
-		if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.StoreResponse {
+		dumpedRequest = postAuthDump
+		if needsDebugDump {
+			dumpedRequestString := string(dumpedRequest)
 			msg := fmt.Sprintf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, formedURL)
 
 			if request.options.Options.Debug || request.options.Options.DebugRequests {

v4.md

@@ -0,0 +1,469 @@
+# Nuclei v4 — Architectural Direction
+
+This is a working design note, not a release plan. It captures the structural
+changes that justify a major-version bump for nuclei — the ones that an
+additive performance branch (like `feat-performance-improvements`) cannot
+accomplish because they break the public surface, the on-disk format, the
+SDK, or the template schema.
+
+The criterion for inclusion is deliberately strict: a change earns a slot
+here only if it is **breaking** *and* unlocks a step-change (5–10× perf, a
+new capability, or removal of a load-bearing constraint). Incremental wins
+belong in feature branches against `dev`.
+
+## Why v4 (one paragraph)
+
+v3 has accumulated load-bearing constraints that show up as friction every
+time the engine is touched: the event model is a typed-erased
+`map[string]interface{}` shared by every protocol, every matcher and every
+extractor; templates are re-parsed from YAML on every startup; worker pools
+are created per template; reflection sits on the HTTP success callback path;
+the binary is monolithic at ~170 MB. Patching around these costs 1.2–2× at
+best (which is what the perf branch achieved). Removing them — and shipping
+the new contract that comes with their removal — is what v4 is for.
+
+---
+
+## #1 — Typed event model (kill `map[string]interface{}`)
+
+**What changes.** Replace `output.InternalEvent = map[string]interface{}`
+with a typed `Event` struct exposing the load-bearing fields directly:
+
+```go
+type Event struct {
+    URL         *url.URL
+    Method      string
+    StatusCode  int
+    Headers     http.Header
+    Body        []byte
+    Duration    time.Duration
+    Request     []byte                 // raw dump
+    Response    []byte                 // raw dump
+    TemplateID  string
+    TemplateInfo model.Info
+    Matched     string
+    Extracts    map[string]any         // user-defined extracts only
+    Cache       *eventcache.Cache      // body decode cache (today: bodycache)
+}
+```
+
+Matchers, extractors and DSL helpers gain typed accessors over this struct.
+Untyped storage stays only for user-declared `Extracts`.
+
+**Why it's breaking.** SDK consumers read `wrapped.InternalEvent[k]`
+everywhere. DSL helper functions accept and return `interface{}`.
+`MakeResultEvent` is part of every protocol. JSON output schema for
+`-jsonl` would change.
+
+**What it unlocks.**
+- The replacer no longer needs `types.ToString(v)` per access.
+- The body-decode cache stops being keyed by a magic string in a generic map
+  (`__nuclei_bodycache`) and becomes a real field.
+- `MergeMaps`, `MergeMapsMany`, and the entire reflect path collapse into
+  typed merges of `Extracts`.
+- Body becomes `[]byte` not `string`, eliminating the implicit copy at
+  `string(body)`.
+- Matchers can take a `*Event` instead of separately-constructed `corpus`
+  and `data` arguments — the `getMatchPart` shim disappears.
+- DSL gets typed function signatures (`StatusCode int`, not `interface{}`).
+
+**Complexity.** High. Touches every protocol package and the DSL function
+registry. Estimated as the single biggest refactor in v4.
+
+**Prereqs.** None — this is a leaf change. Other v4 items get cleaner if
+this lands first.
+
+---
+
+## #2 — Compiled template artifacts (Template IR + `nuclei compile`)
+
+**What changes.** Introduce a binary template bundle format:
+
+```
+nuclei compile -t templates/ -o templates.nucbin
+nuclei -bundle templates.nucbin -u https://...
+```
+
+The bundle contains:
+- Pre-parsed template ASTs (no YAML at runtime).
+- Pre-compiled regex, jq programs, DSL expressions.
+- Pre-verified template signatures (with the verifier identity recorded).
+- An index over IDs, tags, severity, protocols, paths.
+- A version header (`bundle_format: 1`) and a `nuclei_version` constraint.
+
+Today's directory-based loading remains supported as a fallback for local
+development.
+
+**Why it's breaking.** The bundle format becomes part of the public API.
+Reproducibility expectations change — pinning a bundle hash pins behavior.
+CI flows shift from "fetch templates repo at SHA" to "fetch versioned
+bundle artifact."
+
+**What it unlocks.**
+- Cold start collapses from `O(N × YAML parse + verify + compile)` to
+  `mmap + version check`. Expect 10–50× on large catalogs.
+- Bundles can be signed once and distributed via PDCP / OCI / S3.
+- Long-lived nuclei services (DAST server, scheduled scanners) stop paying
+  the parse cost on every spawn.
+- Deterministic, hash-pinned scans become trivial — bundle hash + target
+  list = full provenance.
+
+**Complexity.** Medium-high. The IR design is the hard part; the rest is
+serialization plumbing.
+
+**Prereqs.** Cleaner with #1 (typed event lets the IR avoid embedding any
+`interface{}` placeholders for runtime types).
+
+---
+
+## #3 — Persistent work-stealing scheduler
+
+**What changes.** Replace the per-template worker pool model in
+`pkg/core/executors.go` with one global scheduler over typed task tuples:
+
+```go
+type Task struct {
+    Template *Template
+    Target   *Target
+    Payload  *Payload
+    Cluster  ClusterID            // for cross-template request dedup
+}
+```
+
+The scheduler has work-stealing queues per worker, adaptive concurrency
+tied to observed RTT and CPU load, first-class backpressure with bounded
+queue depth, and a single shared HTTP client pool.
+
+**Why it's breaking.** Resume semantics change (the `ResumeCfg.InFlight`
+bookkeeping is per-template today). The `Engine` API exposed via
+`pkg/core/` is not stable across this change. SPM (stop-at-first-match)
+handlers move from per-request to per-cluster.
+
+**What it unlocks.**
+- Cross-template request dedup beyond opt-in clustering.
+- Cache locality on hot HTTP clients (no more pool teardown per template).
+- Tail-latency-aware throttling (slow targets get fewer workers).
+- The buffered-tasks-channel patch from `feat-performance-improvements`
+  becomes a non-event — backpressure is built in.
+
+**Complexity.** High. Engine rewrite.
+
+**Prereqs.** Easier with #1 (typed event makes task carrying simpler).
+Composes well with #2 (bundle gives the scheduler a static shape to plan
+against).
+
+---
+
+## #4 — Streaming I/O end-to-end
+
+**What changes.**
+- **Input:** target list streamed reservoir-style from stdin/file/socket;
+  no full pre-load. Memory cost decoupled from `wc -l targets`.
+- **Output:** lock-free SPSC ring per writer; on Linux, optional io_uring
+  backend for the file writer. Today's `MultiWriter` fan-out (already added
+  in `feat-performance-improvements`) becomes redundant.
+- **Bodies:** chunked-body matchers where the operator allows it (regex,
+  word, binary). HTML/XML/JSON still need full body for parsing.
+- **Project cache:** drop LevelDB. The current goroutine dump shows 5+ live
+  goroutines (`tCompaction`, `mCompaction`, `mpoolDrain`, buffer pool drain)
+  for the entire run, even when nothing is stored. Replace with `bbolt`
+  (already a transitive dep) or a tiny purpose-built KV.
+
+**Why it's breaking.** Project file format changes (different KV backend).
+Result ordering guarantees may change (lock-free rings break strict order).
+Writer plugins must adopt the new interface.
+
+**What it unlocks.**
+- Order-of-magnitude reduction in resident memory on big scans.
+- Removal of an entire class of "engine is busy doing GC on goleveldb"
+  background noise.
+
+**Complexity.** Medium. Mostly mechanical once the writer interface
+changes.
+
+**Prereqs.** None.
+
+---
+
+## #5 — WASM extension ABI; demote Goja
+
+**What changes.** Add a wasmtime/wazero ABI for matchers, extractors,
+post-processors, and (eventually) protocols. Templates can declare a
+WASM-backed step:
+
+```yaml
+matchers:
+  - type: wasm
+    module: classifiers/login-page.wasm:v1
+    params: {min_score: 0.85}
+```
+
+Goja remains as a compatibility shim for `code:` / `javascript:` templates
+with deprecated status. New templates target WASM.
+
+**Why it's breaking.** Sandbox model changes. The implicit "JS code can
+shell out to anything Goja exposes" contract is replaced by an explicit
+capability list per WASM module. Some `code:` templates won't be portable.
+
+**What it unlocks.**
+- Deterministic CPU/memory limits per extension call.
+- Extensions in any language that targets WASM (Rust, Zig, AssemblyScript,
+  TinyGo).
+- A safe place to run inference modules (see #8).
+- Removes a class of "Goja crashes the runtime" tickets.
+
+**Complexity.** Medium. Wasmtime/wazero embedding is well-understood;
+designing the ABI surface is the design work.
+
+**Prereqs.** None directly; pairs naturally with #6 (modular protocols).
+
+---
+
+## #6 — Modular protocols, slim default binary
+
+**What changes.** The default `nuclei` binary ships only HTTP, DNS, TCP,
+SSL — the 90%-of-users core. Headless (Chrome deps), Code, JavaScript,
+Fuzz, Network-extras, WebSocket, WHOIS, File become separately-distributed
+modules loaded at runtime.
+
+Module loading uses the WASM ABI from #5 where possible (lighter weight,
+sandboxed) or a minimal Go plugin protocol over Unix-domain RPC for
+modules that need full Go-runtime access (today: headless).
+
+**Why it's breaking.** Installation changes (`nuclei install headless`).
+Templates referencing missing protocols error explicitly instead of
+silently working. Distribution becomes per-module versioned.
+
+**What it unlocks.**
+- Default binary drops from ~170 MB to ~30–40 MB.
+- Independent release cadence per protocol.
+- Smaller blast radius for protocol-specific CVEs in dependencies.
+- Container images shrink significantly (Chrome alone is the bulk of the
+  headless bundle).
+
+**Complexity.** Medium-high. The plugin loader and the module distribution
+infrastructure are the work.
+
+**Prereqs.** #5 makes most of this trivial; some protocols (headless)
+likely need the Go-plugin escape hatch.
+
+---
+
+## #7 — Explicit template schema versioning
+
+**What changes.** Promote `template-schema:` to a load-bearing field. The
+v4 engine compiles v3 templates through a translation layer to v4 IR.
+Schema evolution after v4 follows a documented deprecation process.
+
+```yaml
+id: foo
+template-schema: 4
+info: {...}
+http: [...]
+```
+
+**Why it's breaking.** Sets the precedent that template schema and engine
+version are independently versioned. Some v3-only fields may not have v4
+equivalents and require flagging.
+
+**What it unlocks.**
+- Users can upgrade nuclei without a flag-day for the entire templates
+  repo.
+- Schema can move forward without backward-compat ratchets baked into the
+  parser.
+
+**Complexity.** Low for the field itself; medium for the v3→v4 translator.
+
+**Prereqs.** Has to land *before* any other schema-affecting change in v4.
+
+---
+
+## #8 — Optional onboard inference
+
+**Status: addon, not core.** Inference is a v4-era feature because it adds
+non-deterministic-by-default behavior to a tool whose contract has always
+been deterministic. Treat it as opt-in everywhere, locally-runnable by
+default, and architected so a scan run without it is unchanged from today.
+
+### Where it earns its keep
+
+1. **False-positive triage as a post-matcher.** A small on-device
+   classifier (ONNX MiniLM-class, ~30 MB) runs *only on already-matched
+   responses*, scoring `(template_id, status_code, body_excerpt,
+   header_subset) → likely_fp`. Doesn't change the truth output (matched
+   stays matched); adds a confidence score and an optional
+   `--fp-threshold` filter. Highest leverage, lowest risk.
+
+2. **A new `ai:` matcher / extractor type.** Sits alongside
+   `word`/`regex`/`dsl`/`xpath`:
+
+   ```yaml
+   matchers:
+     - type: ai
+       model: minilm-classifier:v1
+       prompt: is_login_page
+       threshold: 0.8
+   ```
+
+   Outputs are cached by `(model_version, prompt_hash, body_hash)`. The
+   body-decode cache added in `feat-performance-improvements` is the right
+   place for this — extend it with a `Classify(label, body) (bool,
+   float32, error)` slot so multiple `ai:` matchers against the same
+   response share one inference call.
+
+3. **Natural-language template search and synthesis as tooling, not
+   runtime.** New CLI verbs:
+   - `nuclei explain <template-id>` — readable summary.
+   - `nuclei find "reflected XSS via Referer header"` — embedding search
+     over the catalog.
+   - `nuclei draft -from-response request.txt response.txt` — generate
+     template skeleton.
+
+   Pure tooling; doesn't touch the scan engine.
+
+4. **Context-aware fuzz payload mutation.** A small generator conditioned
+   on parameter name/type/observed reflection produces candidates that
+   complement the static dictionary fuzzing. Capped per request to avoid
+   throughput collapse.
+
+5. **Tech-stack inference for unknown fingerprints.** A cheap classifier
+   over headers + body fingerprints enriches `tags`/`metadata` when
+   wappalyzergo doesn't know the stack. Feeds into `-as` (auto-scan).
+
+### What makes it non-trivial in nuclei specifically
+
+- **Determinism is load-bearing.** Today you can re-run a scan and assert
+  identical output — CI, audit, regression all rely on it. Inference
+  breaks this unless we pin: model identity, weights hash, temperature=0,
+  and result cache. Hard contract for v4: `(nuclei_version, model_pin,
+  cached_response_bytes) → identical matcher verdict`. Anything else is
+  opt-in via a clearly labeled flag, and the result event records that
+  inference contributed.
+
+- **Latency budget per response is microseconds.** A model invoked on
+  every body would gut throughput. Two mitigations:
+  - Run inference *only* on already-matched responses (FP triage), so
+    scaling is `O(matches)` not `O(requests)`.
+  - Cache by body hash so multi-template clusters hit the model once.
+
+- **Body content is hostile input.** Prompt injection against a local
+  model is a real concern; against a tool-using NL CLI it's worse. Treat
+  model input as untrusted, never let response content control tool-use
+  paths, and run inference inside the WASM sandbox from #5.
+
+- **Binary size and dependency creep.** ONNX-runtime and llama.cpp via cgo
+  are heavy. This is exactly why #6 (modular protocols) matters — `nuclei
+  ai-pack` becomes an opt-in module, default binary stays slim.
+
+- **Local vs. remote provider abstraction.** Same interface, two backends:
+  - Embedded (default, deterministic, offline): ONNX/llama.cpp with
+    version-pinned tiny models bundled into the `ai-pack` module.
+  - Remote (opt-in, visible cost): PDCP-hosted, OpenAI-compatible, etc.
+    Budget controls (`--ai-budget 1000-tokens`, `--ai-budget 0.05-USD`)
+    and explicit per-template enablement.
+
+### Honest limits
+
+- **Don't ship a generative LLM in the default binary.** Embeddings + small
+  classifiers, yes; a 7B-parameter model, no — that lives in
+  `nuclei-ai-pack`.
+- **Don't pitch CVE discovery via LLM-only matchers.** Inference is a
+  triage and tooling layer over deterministic matchers, not a replacement
+  for them. The "zero false positives" pitch collapses the moment a
+  probabilistic matcher is the only signal for a finding.
+- **Don't let inference results auto-mutate templates.** `nuclei draft` is
+  a starting point for humans; auto-promotion is out of scope.
+
+### How it fits the rest of v4
+
+- Sits on top of #1 (typed event gives the model structured features
+  instead of stringly-typed fields).
+- Bakes model pins into #2 (compiled bundle records `model_version` so a
+  scan is reproducible).
+- Runs inside #5 (WASM ABI sandboxes the inference module and enforces
+  capability limits).
+- Distributed as an optional module per #6.
+
+---
+
+## Prioritization
+
+If only two land, pick #1 and #2:
+
+- **#1 (typed event)** clears the path for #3 (scheduler), #4 (streaming),
+  #5 (WASM ABI), and #8 (inference) — every other item gets cheaper.
+- **#2 (compiled artifacts)** delivers the most user-visible win on its
+  own (cold start) and unlocks reproducible deployments + signed bundle
+  distribution.
+
+Sequencing for the rest:
+
+```
+   #7 (schema versioning) ─── must land first
+                ↓
+   #1 (typed event)       ─── biggest refactor
+                ↓
+   ┌──────────┴─────────┐
+   #2 (IR/bundle)   #4 (streaming I/O)
+        ↓                ↓
+   #3 (scheduler)     #6 (modular protocols)
+                          ↓
+                       #5 (WASM ABI)
+                          ↓
+                       #8 (inference)
+```
+
+#5 → #8 is the canonical AI-as-sandboxed-module path. #6 can land before
+#5 with a Go-plugin loader as a stop-gap.
+
+---
+
+## Non-goals (explicitly *not* v4-worthy)
+
+- Replacing Goja with QuickJS / Boa / a different JS runtime — incremental.
+- HTTP/3 support — protocol-level addition, fits in dev.
+- Better workflow language — incremental.
+- Better error messages — incremental.
+- A Web UI — out of scope for the engine; belongs to a separate product.
+- LLM-generated templates as a default behavior — see Honest limits in #8.
+
+---
+
+## Risks / open questions
+
+- **#1 SDK churn.** Heaviest external impact. Need a clean migration shim
+  for the `lib/` SDK consumers — probably an adapter that constructs a
+  `*Event` from a `map[string]any` for one release, with a deprecation
+  flag.
+- **#2 signature model.** Today templates are signed individually. A
+  bundle introduces a new signing question: per-template signatures
+  carried in the bundle, or a single bundle signature? Probably both
+  (per-template for provenance, bundle for distribution integrity).
+- **#3 backpressure visibility.** A persistent scheduler hides per-template
+  throughput from the user. Need first-class metrics (per-template QPS,
+  pool occupancy) to keep operators happy.
+- **#5 ABI stability.** Once external WASM modules ship, the ABI is
+  effectively a public API. Versioning strategy must be specified before
+  the first module is published.
+- **#6 module distribution.** Where do modules live? PDCP hosting? OCI
+  registry? Plain HTTPS download with manifests? Decision affects supply
+  chain trust model.
+- **#8 inference reproducibility.** Even with temperature=0 and pinned
+  weights, inference engines occasionally diverge across CPU
+  microarchitectures (different SIMD paths, different rounding). Need a
+  bit-exact reference implementation or accept a documented
+  "reproducible-modulo-architecture" caveat.
+
+---
+
+## What's already in flight (informs sequencing)
+
+- **`feat-performance-improvements`** (this branch): 10 additive
+  optimizations targeting v3 hot paths. Median 2.22× on a representative
+  bench. Establishes the body-decode cache that #8 reuses, the typed
+  `MergeMapsMany` paths that #1 generalizes, and the buffered-channel /
+  concurrent-writer wins that #3 / #4 subsume entirely.
+- **In-flight community PRs** (referenced during the perf pass): RFC 9111
+  HTTP response cache, scan strategy improvements, additional protocol
+  hardening. None of these block the v4 items above; most either slot
+  into the v3 line or compose cleanly with v4.