Merge dev

pkg/protocols/common/interactsh/eviction_test.go

@@ -0,0 +1,61 @@
+package interactsh
+
+import (
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEvictionFromSeconds(t *testing.T) {
+	t.Parallel()
+	require.Equal(t, time.Duration(0), EvictionFromSeconds(0))
+	require.Equal(t, time.Duration(0), EvictionFromSeconds(-1))
+	require.Equal(t, 300*time.Second, EvictionFromSeconds(300))
+	require.Equal(t, time.Duration(MaxEvictionSeconds)*time.Second, EvictionFromSeconds(int(MaxEvictionSeconds)))
+	if strconv.IntSize == 64 {
+		require.Positive(t, EvictionFromSeconds(int(MaxEvictionSeconds)+1))
+	}
+}
+
+func TestClientEvictionForUsesRequestOverride(t *testing.T) {
+	t.Parallel()
+
+	client, err := New(&Options{Eviction: 60 * time.Second, CacheSize: 10})
+	require.NoError(t, err)
+
+	require.Equal(t, 60*time.Second, client.evictionFor(nil))
+	require.Equal(t, 60*time.Second, client.evictionFor(&RequestData{}))
+	require.Equal(t, 300*time.Second, client.evictionFor(&RequestData{Eviction: 300 * time.Second}))
+}
+
+func TestRequestEventStoresWithPerRequestEviction(t *testing.T) {
+	t.Parallel()
+
+	client, err := New(&Options{Eviction: 60 * time.Second, CacheSize: 10})
+	require.NoError(t, err)
+	client.hostname = "oast.example"
+
+	id := "abc123"
+	url := id + "." + client.hostname
+	data := &RequestData{
+		Event: &output.InternalWrappedEvent{InternalEvent: map[string]interface{}{
+			templateIdAttribute: "test-template",
+			"host":              "example.com",
+		}},
+		Eviction: 50 * time.Millisecond,
+	}
+
+	client.RequestEvent([]string{url}, data)
+
+	got, err := client.requests.Get(id)
+	require.NoError(t, err)
+	require.Equal(t, data, got)
+
+	require.Eventually(t, func() bool {
+		_, err := client.requests.Get(id)
+		return err != nil
+	}, time.Second, 10*time.Millisecond)
+}

pkg/protocols/common/interactsh/interactsh.go

@@ -3,6 +3,7 @@ package interactsh
 import (
 	"bytes"
 	"fmt"
+	"math"
 	"os"
 	"regexp"
 	"strings"
@@ -509,6 +510,10 @@ type RequestData struct {
 	Parameter string
 	Request   *retryablehttp.Request
 
+	// Eviction overrides the client-wide Interactsh request cache TTL when > 0.
+	// Used for per-template interactsh-eviction.
+	Eviction time.Duration
+
 	// These fields route delayed results back to the execution that registered
 	// the request when the Interactsh client is shared concurrently.
 	Output              output.Writer
@@ -550,13 +555,39 @@ func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) {
 		} else {
 			c.requestLifecycleMu.RLock()
 			if data.Scope.track(id) {
-				_ = c.requests.SetWithExpire(id, data, c.eviction)
+				_ = c.requests.SetWithExpire(id, data, c.evictionFor(data))
 			}
 			c.requestLifecycleMu.RUnlock()
 		}
 	}
 }
 
+// evictionFor returns the request cache TTL for a RequestEvent.
+// Per-request Eviction overrides the client-wide default when set.
+func (c *Client) evictionFor(data *RequestData) time.Duration {
+	if data != nil && data.Eviction > 0 {
+		return data.Eviction
+	}
+	return c.eviction
+}
+
+// MaxEvictionSeconds is the largest eviction value in seconds that fits in a
+// time.Duration.
+const MaxEvictionSeconds = math.MaxInt64 / int64(time.Second)
+
+// EvictionFromSeconds converts a template/CLI eviction value in seconds to a
+// duration. Non-positive values mean "use the client default". Values above
+// MaxEvictionSeconds are capped, so they cannot overflow to a negative duration.
+func EvictionFromSeconds(seconds int) time.Duration {
+	if seconds <= 0 {
+		return 0
+	}
+	if int64(seconds) > MaxEvictionSeconds {
+		return time.Duration(MaxEvictionSeconds) * time.Second
+	}
+	return time.Duration(seconds) * time.Second
+}
+
 // HasMatchers returns true if an operator has interactsh part
 // matchers or extractors.
 //

pkg/protocols/interactsh_eviction_test.go

@@ -0,0 +1,31 @@
+package protocols
+
+import (
+	"testing"
+	"time"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
+	"github.com/stretchr/testify/require"
+)
+
+func TestInteractshEvictionDuration(t *testing.T) {
+	t.Parallel()
+	require.Equal(t, time.Duration(0), (*ExecutorOptions)(nil).InteractshEvictionDuration())
+	require.Equal(t, time.Duration(0), (&ExecutorOptions{}).InteractshEvictionDuration())
+	require.Equal(t, 300*time.Second, (&ExecutorOptions{InteractshEviction: 300}).InteractshEvictionDuration())
+}
+
+func TestRegisterInteractshRequestSetsTemplateEviction(t *testing.T) {
+	t.Parallel()
+
+	client, err := interactsh.New(interactsh.DefaultOptions(nil, nil, nil))
+	require.NoError(t, err)
+
+	data := &interactsh.RequestData{}
+	(&ExecutorOptions{Interactsh: client, InteractshEviction: 300}).RegisterInteractshRequest(nil, data)
+	require.Equal(t, 300*time.Second, data.Eviction)
+
+	data = &interactsh.RequestData{}
+	(&ExecutorOptions{Interactsh: client}).RegisterInteractshRequest(nil, data)
+	require.Equal(t, time.Duration(0), data.Eviction)
+}

pkg/protocols/protocols.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"reflect"
 	"sync/atomic"
+	"time"
 
 	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/gologger"
@@ -124,6 +125,10 @@ type ExecutorOptions struct {
 	// Stop execution once first match is found (Assigned while parsing templates)
 	// Note: this is different from Options.StopAtFirstMatch (Assigned from CLI option)
 	StopAtFirstMatch bool
+	// InteractshEviction is an optional per-template Interactsh request cache
+	// TTL in seconds. When > 0 it overrides the global interactions-eviction
+	// duration for requests from this template only.
+	InteractshEviction int
 	// Variables is a list of variables from template
 	Variables variables.Variable
 	// Constants is a list of constants from template
@@ -200,6 +205,7 @@ func (e *ExecutorOptions) RegisterInteractshRequest(urls []string, data *interac
 	data.IssuesClient = e.IssuesClient
 	data.FuzzParamsFrequency = e.FuzzParamsFrequency
 	data.Scope = e.InteractshScope
+	data.Eviction = e.InteractshEvictionDuration()
 	e.Interactsh.RequestEvent(urls, data)
 }
 
@@ -400,6 +406,7 @@ func (e *ExecutorOptions) Copy() *ExecutorOptions {
 		InteractshScope:              e.InteractshScope,
 		HostBackoff:                  e.HostBackoff,
 		StopAtFirstMatch:             e.StopAtFirstMatch,
+		InteractshEviction:           e.InteractshEviction,
 		Variables:                    e.Variables,
 		Constants:                    e.Constants,
 		ExcludeMatchers:              e.ExcludeMatchers,
@@ -428,6 +435,15 @@ func (e *ExecutorOptions) Copy() *ExecutorOptions {
 	return copy
 }
 
+// InteractshEvictionDuration returns the per-template Interactsh request
+// cache TTL, or 0 to use the client-wide default.
+func (e *ExecutorOptions) InteractshEvictionDuration() time.Duration {
+	if e == nil {
+		return 0
+	}
+	return interactsh.EvictionFromSeconds(e.InteractshEviction)
+}
+
 // CloneOperators returns a copy of operators with only template-authored fields.
 // Compiled matcher/extractor state is rebuilt per execution by Request.Compile.
 func CloneOperators(src operators.Operators) operators.Operators {

pkg/templates/cluster.go

@@ -56,10 +56,12 @@ func Cluster(list []*Template) [][]*Template {
 	// Split up templates that might be clusterable
 	for _, template := range list {
 		// it is not possible to cluster flow and multiprotocol due to dependent execution.
+		// A cluster runs with the options of its first template, so a template
+		// with its own interactsh-eviction would lose that TTL.
 		// Info placeholders are filled from the event a finding came from, and a
 		// cluster shares one event that already holds the first template's variables
 		// and extracted values, so a template with them runs on its own.
-		if template.Flow != "" || template.Options.IsMultiProtocol || template.Info.HasPlaceholders() {
+		if template.Flow != "" || template.Options.IsMultiProtocol || template.InteractshEviction > 0 || template.Info.HasPlaceholders() {
 			final = append(final, []*Template{template})
 			continue
 		}

pkg/templates/cluster_test.go

@@ -77,4 +77,12 @@ func TestClusterTemplates(t *testing.T) {
 		got := Cluster(tpls)[0]
 		require.ElementsMatch(t, got, expected)
 	})
+	t.Run("no-cluster-with-interactsh-eviction", func(t *testing.T) {
+		tp1 := &Template{Path: "first.yaml", RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}}
+		tp2 := &Template{Path: "second.yaml", InteractshEviction: 300, RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}}
+		tp1.Options = execOptions
+		tp2.Options = execOptions
+		got := Cluster([]*Template{tp1, tp2})
+		require.ElementsMatch(t, [][]*Template{{tp1}, {tp2}}, got)
+	})
 }

pkg/templates/compile.go

@@ -20,6 +20,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/offlinehttp"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
@@ -810,6 +811,13 @@ func parseTemplateNoVerify(data []byte, srcOptions *protocols.ExecutorOptions) (
 }
 
 func prepareTemplate(template *Template, srcOptions *protocols.ExecutorOptions) (*Template, error) {
+	if template.InteractshEviction < 0 {
+		return nil, errors.New("interactsh-eviction cannot be negative")
+	}
+	if int64(template.InteractshEviction) > interactsh.MaxEvictionSeconds {
+		return nil, fmt.Errorf("interactsh-eviction cannot be greater than %d", interactsh.MaxEvictionSeconds)
+	}
+
 	// Create a copy of the options specifically for this template.
 	options := srcOptions.Copy()
 
@@ -842,6 +850,7 @@ func prepareTemplate(template *Template, srcOptions *protocols.ExecutorOptions)
 	options.TemplateID = template.ID
 	options.TemplateInfo = template.Info
 	options.StopAtFirstMatch = template.StopAtFirstMatch
+	options.InteractshEviction = template.InteractshEviction
 
 	if template.Variables.Len() > 0 {
 		options.Variables = template.Variables

pkg/templates/interactsh_eviction_test.go

@@ -0,0 +1,55 @@
+package templates
+
+import (
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func TestTemplateInteractshEvictionUnmarshal(t *testing.T) {
+	t.Parallel()
+
+	const raw = `
+id: example-oast
+info:
+  name: example
+  author: pdteam
+  severity: info
+interactsh-eviction: 300
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+`
+	var template Template
+	require.NoError(t, yaml.Unmarshal([]byte(raw), &template))
+	require.Equal(t, 300, template.InteractshEviction)
+
+	parsed, err := parseTemplateNoVerify([]byte(raw), &protocols.ExecutorOptions{Options: types.DefaultOptions()})
+	require.NoError(t, err)
+	require.Equal(t, 300, parsed.Options.InteractshEviction)
+	require.Equal(t, 300*time.Second, parsed.Options.InteractshEvictionDuration())
+}
+
+func TestTemplateInteractshEvictionRejectsNegativeValue(t *testing.T) {
+	t.Parallel()
+
+	_, err := prepareTemplate(&Template{InteractshEviction: -1}, nil)
+	require.EqualError(t, err, "interactsh-eviction cannot be negative")
+}
+
+func TestTemplateInteractshEvictionRejectsOverflow(t *testing.T) {
+	t.Parallel()
+
+	if strconv.IntSize < 64 {
+		t.Skip("int cannot hold a value above the limit")
+	}
+	_, err := prepareTemplate(&Template{InteractshEviction: int(interactsh.MaxEvictionSeconds) + 1}, nil)
+	require.ErrorContains(t, err, "interactsh-eviction cannot be greater than")
+}

pkg/templates/templates.go

@@ -131,6 +131,14 @@ type Template struct {
 	//  Stop execution once first match is found
 	StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" json:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop at first match for the template"`
 
+	// description: |
+	//   InteractshEviction is the number of seconds to keep this template's
+	//   Interactsh request pending for OAST callbacks before eviction.
+	//   When unset, the global -interactions-eviction value is used.
+	// examples:
+	//   - value: "300"
+	InteractshEviction int `yaml:"interactsh-eviction,omitempty" json:"interactsh-eviction,omitempty" jsonschema:"title=interactsh eviction seconds,description=Per-template Interactsh request eviction timeout in seconds,minimum=1"`
+
 	// description: |
 	//   Signature is the request signature method
 	//   WARNING: 'signature' will be deprecated and will be removed in a future release. Prefer using 'code' protocol for writing cloud checks