record workflow steps, guard the record hook, write the corpus 0600, sort extracted values in replay

README.md

@@ -206,6 +206,7 @@ OUTPUT:
    -o, -output string            output file to write found issues/vulnerabilities
    -sresp, -store-resp           store all request/response passed through nuclei to output directory
    -srd, -store-resp-dir string  store all request/response passed through nuclei to custom directory (default "output")
+   -store-events string          store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
    -silent                       display findings only
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -j, -jsonl                    write output in JSONL(ines) format
@@ -240,6 +241,7 @@ CONFIGURATIONS:
    -sr, -system-resolvers                use system DNS resolving as error fallback
    -dc, -disable-clustering              disable clustering of requests
    -passive                              enable passive HTTP response processing mode
+   -replay string                        run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
    -fh2, -force-http2                    force http2 connection on requests
    -ev, -env-vars                        enable environment variables to be used in template
    -cc, -client-cert string              client certificate file (PEM-encoded) used for authenticating against scanned hosts

README_CN.md

@@ -204,6 +204,7 @@ OUTPUT:
    -o, -output string            output file to write found issues/vulnerabilities
    -sresp, -store-resp           store all request/response passed through nuclei to output directory
    -srd, -store-resp-dir string  store all request/response passed through nuclei to custom directory (default "output")
+   -store-events string          store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
    -silent                       display findings only
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -j, -jsonl                    write output in JSONL(ines) format
@@ -237,6 +238,7 @@ CONFIGURATIONS:
    -sr, -system-resolvers                use system DNS resolving as error fallback
    -dc, -disable-clustering              disable clustering of requests
    -passive                              enable passive HTTP response processing mode
+   -replay string                        run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
    -fh2, -force-http2                    force http2 connection on requests
    -ev, -env-vars                        enable environment variables to be used in template
    -cc, -client-cert string              client certificate file (PEM-encoded) used for authenticating against scanned hosts

README_ES.md

@@ -204,6 +204,7 @@ OUTPUT:
    -o, -output string            output file to write found issues/vulnerabilities
    -sresp, -store-resp           store all request/response passed through nuclei to output directory
    -srd, -store-resp-dir string  store all request/response passed through nuclei to custom directory (default "output")
+   -store-events string          store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
    -silent                       display findings only
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -j, -jsonl                    write output in JSONL(ines) format
@@ -237,6 +238,7 @@ CONFIGURATIONS:
    -sr, -system-resolvers                use system DNS resolving as error fallback
    -dc, -disable-clustering              disable clustering of requests
    -passive                              enable passive HTTP response processing mode
+   -replay string                        run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
    -fh2, -force-http2                    force http2 connection on requests
    -ev, -env-vars                        enable environment variables to be used in template
    -cc, -client-cert string              client certificate file (PEM-encoded) used for authenticating against scanned hosts

README_FA.md

@@ -221,6 +221,7 @@ OUTPUT:
    -o, -output string            output file to write found issues/vulnerabilities
    -sresp, -store-resp           store all request/response passed through nuclei to output directory
    -srd, -store-resp-dir string  store all request/response passed through nuclei to custom directory (default "output")
+   -store-events string          store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
    -silent                       display findings only
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -j, -jsonl                    write output in JSONL(ines) format
@@ -254,6 +255,7 @@ CONFIGURATIONS:
    -sr, -system-resolvers                use system DNS resolving as error fallback
    -dc, -disable-clustering              disable clustering of requests
    -passive                              enable passive HTTP response processing mode
+   -replay string                        run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
    -fh2, -force-http2                    force http2 connection on requests
    -ev, -env-vars                        enable environment variables to be used in template
    -cc, -client-cert string              client certificate file (PEM-encoded) used for authenticating against scanned hosts

README_ID.md

@@ -204,6 +204,7 @@ OUTPUT:
    -o, -output string            output file to write found issues/vulnerabilities
    -sresp, -store-resp           store all request/response passed through nuclei to output directory
    -srd, -store-resp-dir string  store all request/response passed through nuclei to custom directory (default "output")
+   -store-events string          store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
    -silent                       display findings only
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -j, -jsonl                    write output in JSONL(ines) format
@@ -237,6 +238,7 @@ CONFIGURATIONS:
    -sr, -system-resolvers                use system DNS resolving as error fallback
    -dc, -disable-clustering              disable clustering of requests
    -passive                              enable passive HTTP response processing mode
+   -replay string                        run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
    -fh2, -force-http2                    force http2 connection on requests
    -ev, -env-vars                        enable environment variables to be used in template
    -cc, -client-cert string              client certificate file (PEM-encoded) used for authenticating against scanned hosts

README_JP.md

@@ -204,6 +204,7 @@ OUTPUT:
    -o, -output string            output file to write found issues/vulnerabilities
    -sresp, -store-resp           store all request/response passed through nuclei to output directory
    -srd, -store-resp-dir string  store all request/response passed through nuclei to custom directory (default "output")
+   -store-events string          store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
    -silent                       display findings only
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -j, -jsonl                    write output in JSONL(ines) format
@@ -237,6 +238,7 @@ CONFIGURATIONS:
    -sr, -system-resolvers                use system DNS resolving as error fallback
    -dc, -disable-clustering              disable clustering of requests
    -passive                              enable passive HTTP response processing mode
+   -replay string                        run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
    -fh2, -force-http2                    force http2 connection on requests
    -ev, -env-vars                        enable environment variables to be used in template
    -cc, -client-cert string              client certificate file (PEM-encoded) used for authenticating against scanned hosts

README_KR.md

@@ -204,6 +204,7 @@ OUTPUT:
    -o, -output string            output file to write found issues/vulnerabilities
    -sresp, -store-resp           store all request/response passed through nuclei to output directory
    -srd, -store-resp-dir string  store all request/response passed through nuclei to custom directory (default "output")
+   -store-events string          store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
    -silent                       display findings only
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -j, -jsonl                    write output in JSONL(ines) format
@@ -237,6 +238,7 @@ CONFIGURATIONS:
    -sr, -system-resolvers                use system DNS resolving as error fallback
    -dc, -disable-clustering              disable clustering of requests
    -passive                              enable passive HTTP response processing mode
+   -replay string                        run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
    -fh2, -force-http2                    force http2 connection on requests
    -ev, -env-vars                        enable environment variables to be used in template
    -cc, -client-cert string              client certificate file (PEM-encoded) used for authenticating against scanned hosts

README_PT-BR.md

@@ -204,6 +204,7 @@ OUTPUT:
    -o, -output string            output file to write found issues/vulnerabilities
    -sresp, -store-resp           store all request/response passed through nuclei to output directory
    -srd, -store-resp-dir string  store all request/response passed through nuclei to custom directory (default "output")
+   -store-events string          store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
    -silent                       display findings only
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -j, -jsonl                    write output in JSONL(ines) format
@@ -237,6 +238,7 @@ CONFIGURATIONS:
    -sr, -system-resolvers                use system DNS resolving as error fallback
    -dc, -disable-clustering              disable clustering of requests
    -passive                              enable passive HTTP response processing mode
+   -replay string                        run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
    -fh2, -force-http2                    force http2 connection on requests
    -ev, -env-vars                        enable environment variables to be used in template
    -cc, -client-cert string              client certificate file (PEM-encoded) used for authenticating against scanned hosts

README_TR.md

@@ -204,6 +204,7 @@ OUTPUT:
    -o, -output string            output file to write found issues/vulnerabilities
    -sresp, -store-resp           store all request/response passed through nuclei to output directory
    -srd, -store-resp-dir string  store all request/response passed through nuclei to custom directory (default "output")
+   -store-events string          store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
    -silent                       display findings only
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -j, -jsonl                    write output in JSONL(ines) format
@@ -237,6 +238,7 @@ CONFIGURATIONS:
    -sr, -system-resolvers                use system DNS resolving as error fallback
    -dc, -disable-clustering              disable clustering of requests
    -passive                              enable passive HTTP response processing mode
+   -replay string                        run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
    -fh2, -force-http2                    force http2 connection on requests
    -ev, -env-vars                        enable environment variables to be used in template
    -cc, -client-cert string              client certificate file (PEM-encoded) used for authenticating against scanned hosts

cmd/nuclei/main.go

@@ -356,6 +356,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write found issues/vulnerabilities"),
 		flagSet.BoolVarP(&options.StoreResponse, "store-resp", "sresp", false, "store all request/response passed through nuclei to output directory"),
 		flagSet.StringVarP(&options.StoreResponseDir, "store-resp-dir", "srd", runner.DefaultDumpTrafficOutputFolder, "store all request/response passed through nuclei to custom directory"),
+		flagSet.StringVar(&options.StoreEvents, "store-events", "", "store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)"),
 		flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"),
 		flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"),
 		flagSet.BoolVarP(&options.JSONL, "jsonl", "j", false, "write output in JSONL(ines) format"),
@@ -391,6 +392,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.SystemResolvers, "system-resolvers", "sr", false, "use system DNS resolving as error fallback"),
 		flagSet.BoolVarP(&options.DisableClustering, "disable-clustering", "dc", false, "disable clustering of requests"),
 		flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"),
+		flagSet.StringVar(&options.ReplayFile, "replay", "", "run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests"),
 		flagSet.BoolVarP(&options.ForceAttemptHTTP2, "force-http2", "fh2", false, "force http2 connection on requests"),
 		flagSet.BoolVarP(&options.EnvironmentVariables, "env-vars", "ev", false, "enable environment variables to be used in template"),
 		flagSet.StringVarP(&options.ClientCertFile, "client-cert", "cc", "", "client certificate file (PEM-encoded) used for authenticating against scanned hosts"),

internal/runner/options.go

@@ -64,7 +64,8 @@ func ConfigureOptions() error {
 // would hold the scan until it does.
 func readsTargetsFromStdin(options *types.Options, stdinIsPipe bool) bool {
 	hasOtherTargets := len(options.Targets) > 0 || options.TargetsFilePath != "" || options.Uncover || len(options.UncoverQuery) > 0
-	return stdinIsPipe && !options.DisableStdin && !hasOtherTargets
+	// a replay runs over stored events and has no targets to wait for
+	return stdinIsPipe && !options.DisableStdin && !hasOtherTargets && options.ReplayFile == ""
 }
 
 // ParseOptions parses the command line flags provided by a user

internal/runner/options_test.go

@@ -84,6 +84,7 @@ func TestReadsTargetsFromStdin(t *testing.T) {
 		{"targets given with -l", &types.Options{TargetsFilePath: "targets.txt"}, true, false},
 		{"uncover query", &types.Options{UncoverQuery: goflags.StringSlice{"example"}}, true, false},
 		{"uncover enabled", &types.Options{Uncover: true}, true, false},
+		{"replay", &types.Options{ReplayFile: "events.jsonl"}, true, false},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			require.Equal(t, tc.want, readsTargetsFromStdin(tc.options, tc.stdinIsPipe))

internal/runner/replay.go

@@ -0,0 +1,24 @@
+package runner
+
+import (
+	"slices"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
+	"github.com/projectdiscovery/nuclei/v3/pkg/replay"
+)
+
+// replay writes the findings of the loaded templates over a stored corpus
+// through the usual output, so -o, -j and the other output options apply.
+func (r *Runner) replay(store *loader.Store) error {
+	results, err := replay.Replay(r.options.ReplayFile, slices.Concat(store.Templates(), store.Workflows()))
+	if err != nil {
+		return err
+	}
+	for _, result := range results {
+		if err := r.output.Write(result); err != nil {
+			return err
+		}
+	}
+	r.Logger.Info().Msgf("Replayed %s: %d findings", r.options.ReplayFile, len(results))
+	return nil
+}

internal/runner/runner.go

@@ -57,6 +57,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/honeypotdetector"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
@@ -69,6 +70,7 @@ import (
 	httpProtocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/catchall"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
+	"github.com/projectdiscovery/nuclei/v3/pkg/replay"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
@@ -837,6 +839,22 @@ func (r *Runner) RunEnumeration() error {
 	if r.options.PlanFile != "" {
 		return r.writePlan(store, executorEngine)
 	}
+	if r.options.ReplayFile != "" {
+		return r.replay(store)
+	}
+	if r.options.StoreEvents != "" {
+		recorder, err := replay.NewRecorder(r.options.StoreEvents, slices.Concat(store.Templates(), store.Workflows()))
+		if err != nil {
+			return errors.Wrap(err, "could not create events file")
+		}
+		previous := eventcreator.SetRecord(recorder.Record)
+		defer func() {
+			eventcreator.SetRecord(previous)
+			if err := recorder.Close(); err != nil {
+				r.Logger.Error().Msgf("Could not write %s: %s", r.options.StoreEvents, err)
+			}
+		}()
+	}
 	// TODO: remove below functions after v3 or update warning messages
 	templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
 

internal/tests/integration/network_test.go

@@ -46,6 +46,35 @@ func TestNetwork(t *testing.T) {
 		}
 	})
 
+	t.Run("ReplayStoredEvents", func(t *testing.T) {
+		server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
+			defer func() { _ = conn.Close() }()
+
+			data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
+			if err == nil && string(data) == "PING" {
+				_, _ = conn.Write([]byte("PONG"))
+			}
+		})
+		events := filepath.Join(t.TempDir(), "events.jsonl")
+		results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/basic.yaml", server.URL, suite.debug, "-store-events", events)
+		server.Close()
+		if err != nil {
+			t.Fatalf("recording network events failed: %v", err)
+		}
+		if err := expectResultsCount(results, 1); err != nil {
+			t.Fatal(err)
+		}
+
+		// the server is gone, so the finding can only come from the stored event
+		results, err = testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil, "-t", "protocols/network/basic.yaml", "-replay", events)
+		if err != nil {
+			t.Fatalf("replaying network events failed: %v", err)
+		}
+		if err := expectResultsCount(results, 1); err != nil {
+			t.Fatal(err)
+		}
+	})
+
 	t.Run("PinnedTargetLine", func(t *testing.T) {
 		server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
 			defer func() { _ = conn.Close() }()

pkg/js/devtools/tsgen/parser.go

@@ -284,8 +284,8 @@ func (p *EntityParser) extractClassProperties(node *ast.StructType) []Property {
 					Description: field.Doc.Text(),
 				}
 				if strings.Contains(property.Type, ".") {
-					// external type found
-					property.Type = p.handleExternalType(property.Type)
+					// external struct found
+					property.Type = p.handleExternalStruct(property.Type)
 				}
 				properties = append(properties, property)
 			}
@@ -349,7 +349,7 @@ func (p *EntityParser) extractReturnType(fn *ast.FuncDecl) (out string) {
 		for _, result := range fn.Type.Results.List {
 			tmp := exprToString(result.Type)
 			if strings.Contains(tmp, ".") && !strings.HasPrefix(tmp, "goja.") {
-				tmp = p.handleExternalType(tmp) + " | null" // external object interfaces can always return null
+				tmp = p.handleExternalStruct(tmp) + " | null" // external object interfaces can always return null
 			}
 			returns = append(returns, tmp)
 		}
@@ -374,10 +374,8 @@ func (p *EntityParser) extractReturnType(fn *ast.FuncDecl) (out string) {
 			}
 		}
 		if sliceutil.Contains(returns, "error") {
-			// external types already record that they can be null
-			if !strings.HasSuffix(out, "| null") {
-				out += " | null"
-			}
+			// add | null to the return type
+			out = out + " | null"
 			return
 		}
 	}
@@ -415,18 +413,10 @@ func (p *EntityParser) extractParameters(fn *ast.FuncDecl) []Parameter {
 	return parameters
 }
 
-// handleExternalType returns the TypeScript type for typeName, a type from
-// another package such as ssh.ClientConfig. A struct is queued for
-// scrapeAndCreate, which describes its fields in an interface of the same name.
-// Any other type, such as the net.Conn interface, has no fields to describe,
-// so it becomes any. A typeName that does not resolve is queued too, so that
-// scrapeAndCreate reports it.
-func (p *EntityParser) handleExternalType(typeName string) string {
-	obj, err := p.lookupType(typeName)
-	if err == nil && !isStruct(obj.Type()) {
-		return "any"
-	}
-
+// typeName is in format ssh.ClientConfig
+// it first fetches all fields from the struct and creates a new object
+// with that name and returns name of that object as type
+func (p *EntityParser) handleExternalStruct(typeName string) string {
 	baseType := typeName[strings.LastIndex(typeName, ".")+1:]
 	p.newObjects[typeName] = &Entity{
 		Name:        baseType,

pkg/js/devtools/tsgen/parser_external_test.go

@@ -1,33 +0,0 @@
-package tsgen_test
-
-import (
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/js/devtools/tsgen"
-)
-
-func TestParseExternalTypes(t *testing.T) {
-	t.Parallel()
-
-	ep, err := tsgen.NewEntityParser(filepath.Join("testdata", "external"))
-	require.NoError(t, err)
-	require.NoError(t, ep.Parse())
-
-	entities := make(map[string]tsgen.Entity)
-	for _, e := range ep.GetEntities() {
-		entities[e.Name] = e
-	}
-
-	// net.Conn is an interface, so it has no fields to describe.
-	require.Equal(t, "any | null", entities["Dial"].Function.Returns)
-	require.Equal(t, []tsgen.Property{{Name: "Conn", Type: "any", Description: ""}}, entities["Session"].Object.Properties)
-	require.NotContains(t, entities, "Conn")
-
-	// url.URL is a struct, so its fields become an interface.
-	require.Equal(t, "URL | null", entities["Parse"].Function.Returns)
-	require.Equal(t, "interface", entities["URL"].Type)
-	require.NotEmpty(t, entities["URL"].Object.Properties)
-}

pkg/js/devtools/tsgen/scrape.go

@@ -1,15 +1,13 @@
 package tsgen
 
 import (
-	"errors"
 	"fmt"
 	"go/types"
 	"regexp"
 	"strings"
-)
 
-// errUnresolvedType is returned by lookupType for a name it cannot resolve.
-var errUnresolvedType = errors.New("unresolved type")
+	"github.com/projectdiscovery/utils/errkit"
+)
 
 // scrape.go scrapes all information of exported type from different package
 
@@ -17,13 +15,24 @@ func (p *EntityParser) scrapeAndCreate(typeName string) error {
 	if p.newObjects[typeName] == nil {
 		return nil
 	}
-
-	typeNameObj, err := p.lookupType(typeName)
-	if err != nil {
-		return err
+	// get package name
+	pkgName := strings.Split(typeName, ".")[0]
+	baseTypeName := strings.Split(typeName, ".")[1]
+	// get package
+	pkg, ok := p.imports[pkgName]
+	if !ok {
+		return errkit.Newf("package %v for type %v not found", pkgName, typeName)
+	}
+	// get type
+	obj := pkg.Types.Scope().Lookup(baseTypeName)
+	if obj == nil {
+		return errkit.Newf("type %v not found in package %+v", typeName, pkg)
+	}
+	// Ensure the object is a type name
+	typeNameObj, ok := obj.(*types.TypeName)
+	if !ok {
+		return errkit.Newf("%v is not a type name", typeName)
 	}
-
-	baseTypeName := typeNameObj.Name()
 	// Ensure the type is a named struct type
 	namedStruct, ok := typeNameObj.Type().Underlying().(*types.Struct)
 	if !ok {
@@ -63,36 +72,6 @@ func (p *EntityParser) scrapeAndCreate(typeName string) error {
 	return nil
 }
 
-// lookupType resolves typeName, such as net.Conn, in the packages imported by
-// the parsed files.
-func (p *EntityParser) lookupType(typeName string) (*types.TypeName, error) {
-	pkgName, baseTypeName, _ := strings.Cut(typeName, ".")
-
-	pkg := p.imports[pkgName]
-	if pkg == nil {
-		return nil, fmt.Errorf("%w: package %v for type %v not found", errUnresolvedType, pkgName, typeName)
-	}
-
-	obj := pkg.Types.Scope().Lookup(baseTypeName)
-	if obj == nil {
-		return nil, fmt.Errorf("%w: type %v not found in package %+v", errUnresolvedType, typeName, pkg)
-	}
-
-	typeNameObj, ok := obj.(*types.TypeName)
-	if !ok {
-		return nil, fmt.Errorf("%w: %v is not a type name", errUnresolvedType, typeName)
-	}
-
-	return typeNameObj, nil
-}
-
-// isStruct reports whether t is a struct type.
-func isStruct(t types.Type) bool {
-	_, ok := t.Underlying().(*types.Struct)
-
-	return ok
-}
-
 type ExtObject struct {
 	builtIn map[string]string
 	nested  map[string]map[string]*ExtObject // Changed to map of field names to ExtObject

pkg/js/devtools/tsgen/testdata/external/external.go

@@ -1,32 +0,0 @@
-// Package external is a tsgen fixture that references types from other
-// packages, both structs and non-structs.
-package external
-
-import (
-	"context"
-	"net"
-	"net/url"
-)
-
-// Dial returns an external interface type.
-func Dial(network, address string) (net.Conn, error) {
-	return net.Dial(network, address)
-}
-
-// Parse returns an external struct type.
-func Parse(raw string) (*url.URL, error) {
-	return url.Parse(raw)
-}
-
-// Session holds an external interface type in an exported field.
-type Session struct {
-	Conn net.Conn
-}
-
-type dialer struct{}
-
-// DialContext is exported, but its receiver is not.
-func (dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
-	var d net.Dialer
-	return d.DialContext(ctx, network, address)
-}

pkg/js/libs/gptransport/.nuclei-jsgen-ignore

@@ -1 +0,0 @@
-shared adapter package; not a public JavaScript module

pkg/js/libs/smbsession/.nuclei-jsgen-ignore

@@ -1 +0,0 @@
-shared adapter package; not a public JavaScript module

pkg/protocols/common/helpers/eventcreator/eventcreator.go

@@ -1,6 +1,8 @@
 package eventcreator
 
 import (
+	"sync/atomic"
+
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
@@ -10,6 +12,29 @@ import (
 	"golang.org/x/text/language"
 )
 
+// RecordFunc receives the event of a request before its operators run, which
+// is what a replay needs to run them again.
+type RecordFunc func(request protocols.Request, event output.InternalEvent)
+
+// Every protocol builds its result here, so a recorder set here covers all
+// of them. Scans read it concurrently, hence the atomic.
+var record atomic.Pointer[RecordFunc]
+
+// SetRecord installs fn, nil to stop recording, and returns the callback it
+// replaced so the caller can put it back.
+func SetRecord(fn RecordFunc) RecordFunc {
+	var previous *RecordFunc
+	if fn == nil {
+		previous = record.Swap(nil)
+	} else {
+		previous = record.Swap(&fn)
+	}
+	if previous == nil {
+		return nil
+	}
+	return *previous
+}
+
 // CreateEvent wraps the outputEvent with the result of the operators defined on the request
 func CreateEvent(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool) *output.InternalWrappedEvent {
 	return CreateEventWithAdditionalOptions(request, outputEvent, isResponseDebug, nil)
@@ -19,6 +44,9 @@ func CreateEvent(request protocols.Request, outputEvent output.InternalEvent, is
 // and enables extending the resulting event with additional attributes or values.
 func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool,
 	addAdditionalOptions func(internalWrappedEvent *output.InternalWrappedEvent)) *output.InternalWrappedEvent {
+	if fn := record.Load(); fn != nil {
+		(*fn)(request, outputEvent)
+	}
 	event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
 
 	// Dump response variables if ran in debug mode

pkg/protocols/common/helpers/eventcreator/eventcreator_test.go

@@ -0,0 +1,26 @@
+package eventcreator
+
+import (
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSetRecordRestoresTheCallbackItReplaced(t *testing.T) {
+	var calls []string
+	first := RecordFunc(func(protocols.Request, output.InternalEvent) { calls = append(calls, "first") })
+	second := RecordFunc(func(protocols.Request, output.InternalEvent) { calls = append(calls, "second") })
+
+	require.Nil(t, SetRecord(first))
+	previous := SetRecord(second)
+	(*record.Load())(nil, nil)
+	// a second runner stopping puts the first one's recorder back, not nil
+	SetRecord(previous)
+	(*record.Load())(nil, nil)
+	require.Equal(t, []string{"second", "first"}, calls)
+
+	SetRecord(nil)
+	require.Nil(t, record.Load())
+}

pkg/replay/codec.go

@@ -0,0 +1,93 @@
+package replay
+
+import (
+	"encoding/base64"
+	"unicode/utf8"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
+)
+
+// Matchers type-assert some values, such as an int status code or a []string
+// of dns answers, and binary responses are not valid UTF-8, so plain JSON
+// would change what they see. Those values are wrapped in a one-key object
+// naming their Go type; everything else is stored as JSON.
+const (
+	tagBytes   = "$bytes"
+	tagBinary  = "$binary"
+	tagInt     = "$int"
+	tagStrings = "$strings"
+)
+
+func encodeEvent(event output.InternalEvent) map[string]any {
+	encoded := make(map[string]any, len(event))
+	for key, value := range event {
+		switch v := value.(type) {
+		case string:
+			if utf8.ValidString(v) {
+				encoded[key] = v
+			} else {
+				encoded[key] = map[string]string{tagBinary: base64.StdEncoding.EncodeToString([]byte(v))}
+			}
+		case []byte:
+			encoded[key] = map[string]string{tagBytes: base64.StdEncoding.EncodeToString(v)}
+		case int:
+			encoded[key] = map[string]int{tagInt: v}
+		case []string:
+			encoded[key] = map[string][]string{tagStrings: v}
+		default:
+			// values JSON cannot hold, such as funcs, are not something a
+			// matcher reads
+			if _, err := json.Marshal(v); err == nil {
+				encoded[key] = v
+			}
+		}
+	}
+	return encoded
+}
+
+func decodeEvent(raw map[string]json.Message) (output.InternalEvent, error) {
+	event := make(output.InternalEvent, len(raw))
+	for key, message := range raw {
+		value, err := decodeValue(message)
+		if err != nil {
+			return nil, err
+		}
+		event[key] = value
+	}
+	return event, nil
+}
+
+func decodeValue(message json.Message) (any, error) {
+	var tagged map[string]json.Message
+	if json.Unmarshal(message, &tagged) == nil && len(tagged) == 1 {
+		for tag, inner := range tagged {
+			switch tag {
+			case tagBytes, tagBinary:
+				var text string
+				if err := json.Unmarshal(inner, &text); err != nil {
+					return nil, err
+				}
+				data, err := base64.StdEncoding.DecodeString(text)
+				if err != nil {
+					return nil, err
+				}
+				if tag == tagBinary {
+					return string(data), nil
+				}
+				return data, nil
+			case tagInt:
+				var number int
+				err := json.Unmarshal(inner, &number)
+				return number, err
+			case tagStrings:
+				var list []string
+				err := json.Unmarshal(inner, &list)
+				return list, err
+			}
+		}
+	}
+	var value any
+	err := json.Unmarshal(message, &value)
+	return value, err
+}

pkg/replay/codec_test.go

@@ -0,0 +1,39 @@
+package replay
+
+import (
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEventRoundTripKeepsWhatMatchersRead(t *testing.T) {
+	event := output.InternalEvent{
+		"body":        "<title>admin</title>",
+		"raw":         "\x00\xffBIN\xfe",
+		"data":        []byte{0, 1, 2},
+		"status_code": 200,
+		"answer":      []string{"a", "b"},
+		"duration":    1.5,
+		"matched":     true,
+		"info":        map[string]any{"name": "x"},
+		"callback":    func() {},
+	}
+	data, err := json.Marshal(encodeEvent(event))
+	require.NoError(t, err)
+	var raw map[string]json.Message
+	require.NoError(t, json.Unmarshal(data, &raw))
+	decoded, err := decodeEvent(raw)
+	require.NoError(t, err)
+
+	require.Equal(t, "<title>admin</title>", decoded["body"])
+	require.Equal(t, "\x00\xffBIN\xfe", decoded["raw"], "binary responses keep every byte")
+	require.Equal(t, []byte{0, 1, 2}, decoded["data"])
+	require.Equal(t, 200, decoded["status_code"], "the status matcher reads an int")
+	require.Equal(t, []string{"a", "b"}, decoded["answer"])
+	require.Equal(t, 1.5, decoded["duration"])
+	require.Equal(t, true, decoded["matched"])
+	require.Equal(t, map[string]any{"name": "x"}, decoded["info"])
+	require.NotContains(t, decoded, "callback", "values JSON cannot hold are dropped")
+}

pkg/replay/record.go

@@ -0,0 +1,121 @@
+// Package replay stores the event every request hands its matchers and
+// extractors, for any protocol, and runs templates over a stored corpus again
+// without sending anything. Comparing the findings of two template sets or
+// two builds on the same corpus shows what a change moved.
+package replay
+
+import (
+	"os"
+	"strconv"
+	"sync"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
+)
+
+// record is one line of a corpus.
+type record[E any] struct {
+	TemplateID string `json:"template-id,omitempty"`
+	// Request is the position of the request in its template.
+	Request int `json:"request"`
+	// ClusterKey identifies the request of a single-request template; every
+	// template sending the same request replays the event, as the scan
+	// clusters them onto one.
+	ClusterKey string `json:"cluster-key,omitempty"`
+	Event      E      `json:"event"`
+}
+
+type requestRef struct {
+	templateID string
+	index      int
+	single     bool
+}
+
+// Recorder appends the events of a scan to a corpus file.
+type Recorder struct {
+	mu       sync.Mutex
+	file     *os.File
+	requests map[protocols.Request]requestRef
+	err      error
+}
+
+// clusterKeyed is a request the scan can cluster. The key leaves out what
+// makes a request unclusterable, such as a raw request or a body, so only a
+// clusterable request may share events by it.
+type clusterKeyed interface {
+	TmplClusterKey() uint64
+	IsClusterable() bool
+}
+
+func clusterKey(request protocols.Request) (string, bool) {
+	keyed, ok := request.(clusterKeyed)
+	if !ok || !keyed.IsClusterable() {
+		return "", false
+	}
+	return strconv.FormatUint(keyed.TmplClusterKey(), 10), true
+}
+
+// NewRecorder creates the corpus at path for a scan of templatesList. The
+// corpus holds raw traffic, credentials included, so only its owner can read
+// it.
+func NewRecorder(path string, templatesList []*templates.Template) (*Recorder, error) {
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
+	if err != nil {
+		return nil, err
+	}
+	recorder := &Recorder{file: file, requests: make(map[protocols.Request]requestRef)}
+	for _, source := range sources(templatesList) {
+		single := len(source.requests) == 1 && source.clustered
+		for i, request := range source.requests {
+			recorder.requests[request] = requestRef{templateID: source.id, index: i, single: single}
+		}
+	}
+	return recorder, nil
+}
+
+// Record stores event, which request produced. It runs before the operators,
+// which add their own values to the event.
+func (r *Recorder) Record(request protocols.Request, event output.InternalEvent) {
+	ref, known := r.requests[request]
+	if !known {
+		// a cluster runs one request for all its members and was compiled
+		// apart from them; its members replay it by the cluster key
+		key, ok := clusterKey(request)
+		if !ok {
+			return
+		}
+		templateID, _ := event["template-id"].(string)
+		r.write(record[map[string]any]{TemplateID: templateID, ClusterKey: key, Event: encodeEvent(event)})
+		return
+	}
+	line := record[map[string]any]{TemplateID: ref.templateID, Request: ref.index, Event: encodeEvent(event)}
+	if key, ok := clusterKey(request); ok && ref.single {
+		line.ClusterKey = key
+	}
+	r.write(line)
+}
+
+func (r *Recorder) write(line record[map[string]any]) {
+	data, err := json.Marshal(line)
+	if err != nil {
+		return
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	// written straight to the file: ctrl+c exits without running defers
+	if _, err := r.file.Write(append(data, '\n')); err != nil && r.err == nil {
+		r.err = err
+	}
+}
+
+// Close closes the corpus and reports the first write error.
+func (r *Recorder) Close() error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if err := r.file.Close(); err != nil && r.err == nil {
+		r.err = err
+	}
+	return r.err
+}

pkg/replay/replay.go

@@ -0,0 +1,106 @@
+package replay
+
+import (
+	"bufio"
+	"bytes"
+	"cmp"
+	"fmt"
+	"io"
+	"maps"
+	"os"
+	"slices"
+	"strconv"
+	"strings"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
+)
+
+type target struct {
+	source  source
+	request protocols.Request
+}
+
+// Replay runs the matchers and extractors of templatesList over every event
+// in the corpus at path and returns the findings sorted, so two replays diff
+// line by line. An event replays on the template and request that produced
+// it, and a single-request event also on every template sending the same
+// request.
+func Replay(path string, templatesList []*templates.Template) ([]*output.ResultEvent, error) {
+	byRequest := make(map[string][]target)
+	byCluster := make(map[string][]target)
+	for _, source := range sources(templatesList) {
+		for i, request := range source.requests {
+			byRequest[source.id+"\x00"+strconv.Itoa(i)] = []target{{source: source, request: request}}
+		}
+		if len(source.requests) == 1 && source.clustered {
+			if key, ok := clusterKey(source.requests[0]); ok {
+				byCluster[key] = append(byCluster[key], target{source: source, request: source.requests[0]})
+			}
+		}
+	}
+
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = file.Close() }()
+	var results []*output.ResultEvent
+	reader := bufio.NewReader(file)
+	for number := 1; ; number++ {
+		line, readErr := reader.ReadBytes('\n')
+		if line = bytes.TrimSpace(line); len(line) > 0 {
+			var stored record[map[string]json.Message]
+			if err := json.Unmarshal(line, &stored); err != nil {
+				return nil, fmt.Errorf("%s:%d: %w", path, number, err)
+			}
+			event, err := decodeEvent(stored.Event)
+			if err != nil {
+				return nil, fmt.Errorf("%s:%d: %w", path, number, err)
+			}
+			targets := byRequest[stored.TemplateID+"\x00"+strconv.Itoa(stored.Request)]
+			if stored.ClusterKey != "" {
+				targets = byCluster[stored.ClusterKey]
+			}
+			for _, target := range targets {
+				results = append(results, replayEvent(target, event)...)
+			}
+		}
+		if readErr == io.EOF {
+			break
+		}
+		if readErr != nil {
+			return nil, readErr
+		}
+	}
+	// extractors keep values in a map, so their order changes between runs
+	for _, result := range results {
+		slices.Sort(result.ExtractedResults)
+	}
+	slices.SortStableFunc(results, compareResults)
+	return results, nil
+}
+
+func replayEvent(target target, event output.InternalEvent) []*output.ResultEvent {
+	// operators add their values to the event, and a cluster event carries
+	// the first member's identity
+	event = maps.Clone(event)
+	event["template-id"] = target.source.id
+	event["template-path"] = target.source.path
+	event["template-info"] = target.source.info
+	return eventcreator.CreateEvent(target.request, event, false).Results
+}
+
+func compareResults(a, b *output.ResultEvent) int {
+	return cmp.Or(
+		strings.Compare(a.TemplateID, b.TemplateID),
+		strings.Compare(a.Host, b.Host),
+		strings.Compare(a.Matched, b.Matched),
+		strings.Compare(a.MatcherName, b.MatcherName),
+		strings.Compare(a.ExtractorName, b.ExtractorName),
+		strings.Compare(strings.Join(a.ExtractedResults, "\x00"), strings.Join(b.ExtractedResults, "\x00")),
+	)
+}

pkg/replay/sources.go

@@ -0,0 +1,78 @@
+package replay
+
+import (
+	"github.com/projectdiscovery/nuclei/v3/pkg/model"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v3/pkg/workflows"
+)
+
+// source is a template whose requests produce events: a loaded template, or
+// a step of a loaded workflow, which may not be loaded on its own.
+type source struct {
+	id       string
+	path     string
+	info     model.Info
+	requests []protocols.Request
+	// clustered mirrors templates.Cluster: only a template the scan can
+	// cluster shares events with others by its request
+	clustered bool
+}
+
+type requestLister interface {
+	ProtocolRequests() []protocols.Request
+}
+
+type templateExecuter interface {
+	requestLister
+	TemplateOptions() *protocols.ExecutorOptions
+}
+
+func sources(templatesList []*templates.Template) []source {
+	var list []source
+	seen := make(map[string]bool)
+	add := func(s source) {
+		if !seen[s.id] && len(s.requests) > 0 {
+			seen[s.id] = true
+			list = append(list, s)
+		}
+	}
+	addTemplate := func(template *templates.Template) {
+		if lister, ok := template.Executer.(requestLister); ok {
+			clustered := template.Flow == "" && (template.Options == nil || !template.Options.IsMultiProtocol) && !template.Info.HasPlaceholders()
+			add(source{id: template.ID, path: template.Path, info: template.Info, requests: lister.ProtocolRequests(), clustered: clustered})
+		}
+	}
+	var walk func(steps []*workflows.WorkflowTemplate)
+	walk = func(steps []*workflows.WorkflowTemplate) {
+		for _, step := range steps {
+			for _, pair := range step.Executers {
+				switch executer := pair.Executer.(type) {
+				case templateExecuter:
+					options := executer.TemplateOptions()
+					add(source{id: options.TemplateID, path: options.TemplatePath, info: options.TemplateInfo, requests: executer.ProtocolRequests()})
+				case *templates.ClusterExecuter:
+					// a cluster holds only the first member's request; each
+					// member is parsed again for its own, which the event is
+					// matched to by its cluster key
+					for _, path := range executer.TemplatePaths() {
+						if member, err := templates.Parse(path, nil, pair.Options); err == nil && member != nil {
+							addTemplate(member)
+						}
+					}
+				}
+			}
+			walk(step.Subtemplates)
+			for _, matcher := range step.Matchers {
+				walk(matcher.Subtemplates)
+			}
+		}
+	}
+	for _, template := range templatesList {
+		addTemplate(template)
+		if template.CompiledWorkflow != nil {
+			walk(template.CompiledWorkflow.Workflows)
+		}
+	}
+	return list
+}

pkg/templates/cluster.go

@@ -301,6 +301,15 @@ func (e *ClusterExecuter) TemplateIDs(selection protocols.TemplateSelection) []s
 	return ids
 }
 
+// TemplatePaths returns the paths of every member.
+func (e *ClusterExecuter) TemplatePaths() []string {
+	paths := make([]string, 0, len(e.operators))
+	for _, operator := range e.operators {
+		paths = append(paths, operator.templatePath)
+	}
+	return paths
+}
+
 // operatorsFor returns the members selected for input by per-target profiles.
 func (e *ClusterExecuter) operatorsFor(input *contextargs.MetaInput) []*clusteredOperator {
 	if e.options.TargetScope == nil {

pkg/tmplexec/exec.go

@@ -88,6 +88,17 @@ func (e *TemplateExecuter) Compile() error {
 	return e.engine.Compile()
 }
 
+// TemplateOptions returns the options of the template the requests belong
+// to, which name it.
+func (e *TemplateExecuter) TemplateOptions() *protocols.ExecutorOptions {
+	return e.options
+}
+
+// ProtocolRequests returns the compiled requests in execution order.
+func (e *TemplateExecuter) ProtocolRequests() []protocols.Request {
+	return e.requests
+}
+
 // Requests returns the total number of requests the rule will perform
 func (e *TemplateExecuter) Requests() int {
 	var count int

pkg/types/types.go

@@ -254,6 +254,12 @@ type Options struct {
 	// PlanFile is where the work a scan would do is written as JSON lines;
 	// nuclei exits after writing it, without sending requests.
 	PlanFile string
+	// StoreEvents is where the scan writes the event of every request, as
+	// JSON lines, for -replay.
+	StoreEvents string
+	// ReplayFile is a corpus written by -store-events whose events the
+	// templates are run over, without sending requests.
+	ReplayFile string
 	// NoStrictSyntax disables strict syntax check on nuclei templates (allows custom key-value pairs).
 	NoStrictSyntax bool
 	// Verbose flag indicates whether to show verbose output or not
@@ -715,6 +721,8 @@ func (options *Options) Copy() *Options {
 		Silent:                         options.Silent,
 		Validate:                       options.Validate,
 		PlanFile:                       options.PlanFile,
+		StoreEvents:                    options.StoreEvents,
+		ReplayFile:                     options.ReplayFile,
 		NoStrictSyntax:                 options.NoStrictSyntax,
 		Verbose:                        options.Verbose,
 		VerboseVerbose:                 options.VerboseVerbose,