review fixes

cmd/nuclei/header_test.go

@@ -1,57 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/stretchr/testify/require"
-)
-
-const headerChildEnv = "NUCLEI_HEADER_CHILD_ARGS"
-
-// -H values can contain "#", on the command line, in a header file and in the
-// config file, while lines of a header file that start with "#" are skipped as
-// comments.
-func TestHeaderValuesKeepHash(t *testing.T) {
-	if args := os.Getenv(headerChildEnv); args != "" {
-		options = &types.Options{Logger: gologger.DefaultLogger}
-		os.Args = append([]string{os.Args[0]}, strings.Split(args, "\n")...)
-		readConfig()
-		fmt.Printf("headers=%s\n", strings.Join(options.CustomHeaders, "|"))
-		os.Exit(0)
-	}
-
-	root := t.TempDir()
-	headerFile := filepath.Join(root, "headers.txt")
-	require.NoError(t, os.WriteFile(headerFile, []byte("# auth headers\nX-Token: abc#def\nReferer: https://app.example.com/#/dashboard\n"), 0o600))
-	configFile := filepath.Join(root, "config.yaml")
-	require.NoError(t, os.WriteFile(configFile, []byte("header:\n  - \"X-Config: cfg#value\"\n"), 0o600))
-
-	tests := []struct {
-		name string
-		args []string
-		want string
-	}{
-		{"cli and header file", []string{"-H", "Cookie: sid=1; color=#ff0000", "-H", headerFile}, "Cookie: sid=1; color=#ff0000|X-Token: abc#def|Referer: https://app.example.com/#/dashboard"},
-		{"config file", []string{"-config", configFile}, "X-Config: cfg#value"},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			tempRoot := filepath.Join(t.TempDir(), "tmp")
-			require.NoError(t, os.Mkdir(tempRoot, 0o700))
-
-			args := append(tt.args, "-disable-update-check")
-			cmd := exec.Command(os.Args[0], "-test.run=^TestHeaderValuesKeepHash$", "-test.count=1")
-			cmd.Env = isolatedNucleiStorageEnv(root, tempRoot, headerChildEnv+"="+strings.Join(args, "\n"))
-			output, err := cmd.CombinedOutput()
-			require.NoError(t, err, string(output))
-			require.Contains(t, string(output), "headers="+tt.want+"\n")
-		})
-	}
-}

cmd/nuclei/main.go

@@ -55,17 +55,6 @@ var (
 	inlineSecretsTempMu  sync.Mutex
 )
 
-// normalizeHeaderValue trims a -H value and drops it when the whole line is a
-// "#" comment. The goflags default strips inline comments, which cut values
-// such as "Cookie: color=#fff" at the "#".
-func normalizeHeaderValue(s string) string {
-	s = strings.TrimSpace(s)
-	if strings.HasPrefix(s, "#") {
-		return ""
-	}
-	return s
-}
-
 func main() {
 	options.Logger = gologger.DefaultLogger
 
@@ -266,11 +255,6 @@ func readConfig() *goflags.FlagSet {
 	var pdcpAuth string
 	var fuzzFlag bool
 
-	// FileStringSliceOptions as set up by runner.ConfigureOptions, but keeping
-	// "#" inside -H values
-	headerSliceOptions := goflags.FileStringSliceOptions
-	headerSliceOptions.Normalize = normalizeHeaderValue
-
 	flagSet := goflags.NewFlagSet()
 	flagSet.CaseSensitive = true
 	flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing
@@ -383,7 +367,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "mr", 10, "max number of redirects to follow for http templates"),
 		flagSet.BoolVarP(&options.DisableRedirects, "disable-redirects", "dr", false, "disable redirects for http templates"),
 		flagSet.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking
-		flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", nil, "custom header/cookie to include in all http request in header:value format (cli, file)", headerSliceOptions),
+		flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", nil, "custom header/cookie to include in all http request in header:value format (cli, file)", goflags.FileStringSliceOptions),
 		flagSet.RuntimeMapVarP(&options.Vars, "var", "V", nil, "custom vars in key=value format"),
 		flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"),
 		flagSet.BoolVarP(&options.SystemResolvers, "system-resolvers", "sr", false, "use system DNS resolving as error fallback"),

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/Mzack9999/go-rsync v0.0.0-20250821180103-81ffa574ef4d
 	github.com/RedTeamPentesting/adauth v0.5.4-0.20260511073005-3d18e8a5a687
 	github.com/adrg/xdg v0.5.3
-	github.com/alecthomas/chroma/v2 v2.27.0
+	github.com/alecthomas/chroma v0.10.0
 	github.com/alexsnet/go-vnc v0.1.0
 	github.com/alitto/pond v1.9.2
 	github.com/andybalholm/brotli v1.2.0
@@ -74,7 +74,6 @@ require (
 	github.com/miekg/dns v1.1.73
 	github.com/oiweiwei/go-msrpc v1.2.12
 	github.com/olekukonko/tablewriter v1.0.8
-	github.com/openrdap/rdap v0.9.1
 	github.com/ory/dockertest/v3 v3.12.0
 	github.com/pkg/errors v0.9.1
 	github.com/praetorian-inc/fingerprintx v1.1.15
@@ -176,6 +175,7 @@ require (
 	github.com/STARRY-S/zip v0.2.3 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/akrylysov/pogreb v0.10.2 // indirect
+	github.com/alecthomas/chroma/v2 v2.20.0 // indirect
 	github.com/alecthomas/kingpin/v2 v2.4.0 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
@@ -237,6 +237,7 @@ require (
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/distribution/reference v0.6.0 // indirect
 	github.com/djherbis/times v1.6.0 // indirect
+	github.com/dlclark/regexp2 v1.11.5 // indirect
 	github.com/dlclark/regexp2/v2 v2.2.1 // indirect
 	github.com/docker/cli v29.2.0+incompatible // indirect
 	github.com/docker/go-connections v0.7.0 // indirect
@@ -263,7 +264,7 @@ require (
 	github.com/go-fed/httpsig v1.1.0 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.9.0 // indirect
-	github.com/go-jose/go-jose/v4 v4.1.6 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
 	github.com/go-logr/logr v1.4.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
@@ -366,6 +367,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/opencontainers/runc v1.3.6 // indirect
+	github.com/openrdap/rdap v0.9.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/pierrec/lz4/v4 v4.1.23 // indirect
 	github.com/pjbgf/sha1cd v0.6.0 // indirect

go.sum

@@ -147,13 +147,15 @@ github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi
 github.com/akrylysov/pogreb v0.10.2/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
 github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
 github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
-github.com/alecthomas/chroma/v2 v2.27.0 h1:FodwmyOBgJULFYmDqibcp9pvfDLWdtPRh9v/r5BXYZs=
-github.com/alecthomas/chroma/v2 v2.27.0/go.mod h1:NjJ3ciIgrqBNeIkWZ4e46nseoLDslxU1LmfCoL+wcY8=
+github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
+github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
+github.com/alecthomas/chroma/v2 v2.20.0 h1:sfIHpxPyR07/Oylvmcai3X/exDlE8+FA820NTz+9sGw=
+github.com/alecthomas/chroma/v2 v2.20.0/go.mod h1:e7tViK0xh/Nf4BYHl00ycY6rV7b8iXBksI9E359yNmA=
 github.com/alecthomas/kingpin/v2 v2.3.1/go.mod h1:oYL5vtsvEHZGHxU7DMp32Dvx+qL+ptGn6lWaot2vCNE=
 github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY=
 github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
-github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
-github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/alecthomas/repr v0.5.1 h1:E3G4t2QbHTSNpPKBgMTln5KLkZHLOcU7r37J4pXBuIg=
+github.com/alecthomas/repr v0.5.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -356,6 +358,7 @@ github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c h1:+Zo5Ca9
 github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c/go.mod h1:HJGU9ULdREjOcVGZVPB5s6zYmHi1RxzT71l2wQyLmnE=
 github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
 github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
+github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
 github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/dlclark/regexp2/v2 v2.2.1 h1:mf4KkFUj0gJuarK8P+LgiS+Lit7m9N1yAwEfPbee7R0=
@@ -452,8 +455,8 @@ github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuw
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-jose/go-jose/v4 v4.1.6 h1:sNRkftxFJxwsILiQukqp5z83QBSmntVwmgRY7AqBDO4=
-github.com/go-jose/go-jose/v4 v4.1.6/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
+github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=

internal/pdcp/writer.go

@@ -4,8 +4,6 @@ import (
 	"bufio"
 	"bytes"
 	"context"
-	"crypto/tls"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -89,7 +87,11 @@ func NewUploadWriter(ctx context.Context, logger *gologger.Logger, creds *pdcpau
 	tmp.Update()
 	u.uploadURL = tmp.URL
 
-	u.client = newUploadClient()
+	// create http client
+	opts := retryablehttp.DefaultOptionsSingle
+	opts.NoAdjustTimeout = true
+	opts.Timeout = time.Duration(3) * time.Minute
+	u.client = retryablehttp.NewClient(opts)
 
 	// create context
 	ctx, u.cancel = context.WithCancel(ctx)
@@ -99,32 +101,6 @@ func NewUploadWriter(ctx context.Context, logger *gologger.Logger, creds *pdcpau
 	return u, nil
 }
 
-// newUploadClient returns the client for cloud uploads. The upload sends the
-// API key and the scan results, so the client verifies the server certificate.
-// The retryablehttp default clients skip verification.
-func newUploadClient() *retryablehttp.Client {
-	transport := retryablehttp.DefaultReusePooledTransport()
-	// fastdialer DialTLS does not verify certificates, so net/http does the
-	// TLS handshake over the fastdialer connection.
-	transport.DialTLSContext = nil
-	transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12}
-
-	opts := retryablehttp.DefaultOptionsSingle
-	opts.NoAdjustTimeout = true
-	opts.Timeout = time.Duration(3) * time.Minute
-	opts.HttpClient = &http.Client{Transport: transport}
-	// a certificate error does not go away on retry
-	defaultRetry := retryablehttp.DefaultRetryPolicy()
-	opts.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) {
-		var certErr *tls.CertificateVerificationError
-		if errors.As(err, &certErr) {
-			return false, nil
-		}
-		return defaultRetry(ctx, resp, err)
-	}
-	return retryablehttp.NewClient(opts)
-}
-
 // SetScanID sets the scan id for the upload writer
 func (u *UploadWriter) SetScanID(id string) error {
 	if !xidRegex.MatchString(id) {

internal/pdcp/writer_tls_test.go

@@ -1,55 +0,0 @@
-package pdcp
-
-import (
-	"context"
-	"crypto/tls"
-	"crypto/x509"
-	"net/http"
-	"net/http/httptest"
-	"sync/atomic"
-	"testing"
-
-	"github.com/projectdiscovery/gologger"
-	pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
-	"github.com/stretchr/testify/require"
-)
-
-func newTestUploadWriter(t *testing.T, gotKey *atomic.Value) (*UploadWriter, *httptest.Server) {
-	t.Helper()
-	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		gotKey.Store(r.Header.Get(pdcpauth.ApiKeyHeaderName))
-		_, _ = w.Write([]byte(`{"id":"abcdefghij0123456789"}`))
-	}))
-	t.Cleanup(srv.Close)
-	creds := &pdcpauth.PDCPCredentials{APIKey: "test-key", Server: srv.URL}
-	u, err := NewUploadWriter(context.Background(), gologger.DefaultLogger, creds)
-	require.NoError(t, err)
-	t.Cleanup(u.Close)
-	return u, srv
-}
-
-func TestUploadRejectsUntrustedCertificate(t *testing.T) {
-	var gotKey atomic.Value
-	u, _ := newTestUploadWriter(t, &gotKey)
-
-	err := u.upload([]byte("{}\n"))
-	require.Error(t, err)
-	var unknownAuthority x509.UnknownAuthorityError
-	require.ErrorAs(t, err, &unknownAuthority)
-	require.Nil(t, gotKey.Load(), "API key must not reach a server with an untrusted certificate")
-}
-
-func TestUploadTrustedCertificate(t *testing.T) {
-	var gotKey atomic.Value
-	u, srv := newTestUploadWriter(t, &gotKey)
-
-	pool := x509.NewCertPool()
-	pool.AddCert(srv.Certificate())
-	transport, ok := u.client.HTTPClient.Transport.(*http.Transport)
-	require.True(t, ok)
-	transport.TLSClientConfig = &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}
-
-	require.NoError(t, u.upload([]byte("{}\n")))
-	require.Equal(t, "test-key", gotKey.Load())
-	require.Equal(t, "abcdefghij0123456789", u.scanID)
-}

internal/runner/templates.go

@@ -6,7 +6,7 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/alecthomas/chroma/v2/quick"
+	"github.com/alecthomas/chroma/quick"
 	"github.com/logrusorgru/aurora/v4"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"

internal/tests/integration/javascript_ipmi_test.go

@@ -1,89 +0,0 @@
-//go:build integration
-// +build integration
-
-package integration_test
-
-import (
-	"fmt"
-	"net"
-	"time"
-
-	"github.com/ory/dockertest/v3"
-)
-
-// ipmiSimSetup runs OpenIPMI's BMC simulator with an anonymous user,
-// ADMIN/ADMIN and root/calvin, and cipher suite 0 enabled.
-const ipmiSimSetup = `apt-get update -qq && apt-get install -y -qq --no-install-recommends openipmi >/dev/null && mkdir -p /etc/ipmi && cat > /etc/ipmi/lan.conf <<'EOF'
-name "bmc"
-set_working_mc 0x20
-  startlan 1
-    addr 0.0.0.0 623
-    priv_limit admin
-    allowed_auths_callback none md2 md5 straight
-    allowed_auths_user none md2 md5 straight
-    allowed_auths_operator none md2 md5 straight
-    allowed_auths_admin none md2 md5 straight
-    guid a123456789abcdefa123456789abcdef
-  endlan
-  user 1 true  ""      ""       user  10 none md2 md5 straight
-  user 2 true  "ADMIN" "ADMIN"  admin 10 none md2 md5 straight
-  user 3 true  "root"  "calvin" admin 10 none md2 md5 straight
-EOF
-printf 'mc_setbmc 0x20\nmc_add 0x20 0 no-device-sdrs 0x23 9 8 0x9f 0x1291 0xf02 persist_sdr\nmc_enable 0x20\n' > /etc/ipmi/emu.cmd
-exec ipmi_sim -c /etc/ipmi/lan.conf -f /etc/ipmi/emu.cmd -n`
-
-type javascriptIPMI struct{}
-
-func (j *javascriptIPMI) Execute(filePath string) error {
-	pool, err := dockertest.NewPool("")
-	if err != nil {
-		return fmt.Errorf("could not create docker pool: %w", err)
-	}
-	if err := pool.Client.Ping(); err != nil {
-		return fmt.Errorf("could not connect to Docker: %w", err)
-	}
-	resource, err := pool.RunWithOptions(&dockertest.RunOptions{
-		Repository:   "debian",
-		Tag:          "bookworm-slim",
-		Platform:     "linux/amd64",
-		Cmd:          []string{"sh", "-c", ipmiSimSetup},
-		ExposedPorts: []string{"623/udp"},
-	})
-	if err != nil {
-		return fmt.Errorf("could not start ipmi simulator: %w", err)
-	}
-	defer purge(pool, resource)
-
-	address := net.JoinHostPort("127.0.0.1", resource.GetPort("623/udp"))
-	if err := waitForServiceCheck(address, 3*time.Minute, rmcpPingCheck); err != nil {
-		return err
-	}
-	results, err := runSignedNucleiTemplateAndGetResults(filePath, address, debug)
-	if err != nil {
-		return err
-	}
-	return expectResultsCount(results, 1)
-}
-
-// rmcpPingCheck waits for an answer to an RMCP presence ping, since the
-// port is udp and cannot be probed with a connect.
-func rmcpPingCheck(address string) error {
-	conn, err := net.Dial("udp", address)
-	if err != nil {
-		return err
-	}
-	defer func() { _ = conn.Close() }()
-	if _, err := conn.Write([]byte{0x06, 0x00, 0xff, 0x06, 0x00, 0x00, 0x11, 0xbe, 0x80, 0x00, 0x00, 0x00}); err != nil {
-		return err
-	}
-	_ = conn.SetReadDeadline(time.Now().Add(time.Second))
-	reply := make([]byte, 64)
-	n, err := conn.Read(reply)
-	if err != nil {
-		return err
-	}
-	if n < 9 || reply[8] != 0x40 {
-		return fmt.Errorf("unexpected rmcp reply")
-	}
-	return nil
-}

internal/tests/integration/javascript_test.go

@@ -46,7 +46,6 @@ var jsTestcases = []integrationCase{
 	{Path: "protocols/javascript/http-get.yaml", TestCase: &javascriptHTTPGet{}},
 	{Path: "protocols/javascript/http-client-flow.yaml", TestCase: &javascriptHTTPClientFlow{}},
 	{Path: "protocols/javascript/http-denied.yaml", TestCase: &javascriptHTTPDenied{}},
-	{Path: "protocols/javascript/ipmi.yaml", TestCase: &javascriptIPMI{}, DisableOn: javascriptDockerDisabled, Serial: true},
 	{Path: "protocols/javascript/rsync-test.yaml", TestCase: &javascriptRsyncTest{}, DisableOn: javascriptDockerDisabled, Serial: true},
 	{Path: "protocols/javascript/vnc-pass-brute.yaml", TestCase: &javascriptVncPassBrute{}, DisableOn: javascriptDockerDisabled, Serial: true},
 	{Path: "protocols/javascript/postgres-pass-brute.yaml", TestCase: &javascriptPostgresPassBrute{}, DisableOn: javascriptDockerDisabled, Serial: true},

internal/tests/integration/testdata/protocols/javascript/ipmi.yaml

@@ -1,46 +0,0 @@
-id: ipmi-checks
-
-info:
-  name: IPMI Checks
-  author: pdteam
-  severity: info
-
-javascript:
-  - code: |
-      const ipmi = require('nuclei/ipmi');
-      const caps = ipmi.GetAuthCapabilities(Host, Port);
-      const root = ipmi.GetRAKPHash(Host, Port, 'root');
-
-      // cipher suite 0 lets a wrong password in
-      const anyPassword = new ipmi.Client();
-      const cipherZero = anyPassword.Login(Host, Port, 'ADMIN', 'wrong', 0);
-      anyPassword.Close();
-
-      // an encrypted, signed session, then Get Device ID and admin privilege
-      const client = new ipmi.Client();
-      client.Login(Host, Port, 'ADMIN', 'ADMIN', 3);
-      const device = client.Command(0x06, 0x01, []);
-      const admin = client.Command(0x06, 0x3b, [0x04]);
-      client.Close();
-      const manufacturer = device.Data[6] | device.Data[7] << 8 | device.Data[8] << 16;
-
-      let wrongRejected = false;
-      try { new ipmi.Client().Login(Host, Port, 'ADMIN', 'wrong', 3); } catch (e) { wrongRejected = true; }
-
-      ipmi.Ping(Host, Port) && caps.IPMI20 && caps.AnonymousLogin &&
-        ipmi.GetCipherSuites(Host, Port).includes(0) &&
-        ipmi.GetSystemGUID(Host, Port).length == 32 &&
-        ipmi.FindRAKPPassword(root, ['admin', 'calvin']) == 'calvin' &&
-        !ipmi.CheckRAKPPassword(root, 'admin') &&
-        cipherZero && wrongRejected &&
-        device.CompletionCode == 0 && manufacturer == 4753 &&
-        admin.CompletionCode == 0;
-
-    args:
-      Host: "{{Host}}"
-      Port: "623"
-
-    matchers:
-      - type: dsl
-        dsl:
-          - "success == true"

pkg/catalog/loader/ai_loader.go

@@ -7,22 +7,18 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"regexp"
 	"strings"
 
-	"github.com/alecthomas/chroma/v2/quick"
+	"github.com/alecthomas/chroma/quick"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/retryablehttp-go"
 	pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
 	"github.com/projectdiscovery/utils/errkit"
 )
 
-var (
+const (
 	aiTemplateGeneratorAPIEndpoint = "https://api.projectdiscovery.io/v1/template/ai"
-	// aiTemplateHTTPClient verifies TLS, since the request carries the PDCP API key.
-	aiTemplateHTTPClient = &http.Client{}
-	// aiTemplateIDRegex keeps the template file name inside the pdcp directory.
-	aiTemplateIDRegex = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
 )
 
 type AITemplateResponse struct {
@@ -116,7 +112,7 @@ func generateAITemplate(prompt string) (string, string, error) {
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set(pdcpauth.ApiKeyHeaderName, creds.APIKey)
 
-	resp, err := aiTemplateHTTPClient.Do(req)
+	resp, err := retryablehttp.DefaultClient().Do(req)
 	if err != nil {
 		return "", "", errkit.Newf("Failed to send HTTP request: %v", err)
 	}
@@ -142,9 +138,5 @@ func generateAITemplate(prompt string) (string, string, error) {
 		return "", "", errkit.Newf("Failed to generate template")
 	}
 
-	if !aiTemplateIDRegex.MatchString(result.TemplateID) {
-		return "", "", errkit.Newf("Invalid template ID in API response: %q", result.TemplateID)
-	}
-
 	return result.Completion, result.TemplateID, nil
 }

pkg/catalog/loader/ai_loader_test.go

@@ -1,55 +0,0 @@
-package loader
-
-import (
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"sync/atomic"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func newAITemplateServer(t *testing.T, templateID string, gotKey *atomic.Value) {
-	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		gotKey.Store(r.Header.Get("X-Api-Key"))
-		_ = json.NewEncoder(w).Encode(map[string]string{
-			"completion":  "id: test\n",
-			"template_id": templateID,
-		})
-	}))
-	t.Cleanup(srv.Close)
-
-	oldEndpoint, oldClient := aiTemplateGeneratorAPIEndpoint, aiTemplateHTTPClient
-	t.Cleanup(func() { aiTemplateGeneratorAPIEndpoint, aiTemplateHTTPClient = oldEndpoint, oldClient })
-	aiTemplateGeneratorAPIEndpoint = srv.URL
-	t.Setenv("PDCP_API_KEY", "test-key")
-	if templateID != "" {
-		// Trust the test certificate, to check the template_id only.
-		aiTemplateHTTPClient = srv.Client()
-	}
-}
-
-func TestAITemplateVerifiesTLS(t *testing.T) {
-	var gotKey atomic.Value
-	newAITemplateServer(t, "", &gotKey)
-
-	_, _, err := generateAITemplate("check for exposed admin panels")
-	require.ErrorContains(t, err, "certificate")
-	require.Nil(t, gotKey.Load(), "the API key went to an untrusted server")
-}
-
-func TestAITemplateRejectsUnsafeID(t *testing.T) {
-	for _, id := range []string{"../../.config/nuclei/config", "a/b", `a\b`, "a.b", "a b"} {
-		var gotKey atomic.Value
-		newAITemplateServer(t, id, &gotKey)
-		_, _, err := generateAITemplate("check for exposed admin panels")
-		require.Error(t, err, id)
-	}
-
-	var gotKey atomic.Value
-	newAITemplateServer(t, "Abc-123_x", &gotKey)
-	_, templateID, err := generateAITemplate("check for exposed admin panels")
-	require.NoError(t, err)
-	require.Equal(t, "Abc-123_x", templateID)
-}

pkg/fuzz/execute.go

@@ -155,14 +155,23 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) {
 	baseInteractURLs := append([]string(nil), input.InteractURLs...)
 	if rule.generator == nil {
 		for _, component := range finalComponentList {
+			var dataKeys map[string]struct{}
+
 			optionVars := rule.options.Options.Vars.AsMap()
 			constants := rule.options.Constants
 			dataValues := rule.options.NewVariablesScope(baseValues, optionVars, constants)
-			// Render variables once, allocating their interactsh URLs. The
-			// output may contain runtime data, so it is never rendered again.
+			// get vars from variables while replacing interactsh urls
 			evaluation := rule.options.Variables.EvaluateWithInteractshScope(dataValues, rule.options.Interactsh)
-			input.Values = generators.MergeMaps(evaluation.Values, baseValues, optionVars, constants)
-			input.InteractURLs = mergeInteractURLs(baseInteractURLs, evaluation.InteractURLs)
+			evaluatedValues, interactURLs := evaluation.Values, evaluation.InteractURLs
+			input.Values, dataKeys = mergeFuzzValueLayers(
+				fuzzValueLayer{values: evaluatedValues, kind: fuzzValueTemplate},
+				fuzzValueLayer{values: baseValues, kind: fuzzValueData},
+				fuzzValueLayer{values: optionVars, kind: fuzzValueData},
+				fuzzValueLayer{values: constants, kind: fuzzValueData},
+			)
+			// Replace template-text interactsh markers without evaluating runtime values.
+			input.Values, interactURLs = rule.evaluateVarsWithInteractsh(input.Values, interactURLs, dataKeys)
+			input.InteractURLs = mergeInteractURLs(baseInteractURLs, interactURLs)
 
 			err := rule.executeRuleValues(input, component)
 			if err != nil {
@@ -231,17 +240,15 @@ func (rule *Rule) prepareGeneratorValues(values, baseValues map[string]interface
 	evaluation := rule.options.Variables.EvaluateWithInteractshScope(evaluationValues, rule.options.Interactsh)
 	evaluatedValues, interactURLs := evaluation.Values, evaluation.InteractURLs
 	inputValues, dataKeys := mergeFuzzValueLayers(
-		fuzzValueLayer{values: evaluatedValues, kind: fuzzValueData},
 		fuzzValueLayer{values: values, kind: fuzzValueTemplate},
+		fuzzValueLayer{values: evaluatedValues, kind: fuzzValueTemplate},
 		fuzzValueLayer{values: baseValues, kind: fuzzValueData},
 		fuzzValueLayer{values: optionVars, kind: fuzzValueData},
 		fuzzValueLayer{values: constants, kind: fuzzValueData},
 	)
 
 	// Generator values are template-authored payload definitions until this
-	// render. Rendered variables, runtime/base, CLI option, and constant
-	// values remain data. Payloads layer over variables so a payload that
-	// shares a variable's name is still rendered.
+	// render. Runtime/base, CLI option, and constant values remain data.
 	return rule.evaluateVarsWithInteractsh(inputValues, interactURLs, dataKeys)
 }
 
@@ -319,7 +326,7 @@ func (rule *Rule) evaluateVars(input string) (string, error) {
 func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, interactshUrls []string, dataKeys map[string]struct{}) (map[string]interface{}, []string) {
 	// Check if Interactsh options are configured
 	if rule.options.Interactsh != nil {
-		rendered := maps.Clone(data)
+		data = maps.Clone(data)
 
 		interactshUrlsMap := make(map[string]struct{})
 		for _, url := range interactshUrls {
@@ -328,8 +335,7 @@ func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, intera
 
 		interactshUrls = mapsutil.GetKeys(interactshUrlsMap)
 
-		// Render each template-text value against the unrendered input, so
-		// the result does not depend on map iteration order.
+		// Iterate through template-text data to replace Interactsh URL markers.
 		for k, v := range data {
 			if _, ok := dataKeys[k]; ok {
 				continue
@@ -343,7 +349,7 @@ func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, intera
 				InteractURLs: interactshUrls,
 			})
 			if err == nil {
-				rendered[k] = got.Text
+				data[k] = got.Text
 				for _, url := range got.InteractURLs {
 					if _, ok := interactshUrlsMap[url]; ok {
 						continue
@@ -354,8 +360,6 @@ func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, intera
 				}
 			}
 		}
-
-		data = rendered
 	}
 
 	return data, interactshUrls

pkg/fuzz/execute_values_test.go

@@ -1,224 +0,0 @@
-package fuzz
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"strings"
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
-	"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/component"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/projectdiscovery/retryablehttp-go"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	// requestBodyWithDSL is request data that looks like a DSL expression.
-	requestBodyWithDSL = `PLAIN-{{to_upper("abc")}}-END`
-	queryRuleTarget    = "http://127.0.0.1/echo?p=SEED"
-)
-
-// fakeInteractshClient hands out numbered URLs without contacting a server.
-// Fuzz rules only allocate URLs through NewURLWithData, so the embedded nil
-// IClient is never called.
-type fakeInteractshClient struct {
-	interactsh.IClient
-
-	issued int
-}
-
-func (c *fakeInteractshClient) NewURLWithData(string) (string, error) {
-	c.issued++
-
-	return fmt.Sprintf("oast%d.example", c.issued), nil
-}
-
-// newQueryRule compiles a single-mode rule that replaces the p query value
-// with fuzzValue.
-func newQueryRule(
-	t *testing.T,
-	fuzzValue string,
-	generator *generators.PayloadGenerator,
-	templateVars variables.Variable,
-	client interactsh.IClient,
-) *Rule {
-	t.Helper()
-
-	rule := &Rule{
-		Part: component.RequestQueryComponent,
-		Type: "replace",
-		Mode: "single",
-		Fuzz: SliceOrMapSlice{Value: []string{fuzzValue}},
-	}
-	err := rule.Compile(generator, &protocols.ExecutorOptions{
-		Variables:  templateVars,
-		Options:    &types.Options{},
-		Interactsh: client,
-	})
-	require.NoError(t, err)
-
-	return rule
-}
-
-// executeQueryRule runs rule against queryRuleTarget and returns the generated
-// requests.
-func executeQueryRule(
-	t *testing.T,
-	rule *Rule,
-	method string,
-	body io.Reader,
-	values map[string]any,
-) []GeneratedRequest {
-	t.Helper()
-
-	req, err := retryablehttp.NewRequest(method, queryRuleTarget, body)
-	require.NoError(t, err)
-
-	var got []GeneratedRequest
-	err = rule.Execute(&ExecuteRuleInput{
-		Input:       contextargs.NewWithInput(context.Background(), queryRuleTarget),
-		BaseRequest: req,
-		Values:      values,
-		Callback: func(gr GeneratedRequest) bool {
-			got = append(got, gr)
-
-			return true
-		},
-	})
-	require.NoError(t, err)
-
-	return got
-}
-
-// newRequestDataVariableRule returns a query rule whose template declares
-// b: "{{body}}", so b captures request data.
-func newRequestDataVariableRule(t *testing.T, fuzzValue string, payloads map[string]any) *Rule {
-	t.Helper()
-
-	templateVars := newFuzzTestVariables()
-	templateVars.Set("b", "{{body}}")
-
-	var generator *generators.PayloadGenerator
-
-	if payloads != nil {
-		var err error
-
-		generator, err = generators.New(
-			payloads, generators.BatteringRamAttack, "", disk.NewCatalog(""), "", types.DefaultOptions(),
-		)
-		require.NoError(t, err)
-	}
-
-	// The CLI creates an Interactsh client even with -no-interactsh.
-	return newQueryRule(t, fuzzValue, generator, templateVars, &interactsh.Client{})
-}
-
-// fuzzedQueryValues runs rule against a POST request whose body is
-// requestBodyWithDSL and returns the fuzzed p query value of each generated
-// request.
-func fuzzedQueryValues(t *testing.T, rule *Rule) []string {
-	t.Helper()
-
-	generated := executeQueryRule(t, rule, http.MethodPost, strings.NewReader(requestBodyWithDSL), map[string]any{
-		"body": requestBodyWithDSL,
-	})
-
-	got := make([]string, 0, len(generated))
-	for _, gr := range generated {
-		got = append(got, gr.Request.URL.Query().Get("p"))
-	}
-
-	return got
-}
-
-func TestExecuteKeepsRequestDataInTemplateVariableLiteral(t *testing.T) {
-	t.Parallel()
-
-	t.Run("fuzz value", func(t *testing.T) {
-		t.Parallel()
-
-		rule := newRequestDataVariableRule(t, "{{b}}", nil)
-
-		require.Equal(t, []string{requestBodyWithDSL}, fuzzedQueryValues(t, rule))
-	})
-
-	t.Run("payload value", func(t *testing.T) {
-		t.Parallel()
-
-		rule := newRequestDataVariableRule(t, "{{inj}}", map[string]any{
-			"inj": []any{"{{b}}"},
-		})
-
-		// Each run iterates a fresh value map. Repeat so the result
-		// cannot depend on map iteration order.
-		for range 256 {
-			require.Equal(t, []string{requestBodyWithDSL}, fuzzedQueryValues(t, rule))
-		}
-	})
-}
-
-func TestExecuteRendersTemplateVariableInteractshMarker(t *testing.T) {
-	t.Parallel()
-
-	templateVars := newFuzzTestVariables()
-	templateVars.Set("marker", "{{interactsh-url}}")
-	rule := newQueryRule(t, "{{marker}}", nil, templateVars, &fakeInteractshClient{})
-
-	got := executeQueryRule(t, rule, http.MethodGet, nil, nil)
-
-	require.Len(t, got, 1)
-	require.Equal(t, []string{"oast1.example"}, got[0].InteractURLs)
-	require.Equal(t, "oast1.example", got[0].Request.URL.Query().Get("p"))
-}
-
-func TestPrepareGeneratorValuesRendersPayloadSharingVariableName(t *testing.T) {
-	t.Parallel()
-
-	templateVars := newFuzzTestVariables()
-	templateVars.Set("inj", "variable")
-
-	rule := &Rule{
-		options: &protocols.ExecutorOptions{
-			Variables:  templateVars,
-			Options:    &types.Options{},
-			Interactsh: &fakeInteractshClient{},
-		},
-	}
-
-	values, _ := rule.prepareGeneratorValues(map[string]any{
-		"inj": `{{to_upper("payload")}}`,
-	}, nil)
-
-	require.Equal(t, "PAYLOAD", values["inj"])
-}
-
-func TestPrepareGeneratorValuesRendersPayloadsAgainstUnrenderedInput(t *testing.T) {
-	t.Parallel()
-
-	rule := &Rule{
-		options: &protocols.ExecutorOptions{
-			Variables:  newFuzzTestVariables(),
-			Options:    &types.Options{},
-			Interactsh: &interactsh.Client{},
-		},
-	}
-
-	for range 256 {
-		values, _ := rule.prepareGeneratorValues(map[string]any{
-			"upper": `{{to_upper("x")}}`,
-			"ref":   "{{upper}}",
-		}, nil)
-
-		require.Equal(t, "X", values["upper"])
-		// ref sees the unrendered definition of upper as data.
-		require.Equal(t, `{{to_upper("x")}}`, values["ref"])
-	}
-}

pkg/js/compiler/close_test.go

@@ -1,53 +0,0 @@
-package compiler_test
-
-//nolint:depguard // No depguard rules are configured, so its default list allows only the standard library.
-import (
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-)
-
-// TestCloseWithoutConnection calls Close on clients that never connected.
-// The script must keep running past the call.
-func TestCloseWithoutConnection(t *testing.T) {
-	t.Parallel()
-
-	scripts := map[string]string{
-		"ssh": `
-			const ssh = require('nuclei/ssh');
-			ExportAs('closed', new ssh.SSHClient().Close());
-			true;
-		`,
-		"net": `
-			const net = require('nuclei/net');
-			new net.NetConn().Close();
-			ExportAs('closed', true);
-			true;
-		`,
-	}
-	for name, script := range scripts {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			program, err := compiler.SourceAutoMode(script, false)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			opts := new(compiler.ExecuteOptions)
-			opts.ExecutionId = t.Name()
-			opts.Source = &script // ExportAs is defined only for sources that use it.
-			opts.TimeoutVariants = types.NewTimeoutVariant(5)
-
-			result, err := compiler.New().ExecuteWithOptions(t.Context(), program, compiler.NewExecuteArgs(), opts)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if got := result["closed"]; got != "true" {
-				t.Errorf(`result["closed"] = %v, want "true"`, got)
-			}
-		})
-	}
-}

pkg/js/compiler/pool.go

@@ -24,7 +24,6 @@ import (
 	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libhash"
 	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libhttp"
 	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libikev2"
-	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libipmi"
 	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libkerberos"
 	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libkrbforge"
 	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libkrbroast"

pkg/js/generated/go/libipmi/ipmi.go

@@ -1,39 +0,0 @@
-package ipmi
-
-import (
-	lib_ipmi "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/ipmi"
-
-	"github.com/projectdiscovery/goja"
-	"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
-)
-
-var (
-	module = gojs.NewGojaModule("nuclei/ipmi")
-)
-
-func init() {
-	module.Set(
-		gojs.Objects{
-			// Functions
-			"CheckRAKPPassword":   lib_ipmi.CheckRAKPPassword,
-			"FindRAKPPassword":    lib_ipmi.FindRAKPPassword,
-			"GetAuthCapabilities": lib_ipmi.GetAuthCapabilities,
-			"GetCipherSuites":     lib_ipmi.GetCipherSuites,
-			"GetRAKPHash":         lib_ipmi.GetRAKPHash,
-			"GetSystemGUID":       lib_ipmi.GetSystemGUID,
-			"Ping":                lib_ipmi.Ping,
-
-			// Var and consts
-
-			// Objects / Classes
-			"AuthCapabilities": gojs.GetClassConstructor[lib_ipmi.AuthCapabilities](&lib_ipmi.AuthCapabilities{}),
-			"Client":           gojs.GetClassConstructor[lib_ipmi.Client](&lib_ipmi.Client{}),
-			"CommandResponse":  gojs.GetClassConstructor[lib_ipmi.CommandResponse](&lib_ipmi.CommandResponse{}),
-			"RAKPHash":         gojs.GetClassConstructor[lib_ipmi.RAKPHash](&lib_ipmi.RAKPHash{}),
-		},
-	).Register()
-}
-
-func Enable(runtime *goja.Runtime) {
-	module.Enable(runtime)
-}

pkg/js/generated/ts/index.ts

@@ -8,7 +8,6 @@ export * as grpc from './grpc';
 export * as hash from './hash';
 export * as http from './http';
 export * as ikev2 from './ikev2';
-export * as ipmi from './ipmi';
 export * as kerberos from './kerberos';
 export * as krbforge from './krbforge';
 export * as krbroast from './krbroast';

pkg/js/generated/ts/ipmi.ts

@@ -1,304 +0,0 @@
-
-
-/**
- * CheckRAKPPassword reports whether password produced hash. It runs offline,
- * so it can test any number of passwords without touching the BMC.
- * @example
- * ```javascript
- * const ipmi = require('nuclei/ipmi');
- * const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
- * log(ipmi.CheckRAKPPassword(hash, 'ADMIN'));
- * ```
- */
-export function CheckRAKPPassword(hash: RAKPHash, password: string): boolean {
-    return false;
-}
-
-
-
-/**
- * FindRAKPPassword returns the first of passwords that produced hash, or an
- * empty string.
- * @example
- * ```javascript
- * const ipmi = require('nuclei/ipmi');
- * const hash = ipmi.GetRAKPHash('acme.com', 623, 'root');
- * log(ipmi.FindRAKPPassword(hash, ['calvin', 'root', 'admin']));
- * ```
- */
-export function FindRAKPPassword(hash: RAKPHash, passwords: string[]): string {
-    return "";
-}
-
-
-
-/**
- * GetAuthCapabilities returns the authentication capabilities the BMC
- * advertises on the current channel.
- * @example
- * ```javascript
- * const ipmi = require('nuclei/ipmi');
- * const caps = ipmi.GetAuthCapabilities('acme.com', 623);
- * log(caps.AnonymousLogin);
- * ```
- */
-export function GetAuthCapabilities(ctx: any, host: string, port: number): AuthCapabilities | null {
-    return null;
-}
-
-
-
-/**
- * GetCipherSuites returns the IDs of the RMCP+ cipher suites the BMC
- * offers. Cipher suite 0 authenticates nothing, so any password logs in.
- * @example
- * ```javascript
- * const ipmi = require('nuclei/ipmi');
- * const suites = ipmi.GetCipherSuites('acme.com', 623);
- * log(suites.includes(0));
- * ```
- */
-export function GetCipherSuites(ctx: any, host: string, port: number): number[] | null {
-    return null;
-}
-
-
-
-/**
- * GetRAKPHash starts an RMCP+ session as username and returns the HMAC the
- * BMC answers with. The session is never completed, so no login is
- * attempted; the BMC drops the half-open session after its timeout. It fails
- * when the BMC rejects the username.
- * @example
- * ```javascript
- * const ipmi = require('nuclei/ipmi');
- * const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
- * log(ipmi.CheckRAKPPassword(hash, 'ADMIN'));
- * ```
- */
-export function GetRAKPHash(ctx: any, host: string, port: number, username: string): RAKPHash | null {
-    return null;
-}
-
-
-
-/**
- * GetSystemGUID returns the BMC system GUID, which vendors derive from
- * serial numbers or MAC addresses.
- * @example
- * ```javascript
- * const ipmi = require('nuclei/ipmi');
- * const guid = ipmi.GetSystemGUID('acme.com', 623);
- * log(guid);
- * ```
- */
-export function GetSystemGUID(ctx: any, host: string, port: number): string | null {
-    return null;
-}
-
-
-
-/**
- * Ping reports whether the host answers an RMCP presence ping and advertises
- * IPMI support.
- * @example
- * ```javascript
- * const ipmi = require('nuclei/ipmi');
- * const isIPMI = ipmi.Ping('acme.com', 623);
- * log(isIPMI);
- * ```
- */
-export function Ping(ctx: any, host: string, port: number): boolean | null {
-    return null;
-}
-
-
-
-/**
- * Client holds one RMCP+ session, for commands that need a login.
- * Close it when done: BMCs allow only a few sessions at a time.
- * @example
- * ```javascript
- * const ipmi = require('nuclei/ipmi');
- * const client = new ipmi.Client();
- * client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
- * const id = client.Command(0x06, 0x01, []);
- * client.Close();
- * ```
- */
-export class Client {
-    
-
-    // Constructor of Client
-    constructor() {}
-    /**
-    * Login opens an RMCP+ session as username with the given cipher suite
-    * (0, 1, 2, 3 or 6). Cipher suite 0 skips authentication, so a BMC
-    * that allows it accepts any password. A rejected login is an error.
-    * @example
-    * ```javascript
-    * const ipmi = require('nuclei/ipmi');
-    * const client = new ipmi.Client();
-    * client.Login('acme.com', 623, 'ADMIN', 'not-the-password', 0);
-    * ```
-    */
-    public Login(ctx: any, host: string, port: number, username: string, password: string, cipherSuite: number): boolean | null {
-        return null;
-    }
-    
-
-    /**
-    * Command sends an IPMI request in the session, such as Get Device ID
-    * (netFn 0x06, cmd 0x01) or Set Session Privilege Level (0x06, 0x3b).
-    * @example
-    * ```javascript
-    * const ipmi = require('nuclei/ipmi');
-    * const client = new ipmi.Client();
-    * client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
-    * const admin = client.Command(0x06, 0x3b, [0x04]).CompletionCode == 0;
-    * client.Close();
-    * ```
-    */
-    public Command(netFn: number, cmd: number, data: Uint8Array): CommandResponse | null {
-        return null;
-    }
-    
-
-    /**
-    * Close ends the session on the BMC, freeing its slot.
-    * @example
-    * ```javascript
-    * const ipmi = require('nuclei/ipmi');
-    * const client = new ipmi.Client();
-    * client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
-    * client.Close();
-    * ```
-    */
-    public Close(): void {
-        return;
-    }
-    
-
-}
-
-
-
-/**
- * AuthCapabilities is what a BMC advertises before any login, from Get
- * Channel Authentication Capabilities.
- * @example
- * ```javascript
- * const ipmi = require('nuclei/ipmi');
- * const caps = ipmi.GetAuthCapabilities('acme.com', 623);
- * log(toJSON(caps));
- * ```
- */
-export interface AuthCapabilities {
-    
-    /**
-    * IPMI20 is true when the BMC supports IPMI 2.0 (RMCP+).
-    */
-    
-    IPMI20?: boolean,
-    
-    /**
-    * AuthTypes are the IPMI 1.5 authentication types the channel
-    * accepts: none, md2, md5, straight (cleartext password), oem.
-    */
-    
-    AuthTypes?: string[],
-    
-    /**
-    * AnonymousLogin is true when a null username with a null password
-    * is accepted.
-    */
-    
-    AnonymousLogin?: boolean,
-    
-    /**
-    * NullUsernames is true when accounts with an empty name are enabled.
-    */
-    
-    NullUsernames?: boolean,
-    
-    /**
-    * NonNullUsernames is true when named accounts are enabled.
-    */
-    
-    NonNullUsernames?: boolean,
-    
-    /**
-    * PerMessageAuth is false when the BMC skips authenticating
-    * messages after login.
-    */
-    
-    PerMessageAuth?: boolean,
-    
-    /**
-    * UserLevelAuth is false when user-level commands skip
-    * authentication.
-    */
-    
-    UserLevelAuth?: boolean,
-    
-    /**
-    * OEMID is the IANA enterprise number of the BMC vendor, or 0.
-    */
-    
-    OEMID?: number,
-}
-
-
-
-/**
- * CommandResponse is the reply to an IPMI command.
- */
-export interface CommandResponse {
-    
-    /**
-    * CompletionCode is 0 on success.
-    */
-    
-    CompletionCode?: number,
-    
-    Data?: Uint8Array,
-}
-
-
-
-/**
- * RAKPHash is the HMAC a BMC returns for a user during RMCP+ session
- * setup (CVE-2013-4786). It can be checked offline against candidate
- * passwords, which tests credentials without a login attempt.
- * @example
- * ```javascript
- * const ipmi = require('nuclei/ipmi');
- * const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
- * log(hash.Hashcat);
- * ```
- */
-export interface RAKPHash {
-    
-    Username?: string,
-    
-    /**
-    * Algorithm is the HMAC the BMC used: sha1 or md5.
-    */
-    
-    Algorithm?: string,
-    
-    /**
-    * Salt and HMAC are hex encoded; Salt is everything the HMAC covers.
-    */
-    
-    Salt?: string,
-    
-    HMAC?: string,
-    
-    /**
-    * Hashcat is the hash in hashcat mode 7300 format, for sha1 only.
-    */
-    
-    Hashcat?: string,
-}
-

pkg/js/generated/ts/net.ts

@@ -46,7 +46,6 @@ export class NetConn {
     constructor() {}
     /**
     * Close closes the connection.
-    * Close does nothing if no connection is open.
     * @example
     * ```javascript
     * const net = require('nuclei/net');

pkg/js/generated/ts/postgres.ts

@@ -2,7 +2,7 @@
 
 /**
  * PGClient is a client for Postgres database.
- * Internally client uses lib/pq driver.
+ * Internally client uses go-pg/pg driver.
  * @example
  * ```javascript
  * const postgres = require('nuclei/postgres');

pkg/js/generated/ts/ssh.ts

@@ -106,8 +106,7 @@ export class SSHClient {
     /**
     * Close closes the SSH connection and destroys the client
     * Returns the success state and error. If error is not nil,
-    * state will be false. Close does nothing and returns true
-    * if no connection is open.
+    * state will be false
     * @example
     * ```javascript
     * const ssh = require('nuclei/ssh');

pkg/js/gojs/gojs.go

@@ -128,13 +128,9 @@ func (p *GojaModule) Register() Module {
 	return p
 }
 
-// GetClassConstructor returns a constructor for any given go struct type for goja runtime.
-// Each object it constructs starts as a shallow copy of instance, so objects
-// do not share state.
+// GetClassConstructor returns a constructor for any given go struct type for goja runtime
 func GetClassConstructor[T any](instance *T) func(call goja.ConstructorCall, runtime *goja.Runtime) *goja.Object {
 	return func(call goja.ConstructorCall, runtime *goja.Runtime) *goja.Object {
-		obj := *instance
-
-		return utils.LinkConstructor(call, runtime, &obj)
+		return utils.LinkConstructor[*T](call, runtime, instance)
 	}
 }

pkg/js/gojs/gojs_test.go

@@ -1,46 +0,0 @@
-package gojs_test
-
-//nolint:depguard // No depguard rules are configured, so its default list allows only the standard library.
-import (
-	"testing"
-
-	"github.com/projectdiscovery/goja"
-	"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
-)
-
-type counter struct{ n int }
-
-func (c *counter) Inc() int {
-	c.n++
-
-	return c.n
-}
-
-func TestGetClassConstructorReturnsNewObjects(t *testing.T) {
-	t.Parallel()
-
-	prototype := new(counter)
-	runtime := goja.New()
-
-	err := runtime.Set("Counter", gojs.GetClassConstructor(prototype))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	value, err := runtime.RunString(`
-		const first = new Counter();
-		first.Inc();
-		new Counter().Inc();
-	`)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if got := value.ToInteger(); got != 1 {
-		t.Errorf("new Counter().Inc() = %d, want 1", got)
-	}
-
-	if prototype.n != 0 {
-		t.Errorf("prototype.n = %d, want 0", prototype.n)
-	}
-}

pkg/js/libs/ipmi/ipmi.go

@@ -1,394 +0,0 @@
-package ipmi
-
-import (
-	"context"
-	"crypto/hmac"
-	"crypto/md5"
-	"crypto/sha1"
-	"encoding/hex"
-	"errors"
-	"fmt"
-)
-
-type (
-	// AuthCapabilities is what a BMC advertises before any login, from Get
-	// Channel Authentication Capabilities.
-	// @example
-	// ```javascript
-	// const ipmi = require('nuclei/ipmi');
-	// const caps = ipmi.GetAuthCapabilities('acme.com', 623);
-	// log(toJSON(caps));
-	// ```
-	AuthCapabilities struct {
-		// IPMI20 is true when the BMC supports IPMI 2.0 (RMCP+).
-		IPMI20 bool
-		// AuthTypes are the IPMI 1.5 authentication types the channel
-		// accepts: none, md2, md5, straight (cleartext password), oem.
-		AuthTypes []string
-		// AnonymousLogin is true when a null username with a null password
-		// is accepted.
-		AnonymousLogin bool
-		// NullUsernames is true when accounts with an empty name are enabled.
-		NullUsernames bool
-		// NonNullUsernames is true when named accounts are enabled.
-		NonNullUsernames bool
-		// PerMessageAuth is false when the BMC skips authenticating
-		// messages after login.
-		PerMessageAuth bool
-		// UserLevelAuth is false when user-level commands skip
-		// authentication.
-		UserLevelAuth bool
-		// OEMID is the IANA enterprise number of the BMC vendor, or 0.
-		OEMID int
-	}
-
-	// RAKPHash is the HMAC a BMC returns for a user during RMCP+ session
-	// setup (CVE-2013-4786). It can be checked offline against candidate
-	// passwords, which tests credentials without a login attempt.
-	// @example
-	// ```javascript
-	// const ipmi = require('nuclei/ipmi');
-	// const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
-	// log(hash.Hashcat);
-	// ```
-	RAKPHash struct {
-		Username string
-		// Algorithm is the HMAC the BMC used: sha1 or md5.
-		Algorithm string
-		// Salt and HMAC are hex encoded; Salt is everything the HMAC covers.
-		Salt string
-		HMAC string
-		// Hashcat is the hash in hashcat mode 7300 format, for sha1 only.
-		Hashcat string
-	}
-
-	// Client holds one RMCP+ session, for commands that need a login.
-	// Close it when done: BMCs allow only a few sessions at a time.
-	// @example
-	// ```javascript
-	// const ipmi = require('nuclei/ipmi');
-	// const client = new ipmi.Client();
-	// client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
-	// const id = client.Command(0x06, 0x01, []);
-	// client.Close();
-	// ```
-	Client struct {
-		session *established
-	}
-
-	// CommandResponse is the reply to an IPMI command.
-	CommandResponse struct {
-		// CompletionCode is 0 on success.
-		CompletionCode int
-		Data           []byte
-	}
-)
-
-var authTypes = []struct {
-	bit  byte
-	name string
-}{{0x01, "none"}, {0x02, "md2"}, {0x04, "md5"}, {0x10, "straight"}, {0x20, "oem"}}
-
-// Ping reports whether the host answers an RMCP presence ping and advertises
-// IPMI support.
-// @example
-// ```javascript
-// const ipmi = require('nuclei/ipmi');
-// const isIPMI = ipmi.Ping('acme.com', 623);
-// log(isIPMI);
-// ```
-func Ping(ctx context.Context, host string, port int) (bool, error) {
-	s, err := dial(ctx, host, port)
-	if err != nil {
-		return false, err
-	}
-	defer s.close()
-	return s.ping()
-}
-
-// GetAuthCapabilities returns the authentication capabilities the BMC
-// advertises on the current channel.
-// @example
-// ```javascript
-// const ipmi = require('nuclei/ipmi');
-// const caps = ipmi.GetAuthCapabilities('acme.com', 623);
-// log(caps.AnonymousLogin);
-// ```
-func GetAuthCapabilities(ctx context.Context, host string, port int) (*AuthCapabilities, error) {
-	s, err := dial(ctx, host, port)
-	if err != nil {
-		return nil, err
-	}
-	defer s.close()
-
-	// bit 7 asks for the IPMI 2.0 extended data; an IPMI 1.5 only BMC
-	// rejects it, so ask again without
-	data, err := s.command(cmdGetChannelAuthCaps, []byte{0x80 | currentChannel, privAdmin})
-	if err != nil {
-		if data, err = s.command(cmdGetChannelAuthCaps, []byte{currentChannel, privAdmin}); err != nil {
-			return nil, err
-		}
-	}
-	return parseAuthCapabilities(data)
-}
-
-// parseAuthCapabilities decodes the Get Channel Authentication Capabilities
-// response data (IPMI v2.0 table 22-15).
-func parseAuthCapabilities(data []byte) (*AuthCapabilities, error) {
-	if len(data) < 8 {
-		return nil, fmt.Errorf("short channel authentication capabilities response")
-	}
-	caps := &AuthCapabilities{
-		IPMI20:           data[1]&0x80 != 0 && data[3]&0x02 != 0,
-		AnonymousLogin:   data[2]&0x01 != 0,
-		NullUsernames:    data[2]&0x02 != 0,
-		NonNullUsernames: data[2]&0x04 != 0,
-		UserLevelAuth:    data[2]&0x08 == 0,
-		PerMessageAuth:   data[2]&0x10 == 0,
-		OEMID:            int(data[4]) | int(data[5])<<8 | int(data[6])<<16,
-	}
-	for _, authType := range authTypes {
-		if data[1]&authType.bit != 0 {
-			caps.AuthTypes = append(caps.AuthTypes, authType.name)
-		}
-	}
-	return caps, nil
-}
-
-// GetSystemGUID returns the BMC system GUID, which vendors derive from
-// serial numbers or MAC addresses.
-// @example
-// ```javascript
-// const ipmi = require('nuclei/ipmi');
-// const guid = ipmi.GetSystemGUID('acme.com', 623);
-// log(guid);
-// ```
-func GetSystemGUID(ctx context.Context, host string, port int) (string, error) {
-	s, err := dial(ctx, host, port)
-	if err != nil {
-		return "", err
-	}
-	defer s.close()
-
-	data, err := s.command(cmdGetSystemGUID, nil)
-	if err != nil {
-		return "", err
-	}
-	if len(data) < 16 {
-		return "", fmt.Errorf("short system guid response")
-	}
-	return hex.EncodeToString(data[:16]), nil
-}
-
-// GetCipherSuites returns the IDs of the RMCP+ cipher suites the BMC
-// offers. Cipher suite 0 authenticates nothing, so any password logs in.
-// @example
-// ```javascript
-// const ipmi = require('nuclei/ipmi');
-// const suites = ipmi.GetCipherSuites('acme.com', 623);
-// log(suites.includes(0));
-// ```
-func GetCipherSuites(ctx context.Context, host string, port int) ([]int, error) {
-	s, err := dial(ctx, host, port)
-	if err != nil {
-		return nil, err
-	}
-	defer s.close()
-
-	// the records come in pages of 16 bytes; a shorter page is the last
-	var records []byte
-	for index := byte(0); index < 64; index++ {
-		data, err := s.command(cmdGetCipherSuites, []byte{currentChannel, 0x00, 0x80 | index})
-		if err != nil {
-			return nil, err
-		}
-		if len(data) > 0 {
-			records = append(records, data[1:]...)
-		}
-		if len(data) < 17 {
-			break
-		}
-	}
-
-	return parseCipherSuites(records), nil
-}
-
-// parseCipherSuites returns the suite IDs in Get Channel Cipher Suites
-// record data. A record starts with 0xc0 (standard) or 0xc1 (oem, followed
-// by a 3 byte IANA); algorithm bytes are tagged below 0xc0, so they never
-// look like a record start.
-func parseCipherSuites(records []byte) []int {
-	suites := []int{}
-	for i := 0; i < len(records); i++ {
-		switch records[i] {
-		case 0xc0:
-			if i+1 < len(records) {
-				suites = append(suites, int(records[i+1]))
-				i++
-			}
-		case 0xc1:
-			if i+1 < len(records) {
-				suites = append(suites, int(records[i+1]))
-				i += 4
-			}
-		}
-	}
-	return suites
-}
-
-// GetRAKPHash starts an RMCP+ session as username and returns the HMAC the
-// BMC answers with. The session is never completed, so no login is
-// attempted; the BMC drops the half-open session after its timeout. It fails
-// when the BMC rejects the username.
-// @example
-// ```javascript
-// const ipmi = require('nuclei/ipmi');
-// const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
-// log(ipmi.CheckRAKPPassword(hash, 'ADMIN'));
-// ```
-func GetRAKPHash(ctx context.Context, host string, port int, username string) (*RAKPHash, error) {
-	s, err := dial(ctx, host, port)
-	if err != nil {
-		return nil, err
-	}
-	defer s.close()
-
-	// the session is never completed, so no login is attempted; md5 is for
-	// BMCs that refuse sha1
-	var h *handshake
-	for _, auth := range []byte{authHMACSHA1, authHMACMD5} {
-		if h, err = s.startSession(algorithms{auth: auth}, username, privAdmin); err == nil {
-			break
-		}
-		var refused *rakpError
-		if !errors.As(err, &refused) || refused.step != "the session" {
-			return nil, err
-		}
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	hash := &RAKPHash{Username: username, Algorithm: "sha1", Salt: hex.EncodeToString(h.rakp2Salt()), HMAC: hex.EncodeToString(h.bmcMAC)}
-	if h.auth == authHMACMD5 {
-		hash.Algorithm = "md5"
-	} else {
-		hash.Hashcat = hash.Salt + ":" + hash.HMAC
-	}
-	return hash, nil
-}
-
-// CheckRAKPPassword reports whether password produced hash. It runs offline,
-// so it can test any number of passwords without touching the BMC.
-// @example
-// ```javascript
-// const ipmi = require('nuclei/ipmi');
-// const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
-// log(ipmi.CheckRAKPPassword(hash, 'ADMIN'));
-// ```
-func CheckRAKPPassword(hash *RAKPHash, password string) bool {
-	if hash == nil {
-		return false
-	}
-	salt, err := hex.DecodeString(hash.Salt)
-	if err != nil {
-		return false
-	}
-	want, err := hex.DecodeString(hash.HMAC)
-	if err != nil {
-		return false
-	}
-	newHash := sha1.New
-	if hash.Algorithm == "md5" {
-		newHash = md5.New
-	}
-	return hmac.Equal(hmacOf(newHash, []byte(password), salt), want)
-}
-
-// FindRAKPPassword returns the first of passwords that produced hash, or an
-// empty string.
-// @example
-// ```javascript
-// const ipmi = require('nuclei/ipmi');
-// const hash = ipmi.GetRAKPHash('acme.com', 623, 'root');
-// log(ipmi.FindRAKPPassword(hash, ['calvin', 'root', 'admin']));
-// ```
-func FindRAKPPassword(hash *RAKPHash, passwords []string) string {
-	for _, password := range passwords {
-		if CheckRAKPPassword(hash, password) {
-			return password
-		}
-	}
-	return ""
-}
-
-// Login opens an RMCP+ session as username with the given cipher suite
-// (0, 1, 2, 3 or 6). Cipher suite 0 skips authentication, so a BMC
-// that allows it accepts any password. A rejected login is an error.
-// @example
-// ```javascript
-// const ipmi = require('nuclei/ipmi');
-// const client = new ipmi.Client();
-// client.Login('acme.com', 623, 'ADMIN', 'not-the-password', 0);
-// ```
-func (c *Client) Login(ctx context.Context, host string, port int, username string, password string, cipherSuite int) (bool, error) {
-	alg, ok := cipherSuites[cipherSuite]
-	if !ok {
-		return false, fmt.Errorf("unsupported cipher suite %d", cipherSuite)
-	}
-	c.Close()
-	s, err := dial(ctx, host, port)
-	if err != nil {
-		return false, err
-	}
-	h, err := s.startSession(alg, username, privAdmin)
-	if err != nil {
-		s.close()
-		return false, err
-	}
-	k, err := s.finishSession(h, password)
-	if err != nil {
-		s.close()
-		return false, err
-	}
-	c.session = &established{session: s, algorithms: alg, keys: k, bmcSessionID: h.bmcSessionID, consoleSessionID: h.consoleSessionID}
-	return true, nil
-}
-
-// Command sends an IPMI request in the session, such as Get Device ID
-// (netFn 0x06, cmd 0x01) or Set Session Privilege Level (0x06, 0x3b).
-// @example
-// ```javascript
-// const ipmi = require('nuclei/ipmi');
-// const client = new ipmi.Client();
-// client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
-// const admin = client.Command(0x06, 0x3b, [0x04]).CompletionCode == 0;
-// client.Close();
-// ```
-func (c *Client) Command(netFn int, cmd int, data []byte) (*CommandResponse, error) {
-	if c.session == nil {
-		return nil, fmt.Errorf("not logged in")
-	}
-	code, reply, err := c.session.command(byte(netFn), byte(cmd), data)
-	if err != nil {
-		return nil, err
-	}
-	return &CommandResponse{CompletionCode: int(code), Data: reply}, nil
-}
-
-// Close ends the session on the BMC, freeing its slot.
-// @example
-// ```javascript
-// const ipmi = require('nuclei/ipmi');
-// const client = new ipmi.Client();
-// client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
-// client.Close();
-// ```
-func (c *Client) Close() {
-	if c.session == nil {
-		return
-	}
-	_, _, _ = c.session.command(netFnApp, cmdCloseSession, c.session.bmcSessionID)
-	c.session.close()
-	c.session = nil
-}

pkg/js/libs/ipmi/ipmi_test.go

@@ -1,178 +0,0 @@
-package ipmi
-
-import (
-	"bytes"
-	"context"
-	"crypto/sha1"
-	"encoding/hex"
-	"net"
-	"testing"
-	"time"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/stretchr/testify/require"
-)
-
-// The fixtures were captured from OpenIPMI's ipmi_sim with an enabled user
-// with an empty name and password, ADMIN/ADMIN and root/calvin.
-
-func mustHex(t *testing.T, s string) []byte {
-	t.Helper()
-	data, err := hex.DecodeString(s)
-	require.NoError(t, err)
-	return data
-}
-
-func TestParseAuthCapabilities(t *testing.T) {
-	caps, err := parseAuthCapabilities(mustHex(t, "0197050300000000"))
-	require.NoError(t, err)
-	require.Equal(t, &AuthCapabilities{
-		IPMI20:           true,
-		AuthTypes:        []string{"none", "md2", "md5", "straight"},
-		AnonymousLogin:   true,
-		NonNullUsernames: true,
-		PerMessageAuth:   true,
-		UserLevelAuth:    true,
-	}, caps)
-
-	_, err = parseAuthCapabilities(mustHex(t, "0197"))
-	require.Error(t, err)
-}
-
-func TestParseCipherSuites(t *testing.T) {
-	records := mustHex(t, "c000004080c001014080c002014180c003014181c004014182c005014183c006024080c007024280c008024281c009024282c00a024283c00b024380c00c024381c00d024382c00e024383")
-	require.Equal(t, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, parseCipherSuites(records))
-
-	// an oem record carries a 3 byte IANA before its algorithms
-	require.Equal(t, []int{1, 7}, parseCipherSuites(mustHex(t, "c0010140c107d70200014080")))
-}
-
-func TestRAKPPasswordChecksAreOffline(t *testing.T) {
-	// hashcat -m 7300 cracks this hash to calvin as well
-	hash := &RAKPHash{
-		Username: "root",
-		Salt:     "988200cd0402000069b30c7a5e3deb30504c5b76f7b1840d5e36439998d9e0f59ae49b09119a10b2a123456789abcdefa123456789abcdef1404726f6f74",
-		HMAC:     "23c711e5c7a737c3c0beb9f4680ef31fa6aad398",
-	}
-	require.True(t, CheckRAKPPassword(hash, "calvin"))
-	require.False(t, CheckRAKPPassword(hash, "ADMIN"))
-	require.Equal(t, "calvin", FindRAKPPassword(hash, []string{"admin", "root", "calvin"}))
-	require.Empty(t, FindRAKPPassword(hash, []string{"admin"}))
-	require.False(t, CheckRAKPPassword(nil, "calvin"))
-	require.False(t, CheckRAKPPassword(&RAKPHash{Salt: "zz", HMAC: "00"}, "calvin"))
-}
-
-func TestRequestsHonourNetworkPolicy(t *testing.T) {
-	execID := "ipmi-deny"
-	require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID, ExcludeTargets: []string{"203.0.113.50"}}))
-	t.Cleanup(func() { protocolstate.Close(execID) })
-	ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
-
-	_, err := GetRAKPHash(ctx, "203.0.113.50", 623, "ADMIN")
-	require.ErrorContains(t, err, "denied")
-	require.NotErrorIs(t, err, errNoResponse, "denied before sending, not timed out")
-	_, err = Ping(ctx, "203.0.113.50", 623)
-	require.ErrorContains(t, err, "denied")
-}
-
-func TestPayloadEncryptionRoundTrips(t *testing.T) {
-	key := mustHex(t, "000102030405060708090a0b0c0d0e0f")
-	for size := range 40 {
-		payload := make([]byte, size)
-		for i := range payload {
-			payload[i] = byte(i)
-		}
-		sealed, err := encryptPayload(key, payload)
-		require.NoError(t, err)
-		require.Zero(t, len(sealed)%16, "iv and padded blocks")
-		opened, err := decryptPayload(key, sealed)
-		require.NoError(t, err)
-		require.Equal(t, payload, opened)
-	}
-	_, err := decryptPayload(key, []byte{1, 2, 3})
-	require.Error(t, err)
-}
-
-func TestSignAlignsTheIntegrityPad(t *testing.T) {
-	e := &established{algorithms: cipherSuites[3], keys: &keys{integrity: make([]byte, 20)}}
-	for size := range 8 {
-		packet := append(rmcpHeader(rmcpClassIPMI), make([]byte, 12+size)...)
-		signed := e.sign(packet)
-		// everything from the auth type through the next header is a
-		// multiple of 4, then 12 bytes of HMAC-SHA1-96
-		require.Zero(t, (len(signed)-12-4)%4)
-		require.Equal(t, byte(0x07), signed[len(signed)-13])
-	}
-}
-
-func TestLoginRejectsUnsupportedCipherSuites(t *testing.T) {
-	_, err := (&Client{}).Login(context.Background(), "127.0.0.1", 623, "ADMIN", "ADMIN", 17)
-	require.ErrorContains(t, err, "unsupported cipher suite 17")
-	_, err = (&Client{}).Command(0x06, 0x01, nil)
-	require.ErrorContains(t, err, "not logged in")
-}
-
-// capturedHandshake is a cipher suite 3 session setup with ipmi_sim for
-// ADMIN/ADMIN, up to and including RAKP message 4.
-func capturedHandshake(t *testing.T) (*handshake, []byte) {
-	return &handshake{
-		algorithms:       cipherSuites[3],
-		consoleSessionID: mustHex(t, "244a5ac8"),
-		bmcSessionID:     mustHex(t, "04170000"),
-		consoleRandom:    mustHex(t, "de862025bb0c5ee6c30a58d383d44cb5"),
-		bmcRandom:        mustHex(t, "b56fef2ee0fcab6664a019a4c88df4f6"),
-		bmcGUID:          mustHex(t, "a123456789abcdefa123456789abcdef"),
-		role:             0x14,
-		username:         "ADMIN",
-		bmcMAC:           mustHex(t, "7fac6f776553c58df508b9ff656b5eacb6c2def9"),
-	}, mustHex(t, "00000000244a5ac8231219c5fc212cdd58c09424")
-}
-
-func TestBMCProofsMatchOnlyTheRealPassword(t *testing.T) {
-	h, rakp4 := capturedHandshake(t)
-	sik := func(password string) []byte {
-		return hmacOf(sha1.New, []byte(password), h.consoleRandom, h.bmcRandom, []byte{h.role, byte(len(h.username))}, []byte(h.username))
-	}
-
-	require.Equal(t, h.bmcMAC, hmacOf(sha1.New, []byte("ADMIN"), h.rakp2Salt()), "rakp 2 code")
-	require.NotEqual(t, h.bmcMAC, hmacOf(sha1.New, []byte("wrong"), h.rakp2Salt()))
-
-	require.True(t, rakp4Valid(sha1.New, sik("ADMIN"), h, rakp4))
-	require.False(t, rakp4Valid(sha1.New, sik("wrong"), h, rakp4))
-	tampered := bytes.Clone(rakp4)
-	tampered[len(tampered)-1] ^= 0xff
-	require.False(t, rakp4Valid(sha1.New, sik("ADMIN"), h, tampered), "a forged success must not pass")
-	require.False(t, rakp4Valid(sha1.New, sik("ADMIN"), h, rakp4[:10]))
-}
-
-func TestShortCommandReplyIsRejectedNotPanicked(t *testing.T) {
-	execID := "ipmi-short"
-	require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
-	t.Cleanup(func() { protocolstate.Close(execID) })
-	ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
-
-	// a reply that ends at the completion code, without the checksum
-	bmc, err := net.ListenPacket("udp", "127.0.0.1:0")
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = bmc.Close() })
-	go func() {
-		buf := make([]byte, 512)
-		for {
-			n, addr, err := bmc.ReadFrom(buf)
-			if err != nil {
-				return
-			}
-			reply := bytes.Clone(buf[:21])
-			reply[19] = buf[n-2] // echo the command
-			reply[20] = 0x00
-			_, _ = bmc.WriteTo(reply, addr)
-		}
-	}()
-
-	port := bmc.LocalAddr().(*net.UDPAddr).Port
-	ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
-	defer cancel()
-	_, err = GetSystemGUID(ctx, "127.0.0.1", port)
-	require.ErrorIs(t, err, errNoResponse)
-}

pkg/js/libs/ipmi/rmcp.go

@@ -1,173 +0,0 @@
-package ipmi
-
-import (
-	"bytes"
-	"context"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"net"
-	"time"
-
-	libnet "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/net"
-)
-
-// Wire constants from the IPMI v2.0 specification and DSP0136 (ASF).
-const (
-	rmcpVersion   = 0x06
-	rmcpNoAck     = 0xff
-	rmcpClassASF  = 0x06
-	rmcpClassIPMI = 0x07
-
-	asfIANA         = 0x000011be
-	asfTypePing     = 0x80
-	asfTypePong     = 0x40
-	asfEntitiesIPMI = 0x80
-
-	authTypeNone  = 0x00
-	authTypeRMCPP = 0x06
-
-	netFnApp              = 0x06
-	cmdGetSystemGUID      = 0x37
-	cmdGetChannelAuthCaps = 0x38
-	cmdGetCipherSuites    = 0x54
-
-	bmcAddress     = 0x20
-	consoleAddress = 0x81
-	currentChannel = 0x0e
-	privAdmin      = 0x04
-
-	payloadOpenSessionRequest  = 0x10
-	payloadOpenSessionResponse = 0x11
-	payloadRAKP1               = 0x12
-	payloadRAKP2               = 0x13
-	payloadRAKP3               = 0x14
-	payloadRAKP4               = 0x15
-
-	// rakpNameOnlyLookup asks the BMC to find the user by name alone, so the
-	// role does not have to match the one the account was created with.
-	rakpNameOnlyLookup = 0x10
-
-	attempts = 2
-)
-
-var errNoResponse = errors.New("no ipmi response")
-
-// session is one UDP exchange with a BMC. UDP is lossy, so each request is
-// sent up to attempts times.
-type session struct {
-	conn    net.Conn
-	timeout time.Duration
-}
-
-func dial(ctx context.Context, host string, port int) (*session, error) {
-	conn, err := libnet.Dial(ctx, "udp", net.JoinHostPort(host, fmt.Sprint(port)))
-	if err != nil {
-		return nil, err
-	}
-	timeout := 3 * time.Second
-	if deadline, ok := ctx.Deadline(); ok {
-		timeout = min(timeout, time.Until(deadline)/attempts)
-	}
-	return &session{conn: conn, timeout: timeout}, nil
-}
-
-func (s *session) close() { _ = s.conn.Close() }
-
-// exchange sends packet and returns the first reply accepted by match.
-func (s *session) exchange(packet []byte, match func([]byte) bool) ([]byte, error) {
-	buf := make([]byte, 1024)
-	for range attempts {
-		if _, err := s.conn.Write(packet); err != nil {
-			return nil, err
-		}
-		_ = s.conn.SetReadDeadline(time.Now().Add(s.timeout))
-		for {
-			n, err := s.conn.Read(buf)
-			if err != nil {
-				break
-			}
-			if reply := buf[:n]; match(reply) {
-				return bytes.Clone(reply), nil
-			}
-		}
-	}
-	return nil, errNoResponse
-}
-
-func rmcpHeader(class byte) []byte {
-	return []byte{rmcpVersion, 0x00, rmcpNoAck, class}
-}
-
-// ping sends an ASF presence ping and reports whether the pong advertises
-// IPMI support.
-func (s *session) ping() (bool, error) {
-	packet := append(rmcpHeader(rmcpClassASF), 0, 0, 0, 0, asfTypePing, 0x00, 0x00, 0x00)
-	binary.BigEndian.PutUint32(packet[4:8], asfIANA)
-	reply, err := s.exchange(packet, func(reply []byte) bool {
-		return len(reply) >= 12 && reply[3] == rmcpClassASF && reply[8] == asfTypePong
-	})
-	if err != nil {
-		return false, err
-	}
-	// pong data: IANA (4), OEM (4), supported entities (1), interactions (1)
-	if len(reply) < 21 {
-		return false, nil
-	}
-	return reply[20]&asfEntitiesIPMI != 0, nil
-}
-
-func checksum(data []byte) byte {
-	var sum byte
-	for _, b := range data {
-		sum += b
-	}
-	return -sum
-}
-
-// command sends a session-less IPMI 1.5 request and returns the response
-// data after a successful completion code.
-func (s *session) command(cmd byte, data []byte) ([]byte, error) {
-	header := []byte{bmcAddress, netFnApp << 2}
-	body := append([]byte{consoleAddress, 0x00, cmd}, data...)
-	message := append(append(header, checksum(header)), body...)
-	message = append(message, checksum(body))
-
-	packet := append(rmcpHeader(rmcpClassIPMI), authTypeNone, 0, 0, 0, 0, 0, 0, 0, 0, byte(len(message)))
-	packet = append(packet, message...)
-
-	reply, err := s.exchange(packet, func(reply []byte) bool {
-		// rmcp (4), session header (10), then rqAddr, netFn, checksum,
-		// rsAddr, seq, cmd, completion code
-		// ... completion code, then at least the trailing checksum
-		return len(reply) >= 22 && reply[3] == rmcpClassIPMI && reply[4] == authTypeNone && reply[19] == cmd
-	})
-	if err != nil {
-		return nil, err
-	}
-	if code := reply[20]; code != 0 {
-		return nil, fmt.Errorf("ipmi command 0x%02x failed with completion code 0x%02x", cmd, code)
-	}
-	// drop the trailing checksum
-	return reply[21 : len(reply)-1], nil
-}
-
-// rmcpPlus sends an IPMI 2.0 session-setup payload and returns the reply
-// payload of the expected type.
-func (s *session) rmcpPlus(payloadType byte, payload []byte, replyType byte) ([]byte, error) {
-	packet := append(rmcpHeader(rmcpClassIPMI), authTypeRMCPP, payloadType, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
-	binary.LittleEndian.PutUint16(packet[14:16], uint16(len(payload)))
-	packet = append(packet, payload...)
-
-	reply, err := s.exchange(packet, func(reply []byte) bool {
-		return len(reply) >= 16 && reply[3] == rmcpClassIPMI && reply[4] == authTypeRMCPP && reply[5]&0x3f == replyType
-	})
-	if err != nil {
-		return nil, err
-	}
-	length := int(binary.LittleEndian.Uint16(reply[14:16]))
-	if len(reply) < 16+length {
-		return nil, fmt.Errorf("truncated ipmi payload")
-	}
-	return reply[16 : 16+length], nil
-}

pkg/js/libs/ipmi/session.go

@@ -1,398 +0,0 @@
-package ipmi
-
-import (
-	"bytes"
-	"crypto/aes"
-	"crypto/cipher"
-	"crypto/hmac"
-	"crypto/md5"
-	"crypto/rand"
-	"crypto/sha1"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"hash"
-)
-
-// RMCP+ algorithm numbers (IPMI v2.0 tables 13-17 to 13-19).
-const (
-	authNone     = 0x00
-	authHMACSHA1 = 0x01
-	authHMACMD5  = 0x02
-
-	integrityNone       = 0x00
-	integrityHMACSHA196 = 0x01
-
-	confidentialityNone      = 0x00
-	confidentialityAESCBC128 = 0x01
-
-	payloadIPMI      = 0x00
-	payloadEncrypted = 0x80
-	payloadSigned    = 0x40
-
-	cmdCloseSession = 0x3c
-
-	// rakp status codes worth naming in errors
-	rakpInvalidRole      = 0x09
-	rakpUnauthorizedName = 0x0d
-)
-
-type algorithms struct {
-	auth, integrity, confidentiality byte
-}
-
-// cipherSuites lists the suites a session can be opened with. Suites with
-// SHA-256 or MD5 message integrity are left out until they can be checked
-// against a BMC that implements them correctly.
-var cipherSuites = map[int]algorithms{
-	0: {authNone, integrityNone, confidentialityNone},
-	1: {authHMACSHA1, integrityNone, confidentialityNone},
-	2: {authHMACSHA1, integrityHMACSHA196, confidentialityNone},
-	3: {authHMACSHA1, integrityHMACSHA196, confidentialityAESCBC128},
-	6: {authHMACMD5, integrityNone, confidentialityNone},
-}
-
-func authHash(auth byte) func() hash.Hash {
-	switch auth {
-	case authHMACSHA1:
-		return sha1.New
-	case authHMACMD5:
-		return md5.New
-	}
-	return nil
-}
-
-func hmacOf(newHash func() hash.Hash, key []byte, parts ...[]byte) []byte {
-	mac := hmac.New(newHash, key)
-	for _, part := range parts {
-		_, _ = mac.Write(part)
-	}
-	return mac.Sum(nil)
-}
-
-// handshake is the state of an RMCP+ session setup up to RAKP message 2.
-type handshake struct {
-	algorithms
-	consoleSessionID, bmcSessionID []byte
-	consoleRandom, bmcRandom       []byte
-	bmcGUID                        []byte
-	role                           byte
-	username                       string
-	// bmcMAC is the RAKP message 2 key exchange code, empty without
-	// authentication.
-	bmcMAC []byte
-}
-
-// rakpError is a non-zero status in an RMCP+ session setup reply.
-type rakpError struct {
-	step   string
-	status byte
-}
-
-func (e *rakpError) Error() string {
-	switch e.status {
-	case rakpUnauthorizedName:
-		return fmt.Sprintf("bmc rejected the username in %s (status 0x%02x)", e.step, e.status)
-	case rakpInvalidRole:
-		return fmt.Sprintf("bmc rejected the requested privilege in %s (status 0x%02x)", e.step, e.status)
-	}
-	return fmt.Sprintf("bmc refused %s with status 0x%02x", e.step, e.status)
-}
-
-// startSession opens an RMCP+ session and runs RAKP messages 1 and 2.
-func (s *session) startSession(alg algorithms, username string, privilege byte) (*handshake, error) {
-	if len(username) > 16 {
-		return nil, fmt.Errorf("ipmi usernames are at most 16 bytes")
-	}
-	h := &handshake{
-		algorithms:       alg,
-		consoleSessionID: make([]byte, 4),
-		consoleRandom:    make([]byte, 16),
-		role:             privilege | rakpNameOnlyLookup,
-		username:         username,
-	}
-	if _, err := rand.Read(h.consoleSessionID); err != nil {
-		return nil, err
-	}
-	if _, err := rand.Read(h.consoleRandom); err != nil {
-		return nil, err
-	}
-
-	open := []byte{0x00, privilege, 0x00, 0x00}
-	open = append(open, h.consoleSessionID...)
-	open = append(open,
-		0x00, 0x00, 0x00, 0x08, alg.auth, 0x00, 0x00, 0x00,
-		0x01, 0x00, 0x00, 0x08, alg.integrity, 0x00, 0x00, 0x00,
-		0x02, 0x00, 0x00, 0x08, alg.confidentiality, 0x00, 0x00, 0x00,
-	)
-	opened, err := s.rmcpPlus(payloadOpenSessionRequest, open, payloadOpenSessionResponse)
-	if err != nil {
-		return nil, err
-	}
-	if len(opened) < 2 {
-		return nil, fmt.Errorf("short open session response")
-	}
-	if opened[1] != 0 {
-		return nil, &rakpError{step: "the session", status: opened[1]}
-	}
-	if len(opened) < 12 {
-		return nil, fmt.Errorf("short open session response")
-	}
-	h.bmcSessionID = bytes.Clone(opened[8:12])
-
-	rakp1 := []byte{0x00, 0x00, 0x00, 0x00}
-	rakp1 = append(rakp1, h.bmcSessionID...)
-	rakp1 = append(rakp1, h.consoleRandom...)
-	rakp1 = append(rakp1, h.role, 0x00, 0x00, byte(len(username)))
-	rakp1 = append(rakp1, username...)
-	rakp2, err := s.rmcpPlus(payloadRAKP1, rakp1, payloadRAKP2)
-	if err != nil {
-		return nil, err
-	}
-	if len(rakp2) < 2 {
-		return nil, fmt.Errorf("short rakp message 2")
-	}
-	if rakp2[1] != 0 {
-		return nil, &rakpError{step: "rakp message 1", status: rakp2[1]}
-	}
-	if len(rakp2) < 40 {
-		return nil, fmt.Errorf("short rakp message 2")
-	}
-	h.bmcRandom = bytes.Clone(rakp2[8:24])
-	h.bmcGUID = bytes.Clone(rakp2[24:40])
-	h.bmcMAC = bytes.Clone(rakp2[40:])
-	return h, nil
-}
-
-// rakp2Salt is everything the RAKP message 2 code covers.
-func (h *handshake) rakp2Salt() []byte {
-	salt := make([]byte, 0, 58+len(h.username))
-	salt = append(salt, h.consoleSessionID...)
-	salt = append(salt, h.bmcSessionID...)
-	salt = append(salt, h.consoleRandom...)
-	salt = append(salt, h.bmcRandom...)
-	salt = append(salt, h.bmcGUID...)
-	salt = append(salt, h.role, byte(len(h.username)))
-	return append(salt, h.username...)
-}
-
-// keys are the session keys derived from the session integrity key.
-type keys struct {
-	integrity []byte // K1
-	aes       []byte // first 16 bytes of K2
-}
-
-// finishSession runs RAKP messages 3 and 4, which is where a wrong password
-// is rejected, and derives the session keys. Both directions are proven:
-// the BMC's RAKP message 2 code and RAKP message 4 integrity value must
-// match the password, so a peer that only answers with success statuses
-// cannot pass for a login.
-func (s *session) finishSession(h *handshake, password string) (*keys, error) {
-	if len(password) > 20 {
-		return nil, fmt.Errorf("ipmi passwords are at most 20 bytes")
-	}
-	newHash := authHash(h.auth)
-	if newHash != nil && !hmac.Equal(h.bmcMAC, hmacOf(newHash, []byte(password), h.rakp2Salt())) {
-		return nil, errBMCProof
-	}
-
-	rakp3 := []byte{0x00, 0x00, 0x00, 0x00}
-	rakp3 = append(rakp3, h.bmcSessionID...)
-	if newHash != nil {
-		rakp3 = append(rakp3, hmacOf(newHash, []byte(password),
-			h.bmcRandom, h.consoleSessionID, []byte{h.role, byte(len(h.username))}, []byte(h.username))...)
-	}
-	rakp4, err := s.rmcpPlus(payloadRAKP3, rakp3, payloadRAKP4)
-	if err != nil {
-		return nil, err
-	}
-	if len(rakp4) < 2 {
-		return nil, fmt.Errorf("short rakp message 4")
-	}
-	if rakp4[1] != 0 {
-		return nil, &rakpError{step: "the login", status: rakp4[1]}
-	}
-	if newHash == nil {
-		return &keys{}, nil
-	}
-
-	sik := hmacOf(newHash, []byte(password),
-		h.consoleRandom, h.bmcRandom, []byte{h.role, byte(len(h.username))}, []byte(h.username))
-	if !rakp4Valid(newHash, sik, h, rakp4) {
-		return nil, errBMCProof
-	}
-	return &keys{
-		integrity: hmacOf(newHash, sik, bytes.Repeat([]byte{0x01}, 20)),
-		aes:       hmacOf(newHash, sik, bytes.Repeat([]byte{0x02}, 20))[:16],
-	}, nil
-}
-
-// errBMCProof means the peer's authentication code does not match the
-// password: a wrong password, or something other than the BMC answering.
-var errBMCProof = errors.New("bmc authentication code does not match the password")
-
-// rakp4Valid checks the RAKP message 4 integrity value, the session key's
-// HMAC over the console random, BMC session ID and BMC GUID, truncated to
-// 12 bytes for sha1 and 16 for md5.
-func rakp4Valid(newHash func() hash.Hash, sik []byte, h *handshake, rakp4 []byte) bool {
-	size := 12
-	if h.auth == authHMACMD5 {
-		size = 16
-	}
-	if len(rakp4) < 8+size {
-		return false
-	}
-	want := hmacOf(newHash, sik, h.consoleRandom, h.bmcSessionID, h.bmcGUID)[:size]
-	return hmac.Equal(rakp4[8:8+size], want)
-}
-
-// established is an authenticated RMCP+ session.
-type established struct {
-	*session
-	algorithms
-	keys             *keys
-	bmcSessionID     []byte
-	consoleSessionID []byte
-	sequence         uint32
-	requestSeq       byte
-}
-
-// command sends an IPMI request inside the session and returns its
-// completion code and response data.
-func (e *established) command(netFn, cmd byte, data []byte) (byte, []byte, error) {
-	// the request sequence is 6 bits on the wire
-	e.requestSeq = (e.requestSeq + 1) & 0x3f
-	header := []byte{bmcAddress, netFn << 2}
-	body := append([]byte{consoleAddress, e.requestSeq << 2, cmd}, data...)
-	message := append(append(header, checksum(header)), body...)
-	message = append(message, checksum(body))
-
-	payloadType := byte(payloadIPMI)
-	if e.confidentiality == confidentialityAESCBC128 {
-		payloadType |= payloadEncrypted
-		var err error
-		if message, err = encryptPayload(e.keys.aes, message); err != nil {
-			return 0, nil, err
-		}
-	}
-	if e.integrity != integrityNone {
-		payloadType |= payloadSigned
-	}
-
-	e.sequence++
-	packet := append(rmcpHeader(rmcpClassIPMI), authTypeRMCPP, payloadType)
-	packet = append(packet, e.bmcSessionID...)
-	packet = binary.LittleEndian.AppendUint32(packet, e.sequence)
-	packet = binary.LittleEndian.AppendUint16(packet, uint16(len(message)))
-	packet = append(packet, message...)
-	if e.integrity != integrityNone {
-		packet = e.sign(packet)
-	}
-
-	// a reply must be for this session, pass the integrity check when the
-	// session is signed, and answer this request, not an earlier one that
-	// arrived late
-	var payload []byte
-	_, err := e.exchange(packet, func(reply []byte) bool {
-		payload = e.open(reply)
-		return len(payload) >= 8 && payload[4]>>2 == e.requestSeq && payload[5] == cmd
-	})
-	if err != nil {
-		return 0, nil, err
-	}
-	// rqAddr, netFn, checksum, rsAddr, seq, cmd, completion code, data,
-	// checksum
-	return payload[6], bytes.Clone(payload[7 : len(payload)-1]), nil
-}
-
-// open returns the IPMI message in a session reply, or nil when the reply
-// is not for this session or fails its integrity check.
-func (e *established) open(reply []byte) []byte {
-	if len(reply) < 16 || reply[3] != rmcpClassIPMI || reply[4] != authTypeRMCPP || reply[5]&0x3f != payloadIPMI {
-		return nil
-	}
-	if !bytes.Equal(reply[6:10], e.consoleSessionID) {
-		return nil
-	}
-	length := int(binary.LittleEndian.Uint16(reply[14:16]))
-	if len(reply) < 16+length {
-		return nil
-	}
-	if e.integrity != integrityNone {
-		if reply[5]&payloadSigned == 0 || !e.verify(reply, 16+length) {
-			return nil
-		}
-	}
-	payload := reply[16 : 16+length]
-	if reply[5]&payloadEncrypted != 0 {
-		var err error
-		if payload, err = decryptPayload(e.keys.aes, payload); err != nil {
-			return nil
-		}
-	}
-	return payload
-}
-
-// verify checks a reply's HMAC-SHA1-96 trailer: the pad after the payload,
-// pad length and next header, then the code over everything from the auth
-// type on.
-func (e *established) verify(reply []byte, payloadEnd int) bool {
-	const codeSize = 12
-	if len(reply) < payloadEnd+2+codeSize {
-		return false
-	}
-	codeStart := len(reply) - codeSize
-	if reply[codeStart-1] != 0x07 || int(reply[codeStart-2]) != codeStart-2-payloadEnd {
-		return false
-	}
-	want := hmacOf(sha1.New, e.keys.integrity, reply[4:codeStart])[:codeSize]
-	return hmac.Equal(reply[codeStart:], want)
-}
-
-// sign appends the HMAC-SHA1-96 integrity trailer: pad to a multiple of 4 from the auth
-// type on, pad length, next header, and the truncated HMAC over it all.
-func (e *established) sign(packet []byte) []byte {
-	pad := (4 - (len(packet)-4+2)%4) % 4
-	packet = append(packet, bytes.Repeat([]byte{0xff}, pad)...)
-	packet = append(packet, byte(pad), 0x07)
-	return append(packet, hmacOf(sha1.New, e.keys.integrity, packet[4:])[:12]...)
-}
-
-// encryptPayload is AES-CBC-128 with a random IV in front and the
-// confidentiality pad (1, 2, 3, ... then its length) at the end.
-func encryptPayload(key, payload []byte) ([]byte, error) {
-	block, err := aes.NewCipher(key)
-	if err != nil {
-		return nil, err
-	}
-	pad := (aes.BlockSize - (len(payload)+1)%aes.BlockSize) % aes.BlockSize
-	plain := bytes.Clone(payload)
-	for i := 1; i <= pad; i++ {
-		plain = append(plain, byte(i))
-	}
-	plain = append(plain, byte(pad))
-
-	out := make([]byte, aes.BlockSize+len(plain))
-	if _, err := rand.Read(out[:aes.BlockSize]); err != nil {
-		return nil, err
-	}
-	cipher.NewCBCEncrypter(block, out[:aes.BlockSize]).CryptBlocks(out[aes.BlockSize:], plain)
-	return out, nil
-}
-
-func decryptPayload(key, payload []byte) ([]byte, error) {
-	if len(payload) < 2*aes.BlockSize || len(payload)%aes.BlockSize != 0 {
-		return nil, fmt.Errorf("malformed encrypted payload")
-	}
-	block, err := aes.NewCipher(key)
-	if err != nil {
-		return nil, err
-	}
-	plain := make([]byte, len(payload)-aes.BlockSize)
-	cipher.NewCBCDecrypter(block, payload[:aes.BlockSize]).CryptBlocks(plain, payload[aes.BlockSize:])
-	pad := int(plain[len(plain)-1])
-	if pad+1 > len(plain) {
-		return nil, fmt.Errorf("malformed encrypted payload")
-	}
-	return plain[:len(plain)-pad-1], nil
-}

pkg/js/libs/net/net.go

@@ -6,7 +6,6 @@ import (
 	"crypto/tls"
 	"encoding/base64"
 	"encoding/hex"
-	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -21,8 +20,6 @@ import (
 
 var (
 	defaultTimeout = time.Duration(5) * time.Second
-
-	errNoConnection = errors.New("no connection: open one with net.Open or net.OpenTLS")
 )
 
 // getTimeoutFromContext returns the configured TCP timeout or defaultTimeout.
@@ -274,14 +271,12 @@ type (
 	// const conn = net.Open('tcp', 'acme.com:80');
 	// ```
 	NetConn struct {
-		// conn is nil when no connection is open.
 		conn    net.Conn
 		timeout time.Duration
 	}
 )
 
 // Close closes the connection.
-// Close does nothing if no connection is open.
 // @example
 // ```javascript
 // const net = require('nuclei/net');
@@ -289,13 +284,7 @@ type (
 // conn.Close();
 // ```
 func (c *NetConn) Close() error {
-	if c.conn == nil {
-		return nil
-	}
-
 	err := c.conn.Close()
-	c.conn = nil
-
 	return err
 }
 
@@ -332,10 +321,6 @@ func (c *NetConn) unsetDeadLine() {
 // conn.SendArray(['hello', 'world']);
 // ```
 func (c *NetConn) SendArray(data []interface{}) error {
-	if c.conn == nil {
-		return errNoConnection
-	}
-
 	c.setDeadLine()
 	defer c.unsetDeadLine()
 	input := types.ToByteSlice(data)
@@ -357,10 +342,6 @@ func (c *NetConn) SendArray(data []interface{}) error {
 // conn.SendHex('68656c6c6f');
 // ```
 func (c *NetConn) SendHex(data string) error {
-	if c.conn == nil {
-		return errNoConnection
-	}
-
 	c.setDeadLine()
 	defer c.unsetDeadLine()
 	bin, err := hex.DecodeString(data)
@@ -385,10 +366,6 @@ func (c *NetConn) SendHex(data string) error {
 // conn.Send('hello');
 // ```
 func (c *NetConn) Send(data string) error {
-	if c.conn == nil {
-		return errNoConnection
-	}
-
 	c.setDeadLine()
 	defer c.unsetDeadLine()
 	bin := []byte(data)
@@ -412,10 +389,6 @@ func (c *NetConn) Send(data string) error {
 // const data = conn.RecvFull(1024);
 // ```
 func (c *NetConn) RecvFull(N int) ([]byte, error) {
-	if c.conn == nil {
-		return []byte{}, errNoConnection
-	}
-
 	c.setDeadLine()
 	defer c.unsetDeadLine()
 	if N == 0 {
@@ -441,10 +414,6 @@ func (c *NetConn) RecvFull(N int) ([]byte, error) {
 // log(`Received ${data.length} bytes from the server`)
 // ```
 func (c *NetConn) Recv(N int) ([]byte, error) {
-	if c.conn == nil {
-		return []byte{}, errNoConnection
-	}
-
 	c.setDeadLine()
 	defer c.unsetDeadLine()
 	if N == 0 {

pkg/js/libs/net/net_test.go

@@ -406,41 +406,3 @@ func (f *fakeConn) RemoteAddr() net.Addr             { return nil }
 func (f *fakeConn) SetDeadline(time.Time) error      { return nil }
 func (f *fakeConn) SetReadDeadline(time.Time) error  { return nil }
 func (f *fakeConn) SetWriteDeadline(time.Time) error { return nil }
-
-func TestNetConnWithoutConnection(t *testing.T) {
-	t.Parallel()
-
-	tests := map[string]func(t *testing.T) *NetConn{
-		"never opened": func(*testing.T) *NetConn { return new(NetConn) },
-		"already closed": func(t *testing.T) *NetConn {
-			t.Helper()
-
-			local, remote := net.Pipe()
-
-			t.Cleanup(func() { _ = remote.Close() })
-
-			conn := new(NetConn)
-			conn.conn = local
-			require.NoError(t, conn.Close())
-
-			return conn
-		},
-	}
-	for name, newConn := range tests {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			conn := newConn(t)
-			require.NoError(t, conn.Close())
-			require.ErrorContains(t, conn.Send("x"), "no connection")
-			require.ErrorContains(t, conn.SendHex("78"), "no connection")
-			require.ErrorContains(t, conn.SendArray([]any{120}), "no connection")
-
-			_, err := conn.Recv(1)
-			require.ErrorContains(t, err, "no connection")
-
-			_, err = conn.RecvFull(1)
-			require.ErrorContains(t, err, "no connection")
-		})
-	}
-}

pkg/js/libs/postgres/postgres.go

@@ -3,12 +3,12 @@ package postgres
 import (
 	"context"
 	"fmt"
-	"os"
-	"path/filepath"
-	"strconv"
+	"net"
+	"net/url"
 	"strings"
 	"time"
 
+	"github.com/go-pg/pg/v10"
 	"github.com/praetorian-inc/fingerprintx/pkg/plugins"
 	postgres "github.com/praetorian-inc/fingerprintx/pkg/plugins/services/postgresql"
 	utils "github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
@@ -18,7 +18,7 @@ import (
 
 type (
 	// PGClient is a client for Postgres database.
-	// Internally client uses lib/pq driver.
+	// Internally client uses go-pg/pg driver.
 	// @example
 	// ```javascript
 	// const postgres = require('nuclei/postgres');
@@ -124,7 +124,9 @@ func executeQuery(ctx context.Context, executionId string, host string, port int
 		return nil, protocolstate.ErrHostDenied.Msgf(host)
 	}
 
-	connStr := buildPostgresDSN(host, port, username, password, dbName)
+	target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
+
+	connStr := buildPostgresConnURL(username, password, target, dbName, executionId)
 	db, err := pgwrap.OpenDB(ctx, executionId, connStr)
 	if err != nil {
 		return nil, err
@@ -144,30 +146,17 @@ func executeQuery(ctx context.Context, executionId string, host string, port int
 	return resp, nil
 }
 
-// buildPostgresDSN returns a lib/pq key=value connection string. user, password, dbname, service and passfile
-// are always set, also when empty: lib/pq fills unset values from the scanner's environment (PGUSER, PGPASSWORD,
-// PGDATABASE), a service file (PGSERVICE) and ~/.pgpass (PGPASSFILE), which sends the operator's credentials to the target.
-func buildPostgresDSN(host string, port int, username, password, dbName string) string {
-	quote := strings.NewReplacer(`\`, `\\`, `'`, `\'`)
-	params := []struct{ key, value string }{
-		{"host", host},
-		{"port", strconv.Itoa(port)},
-		{"user", username},
-		{"password", password},
-		{"dbname", dbName},
-		{"sslmode", "disable"},
-		{"connect_timeout", "10"},
-		{"service", ""},
-		{"options", ""},
-		{"application_name", ""},
-		// a path below a file never exists, so no password file is read
-		{"passfile", filepath.Join(os.DevNull, "pgpass")},
-	}
-	kvs := make([]string, 0, len(params))
-	for _, param := range params {
-		kvs = append(kvs, param.key+"='"+quote.Replace(param.value)+"'")
-	}
-	return strings.Join(kvs, " ")
+func buildPostgresConnURL(username, password, target, dbName, executionId string) string {
+	values := url.Values{}
+	values.Set("sslmode", "disable")
+	values.Set("executionId", executionId)
+
+	return fmt.Sprintf("postgres://%s@%s/%s?%s",
+		url.UserPassword(username, password).String(),
+		target,
+		url.PathEscape(dbName),
+		values.Encode(),
+	)
 }
 
 // ConnectWithDB connects to Postgres database using given credentials and database name.
@@ -205,20 +194,32 @@ func connect(ctx context.Context, executionId string, host string, port int, use
 		return false, protocolstate.ErrHostDenied.Msgf(host)
 	}
 
-	execCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
+
+	execCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	// lib/pq with every credential set, as go-pg replaces an empty user, password or database
-	// with PGUSER, PGPASSWORD or PGDATABASE from the scanner's environment
-	db, err := pgwrap.OpenDB(execCtx, executionId, buildPostgresDSN(host, port, username, password, dbName))
-	if err != nil {
-		return false, err
+	dialer := protocolstate.GetDialersWithId(executionId)
+	if dialer == nil {
+		return false, fmt.Errorf("dialers not initialized for %s", executionId)
 	}
+
+	db := pg.Connect(&pg.Options{
+		Addr:     target,
+		User:     username,
+		Password: password,
+		Database: dbName,
+		Dialer: func(dialCtx context.Context, network, addr string) (net.Conn, error) {
+			return dialer.Fastdialer.Dial(dialCtx, network, addr)
+		},
+		IdleCheckFrequency: -1,
+	}).WithTimeout(10 * time.Second)
+
 	defer func() {
 		_ = db.Close()
 	}()
 
-	_, err = db.ExecContext(execCtx, "select 1")
+	_, err := db.ExecContext(execCtx, "select 1")
 	if err != nil {
 		switch true {
 		case strings.Contains(err.Error(), "connect: connection refused"):

pkg/js/libs/postgres/postgres_test.go

@@ -1,165 +1,74 @@
 package postgres
 
 import (
-	"context"
-	"encoding/binary"
-	"io"
-	"net"
-	"os"
-	"path/filepath"
+	"net/url"
 	"strings"
 	"testing"
 
 	"github.com/lib/pq"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/stretchr/testify/require"
 )
 
-func TestBuildPostgresDSNEscapesDBName(t *testing.T) {
-	dbName := "testdb' sslrootcert='/etc/passwd' sslmode='verify-ca"
-	cfg, err := pq.NewConfig(buildPostgresDSN("127.0.0.1", 5432, "postgres", "x", dbName))
-	require.NoError(t, err)
+func TestBuildPostgresConnectionURLDoesNotAllowDBNameQueryInjection(t *testing.T) {
+	dbName := "testdb?sslrootcert=/etc/passwd&sslmode=verify-ca&junk="
+	connStr := buildPostgresConnURL("postgres", "x", "127.0.0.1:5432", dbName, "exec-1")
 
-	require.Equal(t, dbName, cfg.Database)
-	require.Equal(t, pq.SSLModeDisable, cfg.SSLMode)
-	require.Empty(t, cfg.SSLRootCert)
-	require.Empty(t, cfg.Runtime)
-}
+	u, err := url.Parse(connStr)
+	if err != nil {
+		t.Fatalf("parse connection URL: %v", err)
+	}
 
-func TestBuildPostgresDSNEscapesCredentials(t *testing.T) {
-	username := `user:[email protected]' host='other`
-	password := `pa:ss@word?x\' sslmode='require`
-	cfg, err := pq.NewConfig(buildPostgresDSN("127.0.0.1", 5432, username, password, "postgres"))
-	require.NoError(t, err)
+	if got := strings.TrimPrefix(u.Path, "/"); got != dbName {
+		t.Fatalf("database name = %q, want %q", got, dbName)
+	}
 
-	require.Equal(t, username, cfg.User)
-	require.Equal(t, password, cfg.Password)
-	require.Equal(t, "127.0.0.1", cfg.Host)
-	require.Equal(t, uint16(5432), cfg.Port)
-	require.Equal(t, "postgres", cfg.Database)
-	require.Equal(t, pq.SSLModeDisable, cfg.SSLMode)
-}
-
-// setOperatorPostgresEnv sets every source lib/pq and go-pg read credentials from
-func setOperatorPostgresEnv(t *testing.T) {
-	home := t.TempDir()
-	pgpass := filepath.Join(home, ".pgpass")
-	require.NoError(t, os.WriteFile(pgpass, []byte("*:*:*:*:OPERATOR-PGPASS-SECRET\n"), 0600))
-	serviceFile := filepath.Join(home, ".pg_service.conf")
-	require.NoError(t, os.WriteFile(serviceFile, []byte("[operator]\npassword=OPERATOR-SERVICE-SECRET\n"), 0600))
-
-	t.Setenv("HOME", home)
-	t.Setenv("PGPASSFILE", pgpass)
-	t.Setenv("PGSERVICEFILE", serviceFile)
-	t.Setenv("PGSERVICE", "operator")
-	t.Setenv("PGPASSWORD", "OPERATOR-ENV-PGPASSWORD")
-	t.Setenv("PGUSER", "operator-user")
-	t.Setenv("PGDATABASE", "operator-db")
-	t.Setenv("PGOPTIONS", "-c search_path=operator")
-	t.Setenv("PGAPPNAME", "operator-app")
-}
-
-func TestBuildPostgresDSNIgnoresOperatorEnvironment(t *testing.T) {
-	setOperatorPostgresEnv(t)
-
-	cfg, err := pq.NewConfig(buildPostgresDSN("127.0.0.1", 5432, "", "", ""))
-	require.NoError(t, err)
-	require.Empty(t, cfg.User)
-	require.Empty(t, cfg.Password)
-	require.Empty(t, cfg.Database)
-	require.Empty(t, cfg.Service)
-	require.Empty(t, cfg.Options)
-	require.Empty(t, cfg.ApplicationName)
-}
-
-type startup struct {
-	user, database, password string
-}
-
-// fakePostgres asks every client for a cleartext password, records it and rejects the login
-func fakePostgres(t *testing.T) (int, <-chan startup) {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = listener.Close() })
-
-	logins := make(chan startup, 16)
-	go func() {
-		for {
-			conn, err := listener.Accept()
-			if err != nil {
-				return
-			}
-			go func(conn net.Conn) {
-				defer func() { _ = conn.Close() }()
-				readMessage := func(typed bool) []byte {
-					header := make([]byte, 4)
-					if typed {
-						header = make([]byte, 5)
-					}
-					if _, err := io.ReadFull(conn, header); err != nil {
-						return nil
-					}
-					body := make([]byte, binary.BigEndian.Uint32(header[len(header)-4:])-4)
-					if _, err := io.ReadFull(conn, body); err != nil {
-						return nil
-					}
-					return body
-				}
-				params := readMessage(false)
-				if len(params) < 4 {
-					return
-				}
-				fields := strings.Split(string(params[4:]), "\x00")
-				var login startup
-				for i := 0; i+1 < len(fields); i += 2 {
-					switch fields[i] {
-					case "user":
-						login.user = fields[i+1]
-					case "database":
-						login.database = fields[i+1]
-					}
-				}
-				// AuthenticationCleartextPassword
-				if _, err := conn.Write([]byte{'R', 0, 0, 0, 8, 0, 0, 0, 3}); err != nil {
-					return
-				}
-				password := readMessage(true)
-				login.password = strings.TrimSuffix(string(password), "\x00")
-				logins <- login
-				errorBody := "SFATAL\x00C28P01\x00Mpassword authentication failed\x00\x00"
-				header := []byte{'E', 0, 0, 0, 0}
-				binary.BigEndian.PutUint32(header[1:], uint32(len(errorBody)+4))
-				_, _ = conn.Write(append(header, errorBody...))
-			}(conn)
+	values := u.Query()
+	if got := values.Get("sslmode"); got != "disable" {
+		t.Fatalf("sslmode = %q, want disable", got)
+	}
+	if got := values.Get("executionId"); got != "exec-1" {
+		t.Fatalf("executionId = %q, want exec-1", got)
+	}
+	deniedParams := []string{"sslrootcert", "sslcert", "sslkey", "service", "junk"}
+	for _, denied := range deniedParams {
+		if got := values.Get(denied); got != "" {
+			t.Fatalf("%s was injected with value %q", denied, got)
 		}
-	}()
-	return listener.Addr().(*net.TCPAddr).Port, logins
+	}
+
+	pqDSN, err := pq.ParseURL(connStr) //nolint:staticcheck // validates lib/pq URL parsing of the generated DSN.
+	if err != nil {
+		t.Fatalf("parse connection URL as lib/pq DSN: %v", err)
+	}
+	if !strings.Contains(pqDSN, "dbname='"+dbName+"'") {
+		t.Fatalf("lib/pq DSN = %q, want dbName preserved as dbname", pqDSN)
+	}
+	for _, denied := range deniedParams {
+		if strings.Contains(pqDSN, " "+denied+"=") {
+			t.Fatalf("%s was injected into lib/pq DSN %q", denied, pqDSN)
+		}
+	}
 }
 
-func TestPostgresEmptyCredentialsAreNotFilledFromEnvironment(t *testing.T) {
-	setOperatorPostgresEnv(t)
+func TestBuildPostgresConnectionURLEscapesCredentials(t *testing.T) {
+	username := "user:[email protected]"
+	password := "pa:ss@word?x"
+	connStr := buildPostgresConnURL(username, password, "127.0.0.1:5432", "postgres", "exec-1")
 
-	executionID := t.Name()
-	require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: executionID}))
-	t.Cleanup(func() { protocolstate.Close(executionID) })
-
-	port, logins := fakePostgres(t)
-	ctx := context.Background()
-
-	_, err := executeQuery(ctx, executionID, "127.0.0.1", port, "postgres", "", "postgres", "select 1")
-	require.Error(t, err)
-	require.Equal(t, startup{user: "postgres", database: "postgres"}, <-logins)
-
-	connected, err := connect(ctx, executionID, "127.0.0.1", port, "postgres", "", "postgres")
-	require.NoError(t, err)
-	require.False(t, connected)
-	require.Equal(t, startup{user: "postgres", database: "postgres"}, <-logins)
-
-	_, _ = connect(ctx, executionID, "127.0.0.1", port, "", "", "")
-	require.Equal(t, startup{}, <-logins)
-
-	_, _ = executeQuery(ctx, executionID, "127.0.0.1", port, "admin", "secret", "db", "select 1")
-	require.Equal(t, startup{user: "admin", database: "db", password: "secret"}, <-logins)
+	u, err := url.Parse(connStr)
+	if err != nil {
+		t.Fatalf("parse connection URL: %v", err)
+	}
 
+	if got := u.User.Username(); got != username {
+		t.Fatalf("username = %q, want %q", got, username)
+	}
+	if got, _ := u.User.Password(); got != password {
+		t.Fatalf("password = %q, want %q", got, password)
+	}
+	if got := u.Host; got != "127.0.0.1:5432" {
+		t.Fatalf("host = %q, want 127.0.0.1:5432", got)
+	}
+	if got := strings.TrimPrefix(u.Path, "/"); got != "postgres" {
+		t.Fatalf("database name = %q, want postgres", got)
+	}
 }

pkg/js/libs/ssh/ssh.go

@@ -23,7 +23,6 @@ type (
 	// const client = new ssh.SSHClient();
 	// ```
 	SSHClient struct {
-		// connection is nil when no connection is open.
 		connection *ssh.Client
 		timeout    time.Duration
 	}
@@ -159,8 +158,7 @@ func (c *SSHClient) Run(cmd string) (string, error) {
 
 // Close closes the SSH connection and destroys the client
 // Returns the success state and error. If error is not nil,
-// state will be false. Close does nothing and returns true
-// if no connection is open.
+// state will be false
 // @example
 // ```javascript
 // const ssh = require('nuclei/ssh');
@@ -169,17 +167,9 @@ func (c *SSHClient) Run(cmd string) (string, error) {
 // const closed = client.Close();
 // ```
 func (c *SSHClient) Close() (bool, error) {
-	if c.connection == nil {
-		return true, nil
-	}
-
-	err := c.connection.Close()
-	c.connection = nil
-
-	if err != nil {
+	if err := c.connection.Close(); err != nil {
 		return false, err
 	}
-
 	return true, nil
 }
 

pkg/js/libs/ssh/ssh_test.go

@@ -1,162 +0,0 @@
-package ssh_test
-
-//nolint:depguard // No depguard rules are configured, so its default list allows only the standard library.
-import (
-	"context"
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
-	"errors"
-	"net"
-	"strings"
-	"testing"
-
-	libssh "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/ssh"
-	"github.com/zmap/zgrab2/lib/ssh"
-)
-
-const (
-	testUser     = "user"
-	testPassword = "secret"
-)
-
-var errAccessDenied = errors.New("access denied")
-
-func TestCloseWithoutConnection(t *testing.T) {
-	t.Parallel()
-
-	host, port := startServer(t)
-
-	tests := map[string]func(t *testing.T, client *libssh.SSHClient){
-		"never connected": func(*testing.T, *libssh.SSHClient) {},
-		"failed connect": func(t *testing.T, client *libssh.SSHClient) {
-			t.Helper()
-
-			_, err := client.Connect(executionContext(t), host, port, testUser, "wrong")
-			if err == nil {
-				t.Fatal("Connect() with a wrong password succeeded")
-			}
-		},
-		"already closed": func(t *testing.T, client *libssh.SSHClient) {
-			t.Helper()
-
-			connect(t, client, host, port)
-			closeClient(t, client)
-		},
-	}
-	for name, setup := range tests {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			client := new(libssh.SSHClient)
-			setup(t, client)
-			closeClient(t, client)
-
-			_, err := client.Run("id")
-			if err == nil || !strings.Contains(err.Error(), "no connection") {
-				t.Errorf("Run() error = %v, want no connection", err)
-			}
-		})
-	}
-}
-
-func TestCloseConnected(t *testing.T) {
-	t.Parallel()
-
-	host, port := startServer(t)
-	client := new(libssh.SSHClient)
-	connect(t, client, host, port)
-	closeClient(t, client)
-}
-
-func connect(t *testing.T, client *libssh.SSHClient, host string, port int) {
-	t.Helper()
-
-	_, err := client.Connect(executionContext(t), host, port, testUser, testPassword)
-	if err != nil {
-		t.Fatalf("Connect() error = %v", err)
-	}
-}
-
-func closeClient(t *testing.T, client *libssh.SSHClient) {
-	t.Helper()
-
-	closed, err := client.Close()
-	if err != nil || !closed {
-		t.Fatalf("Close() = %v, %v; want true, nil", closed, err)
-	}
-}
-
-func executionContext(t *testing.T) context.Context {
-	t.Helper()
-
-	return context.WithValue(t.Context(), "executionId", t.Name()) //nolint:staticcheck // Connect reads this string key.
-}
-
-// startServer serves SSH on a loopback port and accepts only
-// testUser with testPassword. It rejects every channel.
-func startServer(t *testing.T) (string, int) {
-	t.Helper()
-
-	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	signer, err := ssh.NewSignerFromKey(key)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	config := new(ssh.ServerConfig)
-	config.PasswordCallback = checkPassword
-	config.AddHostKey(signer)
-
-	listener, err := new(net.ListenConfig).Listen(t.Context(), "tcp", "127.0.0.1:0")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() { _ = listener.Close() })
-
-	go func() {
-		for {
-			conn, err := listener.Accept()
-			if err != nil {
-				return
-			}
-
-			go serve(conn, config)
-		}
-	}()
-
-	addr, ok := listener.Addr().(*net.TCPAddr)
-	if !ok {
-		t.Fatalf("listener address %v is not TCP", listener.Addr())
-	}
-
-	return addr.IP.String(), addr.Port
-}
-
-func checkPassword(conn ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) {
-	if conn.User() != testUser || string(password) != testPassword {
-		return nil, errAccessDenied
-	}
-
-	return new(ssh.Permissions), nil
-}
-
-func serve(conn net.Conn, config *ssh.ServerConfig) {
-	defer func() { _ = conn.Close() }()
-
-	_, chans, reqs, err := ssh.NewServerConn(conn, config)
-	if err != nil {
-		return
-	}
-
-	go ssh.DiscardRequests(reqs)
-
-	for ch := range chans {
-		_ = ch.Reject(ssh.Prohibited, "test server")
-	}
-}

pkg/output/format_json.go

@@ -1,10 +1,6 @@
 package output
 
-import (
-	"fmt"
-
-	json "github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
-)
+import json "github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
 
 // formatJSON formats the output for json based formatting
 func (w *StandardWriter) formatJSON(output *ResultEvent) ([]byte, error) {
@@ -12,50 +8,5 @@ func (w *StandardWriter) formatJSON(output *ResultEvent) ([]byte, error) {
 		output.Request = ""
 		output.Response = ""
 	}
-	if len(output.Info.Metadata) > 0 {
-		output.Info.Metadata = sanitizeMetadataForJSON(output.Info.Metadata)
-	}
-	if len(output.Metadata) > 0 {
-		output.Metadata = sanitizeMetadataForJSON(output.Metadata)
-	}
 	return json.Marshal(output)
 }
-
-// sanitizeMetadataForJSON normalizes any map[interface{}]interface{} or nested
-// collections in metadata to map[string]interface{}, ensuring JSON encoders
-// (such as sonic or standard encoding/json) can serialize them without error (#7971).
-func sanitizeMetadataForJSON(m map[string]interface{}) map[string]interface{} {
-	if len(m) == 0 {
-		return m
-	}
-	sanitized := sanitizeJSONValue(m)
-	if res, ok := sanitized.(map[string]interface{}); ok {
-		return res
-	}
-	return m
-}
-
-func sanitizeJSONValue(v interface{}) interface{} {
-	switch typed := v.(type) {
-	case map[interface{}]interface{}:
-		converted := make(map[string]interface{}, len(typed))
-		for k, val := range typed {
-			converted[fmt.Sprint(k)] = sanitizeJSONValue(val)
-		}
-		return converted
-	case map[string]interface{}:
-		converted := make(map[string]interface{}, len(typed))
-		for k, val := range typed {
-			converted[k] = sanitizeJSONValue(val)
-		}
-		return converted
-	case []interface{}:
-		converted := make([]interface{}, len(typed))
-		for i, val := range typed {
-			converted[i] = sanitizeJSONValue(val)
-		}
-		return converted
-	default:
-		return v
-	}
-}

pkg/output/output_test.go

@@ -209,33 +209,3 @@ type testWriteCloser struct {
 func (w testWriteCloser) Close() error {
 	return nil
 }
-
-
-func TestFormatJSONSanitizesNestedMetadataMaps(t *testing.T) {
-	w, err := NewStandardWriter(&types.Options{})
-	require.NoError(t, err)
-
-	event := &ResultEvent{
-		TemplateID: "nested-metadata",
-		Type:       "file",
-		Host:       "target",
-		Matched:    "target/input.txt",
-		Info: model.Info{
-			Name: "Nested metadata",
-			Metadata: map[string]interface{}{
-				"labels": map[interface{}]interface{}{
-					"en": "Hello",
-					"nested_list": []interface{}{
-						map[interface{}]interface{}{
-							"inner_key": "inner_val",
-						},
-					},
-				},
-			},
-		},
-	}
-
-	data, err := w.formatJSON(event)
-	require.NoError(t, err, "formatJSON should not fail on nested map[interface{}]interface{} (#7971)")
-	require.Contains(t, string(data), `"labels":{"en":"Hello","nested_list":[{"inner_key":"inner_val"}]}`)
-}

pkg/protocols/code/code.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/alecthomas/chroma/v2/quick"
+	"github.com/alecthomas/chroma/quick"
 	"github.com/ditashi/jsbeautifier-go/jsbeautifier"
 	"github.com/projectdiscovery/goja"
 

pkg/protocols/common/expressions/expressions.go

@@ -2,7 +2,6 @@ package expressions
 
 import (
 	"fmt"
-	"slices"
 	"strings"
 
 	"github.com/projectdiscovery/govaluate"
@@ -49,75 +48,55 @@ func EvaluateWithOptions(data string, base map[string]interface{}, options *type
 	return evaluate(data, base, options)
 }
 
-// evaluate renders data, the template text, with base, the data values.
-//
-// Expressions are located in data before anything is substituted, and each
-// result is written at its expression's position. The text between
-// expressions only has its placeholders replaced. Nothing that evaluate
-// writes is scanned again, so a value or a helper result that looks like an
-// expression stays literal and never takes another expression's result. On
-// error, data is returned unchanged.
-//
-// Expressions can be simple (base keys) or complex (helper functions and base
-// keys). Literals such as {{2+2}} are not expressions.
 func evaluate(data string, base map[string]interface{}, options *types.Options) (string, error) {
-	spans := findExpressionSpans(data, marker.ParenthesisOpen, marker.ParenthesisClose, base)
-	if len(spans) == 0 {
-		return replacer.Replace(data, base), nil
-	}
+	expressions := FindExpressions(data, marker.ParenthesisOpen, marker.ParenthesisClose, base)
 
-	var (
-		builder strings.Builder
-		last    int
-	)
+	// replace simple placeholders (key => value) MarkerOpen + key + MarkerClose and General + key + General to value
+	data = replacer.Replace(data, base)
 
-	builder.Grow(len(data))
+	// expressions can be:
+	// - simple: containing base values keys (variables)
+	// - complex: containing helper functions [ + variables]
+	// literals like {{2+2}} are not considered expressions
+	for _, expression := range expressions {
+		originalExpression := expression
+		// data has already had simple placeholders replaced; keep the same
+		// marker shape for output replacement, but never compile this string.
+		replacedExpression := replacer.Replace(expression, base)
+		expression = replaceStringPlaceholders(expression, base)
 
-	for _, span := range spans {
-		result, err := evaluateExpression(span.expression, base, options)
+		// turns expressions (either helper functions+base values or base values)
+		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions)
 		if err != nil {
-			return data, err
+			return data, fmt.Errorf("failed to compile expression %q: %w", originalExpression, err)
 		}
 
-		builder.WriteString(replacer.Replace(data[last:span.start], base))
-		builder.WriteString(result)
-
-		last = span.end
-	}
-
-	builder.WriteString(replacer.Replace(data[last:], base))
-
-	return builder.String(), nil
-}
-
-// evaluateExpression compiles and evaluates one template-authored expression,
-// the text between its markers, and returns the result as text.
-func evaluateExpression(expression string, base map[string]any, options *types.Options) (string, error) {
-	source := replaceStringPlaceholders(expression, base)
-
-	compiled, err := govaluate.NewEvaluableExpressionWithFunctions(source, dsl.HelperFunctions)
-	if err != nil {
-		return "", fmt.Errorf("failed to compile expression %q: %w", expression, err)
-	}
-
-	result, err := dsl.EvalWithOptions(compiled, base, options)
-	if err != nil {
-		return "", fmt.Errorf("failed to evaluate expression %q: %w", expression, err)
-	}
-
-	// Preserve unresolved markers only when a helper call would otherwise
-	// hide them from downstream validation. Plain expressions such as
-	// comparisons should evaluate normally.
-	if markers := unresolvedVarMarkers(compiled.Vars(), base); markers != "" {
-		usesFunctions := slices.ContainsFunc(compiled.Tokens(), func(token govaluate.ExpressionToken) bool {
-			return token.Kind == govaluate.FUNCTION
-		})
-		if usesFunctions && ContainsUnresolvedVariables(fmt.Sprint(result)) == nil {
-			return markers, nil
+		result, err := dsl.EvalWithOptions(compiled, base, options)
+		if err != nil {
+			return data, fmt.Errorf("failed to evaluate expression %q: %w", originalExpression, err)
 		}
-	}
 
-	return types.ToString(result), nil
+		replacement := result
+		// Preserve unresolved markers only when a helper call would otherwise
+		// hide them from downstream validation. Plain expressions such as
+		// comparisons should evaluate normally.
+		if markers := unresolvedVarMarkers(compiled.Vars(), base); markers != "" {
+			usesFunctions := false
+			for _, token := range compiled.Tokens() {
+				if token.Kind == govaluate.FUNCTION {
+					usesFunctions = true
+					break
+				}
+			}
+			if usesFunctions && ContainsUnresolvedVariables(fmt.Sprint(result)) == nil {
+				replacement = markers
+			}
+		}
+
+		// replace incrementally
+		data = replacer.ReplaceOne(data, replacedExpression, replacement)
+	}
+	return data, nil
 }
 
 func replaceStringPlaceholders(expression string, base map[string]interface{}) string {
@@ -227,63 +206,59 @@ func EscapeStringValue(value string, quote byte) string {
 // maxIterations to avoid infinite loop
 const maxIterations = 250
 
-// FindExpressions returns the expressions in data, the text between each
-// matching OpenMarker and CloseMarker pair, in order.
 func FindExpressions(data, OpenMarker, CloseMarker string, base map[string]interface{}) []string {
-	spans := findExpressionSpans(data, OpenMarker, CloseMarker, base)
-	if len(spans) == 0 {
-		return nil
-	}
+	var (
+		iterations int
+		exps       []string
+	)
+	for iterations <= maxIterations {
+		// check if we reached the maximum number of iterations
 
-	exps := make([]string, 0, len(spans))
-	for _, span := range spans {
-		exps = append(exps, span.expression)
-	}
-
-	return exps
-}
-
-// expressionSpan is one expression in template text: data[start:end] is the
-// whole marker and expression is the text between its markers.
-type expressionSpan struct {
-	start, end int
-	expression string
-}
-
-// findExpressionSpans finds the expressions in data and where each one sits.
-// For each open marker, it takes the nearest close marker that makes the text
-// between them an expression. When none does, the search resumes right after
-// the open marker.
-func findExpressionSpans(data, openMarker, closeMarker string, base map[string]any) []expressionSpan {
-	var spans []expressionSpan
-
-	for offset, iterations := 0, 0; iterations <= maxIterations; iterations++ {
-		openIndex := strings.Index(data[offset:], openMarker)
-		if openIndex < 0 {
+		iterations++
+		// attempt to find open markers
+		indexOpenMarker := strings.Index(data, OpenMarker)
+		// exits if not found
+		if indexOpenMarker < 0 {
 			break
 		}
 
-		start := offset + openIndex
-		inner := start + len(openMarker)
-		offset = inner
+		indexOpenMarkerOffset := indexOpenMarker + len(OpenMarker)
 
-		for search := inner; ; {
-			closeIndex := stringsutil.IndexAt(data, closeMarker, search)
-			if closeIndex < 0 {
-				break
+		shouldSearchCloseMarker := true
+		closeMarkerFound := false
+		innerData := data
+		var potentialMatch string
+		var indexCloseMarker, indexCloseMarkerOffset int
+		skip := indexOpenMarkerOffset
+		for shouldSearchCloseMarker {
+			// attempt to find close marker
+			indexCloseMarker = stringsutil.IndexAt(innerData, CloseMarker, skip)
+			// if no close markers are found exit
+			if indexCloseMarker < 0 {
+				shouldSearchCloseMarker = false
+				continue
 			}
+			indexCloseMarkerOffset = indexCloseMarker + len(CloseMarker)
 
-			search = closeIndex + len(closeMarker)
-			if expression := data[inner:closeIndex]; isExpression(expression, base) {
-				spans = append(spans, expressionSpan{start: start, end: search, expression: expression})
-				offset = search
-
-				break
+			potentialMatch = innerData[indexOpenMarkerOffset:indexCloseMarker]
+			if isExpression(potentialMatch, base) {
+				closeMarkerFound = true
+				shouldSearchCloseMarker = false
+				exps = append(exps, potentialMatch)
+			} else {
+				skip = indexCloseMarkerOffset
 			}
 		}
-	}
 
-	return spans
+		if closeMarkerFound {
+			// move after the close marker
+			data = data[indexCloseMarkerOffset:]
+		} else {
+			// move after the open marker
+			data = data[indexOpenMarkerOffset:]
+		}
+	}
+	return exps
 }
 
 func isExpression(data string, base map[string]interface{}) bool {

pkg/protocols/common/expressions/expressions_test.go

@@ -109,71 +109,6 @@ func TestEvaluateDoesNotReinterpretResolvedValues(t *testing.T) {
 	}
 }
 
-func TestEvaluateReplacesExpressionsInTheirOwnPosition(t *testing.T) {
-	t.Parallel()
-
-	// tpl and echo stand for response values that repeat an expression's text.
-	values := map[string]any{
-		"seed": "hello",
-		"tpl":  "{{to_upper(seed)}}",
-		"echo": "{{base64('127.0.0.1')}}",
-		"Host": "127.0.0.1",
-	}
-
-	items := []struct {
-		name     string
-		input    string
-		expected string
-	}{
-		{
-			name:     "resolved value echoes a later expression",
-			input:    "[{{tpl}}] {{to_upper(seed)}}",
-			expected: "[{{to_upper(seed)}}] HELLO",
-		},
-		{
-			name:     "general marker value echoes a later expression",
-			input:    "[§tpl§] {{to_upper(seed)}}",
-			expected: "[{{to_upper(seed)}}] HELLO",
-		},
-		{
-			name:     "resolved value echoes an expression with substituted arguments",
-			input:    "{{echo}} {{base64('{{Host}}')}}",
-			expected: "{{base64('127.0.0.1')}} MTI3LjAuMC4x",
-		},
-		{
-			name:     "helper result echoes a later expression",
-			input:    "{{concat('{{', 'to_upper(seed)', '}}')}} {{to_upper(seed)}}",
-			expected: "{{to_upper(seed)}} HELLO",
-		},
-		{
-			name:     "repeated expression is evaluated in each position",
-			input:    "{{to_upper(seed)}}-{{to_upper(seed)}}",
-			expected: "HELLO-HELLO",
-		},
-	}
-
-	for _, item := range items {
-		t.Run(item.name, func(t *testing.T) {
-			t.Parallel()
-
-			value, err := Evaluate(item.input, values)
-			require.NoError(t, err)
-			require.Equal(t, item.expected, value)
-		})
-	}
-}
-
-func TestEvaluateDoesNotResolveGeneralMarkersFromResolvedValues(t *testing.T) {
-	t.Parallel()
-
-	value, err := Evaluate("Authorization: {{body}}", map[string]any{
-		"body":         "§secret_token§",
-		"secret_token": "top-secret-cia-mi6-kgb-mossad-classified",
-	})
-	require.NoError(t, err)
-	require.Equal(t, "Authorization: §secret_token§", value)
-}
-
 func TestEvaluateDoesNotExecuteHelpersFromResolvedValues(t *testing.T) {
 	var calls int
 
@@ -271,18 +206,6 @@ func TestEvaluateReturnsErrorForInvalidTemplateExpression(t *testing.T) {
 	require.ErrorContains(t, err, `failed to evaluate expression "base64()"`)
 }
 
-func TestEvaluateReturnsInputOnError(t *testing.T) {
-	t.Parallel()
-
-	input := "{{secret_token}} {{base64()}}"
-
-	value, err := Evaluate(input, map[string]any{
-		"secret_token": "top-secret-cia-mi6-kgb-mossad-classified",
-	})
-	require.Error(t, err)
-	require.Equal(t, input, value)
-}
-
 func TestEvaluateErrorDoesNotLeakResolvedValues(t *testing.T) {
 	_, err := Evaluate("{{base64('{{secret_token}}', 'extra')}}", map[string]interface{}{
 		"secret_token": "top-secret-cia-mi6-kgb-mossad-classified",

pkg/protocols/common/expressions/variables.go

@@ -115,26 +115,6 @@ func ContainsVariablesWithIgnoreList(skipNames map[string]interface{}, items ...
 	return nil
 }
 
-// ReferencesVariables reports whether data contains a {{...}} marker that
-// refers to a variable in names, either alone or inside an expression. It
-// parses the marker but does not evaluate it.
-func ReferencesVariables(data string, names map[string]interface{}) bool {
-	for _, match := range unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) {
-		expr, err := govaluate.NewEvaluableExpressionWithFunctions(match[1], dsl.HelperFunctions)
-		if err != nil {
-			continue
-		}
-
-		for _, name := range expr.Vars() {
-			if _, ok := names[name]; ok {
-				return true
-			}
-		}
-	}
-
-	return false
-}
-
 func hasLiteralsOnly(data string) bool {
 	expr, err := govaluate.NewEvaluableExpressionWithFunctions(data, dsl.HelperFunctions)
 	if err != nil {

pkg/protocols/common/expressions/variables_test.go

@@ -44,26 +44,6 @@ func TestUnresolvedVariablesCheck(t *testing.T) {
 	}
 }
 
-func TestReferencesVariables(t *testing.T) {
-	names := map[string]interface{}{"Hostname": "", "Port": ""}
-
-	tests := []struct {
-		data string
-		want bool
-	}{
-		{data: "user@{{Hostname}}", want: true},
-		{data: "{{to_lower(Hostname)}}:{{Port}}", want: true},
-		{data: "{{Hostname_suffix}}", want: false},
-		{data: "{{secret}}-{{to_upper(seed)}}", want: false},
-		{data: "{{7*7}}", want: false},
-		{data: "Hostname", want: false},
-	}
-
-	for _, test := range tests {
-		require.Equal(t, test.want, ReferencesVariables(test.data, names), test.data)
-	}
-}
-
 func TestUnresolvedVariablesCheckDoesNotExecuteHelpers(t *testing.T) {
 	var calls int
 	withTestHelperFunction(t, "test_side_effect", func(args ...interface{}) (interface{}, error) {

pkg/protocols/common/protocolinit/init.go

@@ -35,7 +35,5 @@ func Init(options *types.Options) error {
 }
 
 func Close(executionId string) {
-	dnsclientpool.Close(executionId)
-	rdapclientpool.Close(executionId)
 	protocolstate.Close(executionId)
 }

pkg/protocols/common/replacer/replacer.go

@@ -3,186 +3,27 @@ package replacer
 import (
 	"strings"
 
+	"github.com/projectdiscovery/fasttemplate"
+	convUtil "github.com/projectdiscovery/utils/conversion"
+
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/marker"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 )
 
-// Replace substitutes each {{key}} and §key§ placeholder in template whose key
-// is in values, scanning template once from left to right.
-//
-// Values are data: Replace never scans an inserted value for placeholders, so
-// a value cannot pull in another value, alone or together with the template
-// text around it. Text that does not form a known placeholder is kept as is.
+// Replace replaces placeholders in template with values on the fly.
 func Replace(template string, values map[string]interface{}) string {
-	if len(values) == 0 {
-		return template
+	valuesMap := make(map[string]interface{}, len(values))
+	for k, v := range values {
+		// fasttemplate copies string values to []byte on every write, so
+		// pass a read-only []byte view instead
+		valuesMap[k] = convUtil.Bytes(types.ToString(v))
 	}
-
-	scan := newScanner(template, values)
-
-	start := scan.nextPlaceholder(0)
-	if start < 0 {
-		return template
-	}
-
-	var builder strings.Builder
-
-	builder.Grow(len(template))
-
-	written := 0 // template[:written] is already in builder
-
-	for pos := start; pos >= 0; {
-		value, end, ok := scan.resolve(pos)
-		if !ok {
-			// Not a known placeholder: keep it as text and look again one
-			// byte later, so a placeholder nested in it still resolves.
-			pos = scan.nextPlaceholder(pos + 1)
-
-			continue
-		}
-
-		builder.WriteString(template[written:pos])
-		builder.WriteString(value)
-
-		written = end
-		pos = scan.nextPlaceholder(end)
-	}
-
-	builder.WriteString(template[written:])
-
-	return builder.String()
+	replaced := fasttemplate.ExecuteStringStd(template, marker.ParenthesisOpen, marker.ParenthesisClose, valuesMap)
+	final := fasttemplate.ExecuteStringStd(replaced, marker.General, marker.General, valuesMap)
+	return final
 }
 
-// directLookupLimit is the longest candidate key that resolve looks up in
-// values without first comparing it to the longest key there.
-const directLookupLimit = 32
-
-// scanner finds the placeholders of one template. Replace asks for positions
-// that never decrease, so each finder searches every byte at most once. A
-// candidate key longer than every key in values is rejected without hashing
-// it. Together, these keep Replace linear in the template length.
-type scanner struct {
-	template                  string
-	values                    map[string]any
-	longestKey                int // length of the longest key in values; -1 until needed
-	braceOpen, braceClose     finder
-	generalOpen, generalClose finder
-}
-
-func newScanner(template string, values map[string]any) scanner {
-	return scanner{
-		template:     template,
-		values:       values,
-		longestKey:   -1,
-		braceOpen:    newFinder(marker.ParenthesisOpen),
-		braceClose:   newFinder(marker.ParenthesisClose),
-		generalOpen:  newFinder(marker.General),
-		generalClose: newFinder(marker.General),
-	}
-}
-
-// nextPlaceholder returns the index of the next {{ or § at or after from that
-// can still be closed, or -1 when there is none. Once no closing marker of a
-// kind is left, no later opening marker of that kind can close.
-func (s *scanner) nextPlaceholder(from int) int {
-	brace, general := -1, -1
-
-	if !s.braceClose.exhausted() {
-		brace = s.braceOpen.index(s.template, from)
-	}
-
-	if !s.generalClose.exhausted() {
-		general = s.generalOpen.index(s.template, from)
-	}
-
-	if brace < 0 || (general >= 0 && general < brace) {
-		return general
-	}
-
-	return brace
-}
-
-// resolve returns the value of the placeholder that starts at index start,
-// and the index just past it, when its key is in values.
-func (s *scanner) resolve(start int) (string, int, bool) {
-	open, closing := marker.ParenthesisOpen, &s.braceClose
-	if strings.HasPrefix(s.template[start:], marker.General) {
-		open, closing = marker.General, &s.generalClose
-	}
-
-	keyStart := start + len(open)
-
-	keyEnd := closing.index(s.template, keyStart)
-	if keyEnd < 0 {
-		return "", 0, false
-	}
-
-	key := s.template[keyStart:keyEnd]
-	if len(key) > directLookupLimit && len(key) > s.longestKeyLen() {
-		return "", 0, false
-	}
-
-	value, ok := s.values[key]
-	if !ok {
-		return "", 0, false
-	}
-
-	return types.ToString(value), keyEnd + len(closing.needle), true
-}
-
-// longestKeyLen returns the length of the longest key in values.
-func (s *scanner) longestKeyLen() int {
-	if s.longestKey < 0 {
-		s.longestKey = 0
-		for key := range s.values {
-			s.longestKey = max(s.longestKey, len(key))
-		}
-	}
-
-	return s.longestKey
-}
-
-// finder finds one marker in a template. Its callers pass positions that
-// never decrease, so it keeps the last match until the scan passes it, and a
-// failed search answers every later call.
-type finder struct {
-	needle   string
-	next     int // last match, or -1 when none is left
-	searched bool
-}
-
-func newFinder(needle string) finder {
-	return finder{needle: needle, next: -1, searched: false}
-}
-
-// index returns the index of the first needle at or after from, or -1.
-func (f *finder) index(template string, from int) int {
-	if f.searched && (f.next < 0 || f.next >= from) {
-		return f.next
-	}
-
-	f.searched = true
-
-	f.next = strings.Index(template[from:], f.needle)
-	if f.next >= 0 {
-		f.next += from
-	}
-
-	return f.next
-}
-
-// exhausted reports whether a search found no match left in the template.
-func (f *finder) exhausted() bool {
-	return f.searched && f.next < 0
-}
-
-// ReplaceOne replaces the first {{key}} and the first §key§ in template with
-// value.
-//
-// Deprecated: ReplaceOne finds the placeholder by searching template, so when
-// template already holds substituted values it can replace text that a value
-// introduced instead of the placeholder the caller meant. Render by position
-// with Replace instead.
+// Replace replaces one placeholder in template with one value on the fly.
 func ReplaceOne(template string, key string, value interface{}) string {
 	data := replaceOneWithMarkers(template, key, value, marker.ParenthesisOpen, marker.ParenthesisClose)
 	return replaceOneWithMarkers(data, key, value, marker.General, marker.General)

pkg/protocols/common/replacer/replacer_bench_test.go

@@ -1,39 +0,0 @@
-package replacer_test
-
-import (
-	"strconv"
-	"strings"
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
-)
-
-func BenchmarkReplace(b *testing.B) {
-	// An HTTP request renders with about 60 values, enough that map lookups
-	// hash their keys.
-	values := map[string]any{"Hostname": "example.com", "token": "abc"}
-	for i := range 60 {
-		values["var"+strconv.Itoa(i)] = i
-	}
-
-	header := "POST /?t={{token}} HTTP/1.1\r\nHost: {{Hostname}}\r\n\r\n"
-
-	for _, item := range []struct {
-		name     string
-		template string
-	}{
-		{name: "raw request", template: header},
-		{name: "64KB JSON body", template: header + strings.Repeat(`{"k": "v", "n": [1, 2]} `, 2730)},
-		{name: "64KB non-ASCII body", template: header + strings.Repeat("Привет, мир! ", 2730)},
-		{name: "64KB unclosed markers", template: strings.Repeat(`{{`, 32768)},
-		{name: "64KB markers closed at the end", template: strings.Repeat(`{`, 65536) + `}}`},
-	} {
-		b.Run(item.name, func(b *testing.B) {
-			b.ReportAllocs()
-
-			for b.Loop() {
-				replacer.Replace(item.template, values)
-			}
-		})
-	}
-}

pkg/protocols/common/replacer/replacer_test.go

@@ -73,24 +73,6 @@ func TestReplacerReplace(t *testing.T) {
 			values:   map[string]interface{}{"var1": "first variable", "var2": "second variable"},
 			expected: "first variable and first variable and another second variable",
 		},
-		{
-			name:     "Extra braces around a placeholder",
-			template: "${{{num}}*{{{num}}",
-			values:   map[string]any{"num": "7"},
-			expected: "${7*{7",
-		},
-		{
-			name:     "Unknown general marker before a known one",
-			template: "§unknown§var1§",
-			values:   map[string]any{"var1": "variable 1"},
-			expected: "§unknownvariable 1",
-		},
-		{
-			name:     "Unclosed markers",
-			template: "{{{{ §§ {{var1}",
-			values:   map[string]any{"var1": "variable 1"},
-			expected: "{{{{ §§ {{var1}",
-		},
 	}
 
 	for _, test := range tests {
@@ -100,48 +82,6 @@ func TestReplacerReplace(t *testing.T) {
 	}
 }
 
-func TestReplacerReplaceDoesNotRescanValues(t *testing.T) {
-	t.Parallel()
-
-	values := map[string]any{
-		"secret":      "leaked",
-		"token":       "leaked",
-		"general":     "§secret§",
-		"parenthesis": "{{secret}}",
-		"prefix":      "§",
-	}
-
-	tests := []struct {
-		name     string
-		template string
-		expected string
-	}{
-		{
-			name:     "general marker inside a parenthesis value",
-			template: "{{general}}",
-			expected: "§secret§",
-		},
-		{
-			name:     "parenthesis marker inside a general value",
-			template: "§parenthesis§",
-			expected: "{{secret}}",
-		},
-		{
-			name:     "value completes a marker with the template text after it",
-			template: "{{prefix}}token§",
-			expected: "§token§",
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			t.Parallel()
-
-			require.Equal(t, test.expected, Replace(test.template, values))
-		})
-	}
-}
-
 func TestReplacerReplaceOne(t *testing.T) {
 	tests := []struct {
 		name     string

pkg/protocols/dns/dns.go

@@ -15,6 +15,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool"
+	"github.com/projectdiscovery/retryabledns"
 	fileutil "github.com/projectdiscovery/utils/file"
 )
 
@@ -86,7 +87,7 @@ type Request struct {
 	generator *generators.PayloadGenerator
 
 	CompiledOperators *operators.Operators `yaml:"-" json:"-"`
-	dnsClient         *dnsclientpool.Client
+	dnsClient         *retryabledns.Client
 	options           *protocols.ExecutorOptions
 
 	// cache any variables that may be needed for operation.
@@ -204,7 +205,7 @@ func resolverHost(resolver string) string {
 	return r
 }
 
-func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadata map[string]interface{}) (*dnsclientpool.Client, error) {
+func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadata map[string]interface{}) (*retryabledns.Client, error) {
 	dnsClientOptions := &dnsclientpool.Configuration{
 		Retries: request.Retries,
 		Proxy:   options.Options.AliveSocksProxy,

pkg/protocols/dns/dnsclientpool/clientpool.go

@@ -12,9 +12,9 @@ import (
 )
 
 var (
-	clientPool *mapsutil.SyncLockMap[string, *Client]
+	clientPool *mapsutil.SyncLockMap[string, *retryabledns.Client]
 
-	normalClient *Client
+	normalClient *retryabledns.Client
 	m            sync.Mutex
 )
 
@@ -46,21 +46,20 @@ func resolversFor(options *types.Options) []string {
 	return defaultResolvers
 }
 
-func cachedClient(resolvers []string) (*Client, error) {
+func cachedClient(resolvers []string) (*retryabledns.Client, error) {
 	m.Lock()
 	defer m.Unlock()
 	if clientPool == nil {
-		clientPool = mapsutil.NewSyncLockMap[string, *Client]()
+		clientPool = mapsutil.NewSyncLockMap[string, *retryabledns.Client]()
 	}
 	key := strings.Join(resolvers, ",")
 	if client, ok := clientPool.Get(key); ok {
 		return client, nil
 	}
-	dnsClient, err := retryabledns.New(resolvers, 1)
+	client, err := retryabledns.New(resolvers, 1)
 	if err != nil {
 		return nil, err
 	}
-	client := &Client{Client: dnsClient}
 	_ = clientPool.Set(key, client)
 	return client, nil
 }
@@ -89,13 +88,10 @@ func (c *Configuration) Hash() string {
 }
 
 // Get creates or gets a client for the protocol based on custom configuration
-func Get(options *types.Options, configuration *Configuration) (*Client, error) {
+func Get(options *types.Options, configuration *Configuration) (*retryabledns.Client, error) {
 	if (configuration.Retries <= 1) && len(configuration.Resolvers) == 0 {
 		return cachedClient(resolversFor(options))
 	}
-	if len(options.InternalResolversList) == 0 && hasDohResolver(configuration.Resolvers) {
-		return getPolicyClient(options, configuration)
-	}
 	hash := configuration.Hash()
 	if client, ok := clientPool.Get(hash); ok {
 		return client, nil
@@ -107,7 +103,7 @@ func Get(options *types.Options, configuration *Configuration) (*Client, error)
 	} else if len(configuration.Resolvers) > 0 {
 		resolvers = configuration.Resolvers
 	}
-	dnsClient, err := retryabledns.NewWithOptions(retryabledns.Options{
+	client, err := retryabledns.NewWithOptions(retryabledns.Options{
 		BaseResolvers: resolvers,
 		MaxRetries:    configuration.Retries,
 		Proxy:         options.AliveSocksProxy,
@@ -115,7 +111,6 @@ func Get(options *types.Options, configuration *Configuration) (*Client, error)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create dns client")
 	}
-	client := &Client{Client: dnsClient}
 	_ = clientPool.Set(hash, client)
 
 	return client, nil

pkg/protocols/dns/dnsclientpool/doh.go

@@ -1,171 +0,0 @@
-package dnsclientpool
-
-import (
-	"context"
-	"crypto/tls"
-	"fmt"
-	"net"
-	"net/http"
-	"strings"
-	"sync"
-	"sync/atomic"
-
-	"github.com/miekg/dns"
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/projectdiscovery/retryabledns"
-	"github.com/projectdiscovery/retryabledns/doh"
-)
-
-// Client sends the DNS queries of a template. It sends queries for template
-// DoH resolvers itself, because the retryabledns DoH client dials and follows
-// redirects without the network policy of the scan.
-type Client struct {
-	*retryabledns.Client
-
-	resolvers []resolver
-	retries   int
-	index     uint32
-	doh       *doh.Client
-	transport *http.Transport
-}
-
-// resolver is one template resolver. url is empty for a resolver that
-// retryabledns queries over UDP, TCP or DoT.
-type resolver struct {
-	url    string
-	method doh.Method
-}
-
-var (
-	// policyClients holds the clients with DoH resolvers of each execution,
-	// because each one dials with the dialer of its execution.
-	policyClients = map[string]map[string]*Client{}
-	policyMu      sync.Mutex
-)
-
-func hasDohResolver(resolvers []string) bool {
-	for _, r := range resolvers {
-		if strings.HasPrefix(r, "doh:") {
-			return true
-		}
-	}
-	return false
-}
-
-// parseDohResolver returns the URL and method of a resolver such as
-// doh:https://dns.example/dns-query:get, in the same way as retryabledns.
-func parseDohResolver(r string) (resolver, bool) {
-	if !strings.HasPrefix(r, "doh:") {
-		return resolver{}, false
-	}
-	u := strings.TrimPrefix(r, "doh:")
-	method := doh.MethodPost
-	for _, suffix := range []string{":get", ":post", ":jsonapi"} {
-		if strings.HasSuffix(u, suffix) {
-			u = strings.TrimSuffix(u, suffix)
-			if suffix == ":get" {
-				method = doh.MethodGet
-			}
-			break
-		}
-	}
-	return resolver{url: u, method: method}, true
-}
-
-func getPolicyClient(options *types.Options, configuration *Configuration) (*Client, error) {
-	executionID := options.ExecutionId
-	hash := configuration.Hash()
-
-	policyMu.Lock()
-	defer policyMu.Unlock()
-	if client, ok := policyClients[executionID][hash]; ok {
-		return client, nil
-	}
-
-	client := &Client{retries: configuration.Retries}
-	var networkResolvers []string
-	for _, r := range configuration.Resolvers {
-		if parsed, ok := parseDohResolver(r); ok {
-			client.resolvers = append(client.resolvers, parsed)
-			continue
-		}
-		client.resolvers = append(client.resolvers, resolver{})
-		networkResolvers = append(networkResolvers, r)
-	}
-	// Trace and the non DoH resolvers still go through retryabledns.
-	if len(networkResolvers) == 0 {
-		networkResolvers = defaultResolvers
-	}
-	dnsClient, err := retryabledns.NewWithOptions(retryabledns.Options{
-		BaseResolvers: networkResolvers,
-		MaxRetries:    1,
-		Proxy:         options.AliveSocksProxy,
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "could not create dns client")
-	}
-	client.Client = dnsClient
-
-	// The scan dialer checks every address it connects to, so redirects
-	// and names that resolve to a denied address are refused too. It also
-	// goes through the scan proxy when one is set.
-	client.transport = &http.Transport{
-		DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
-			dialers := protocolstate.GetDialersWithId(executionID)
-			if dialers == nil || dialers.Fastdialer == nil {
-				return nil, fmt.Errorf("dialers not initialized for %s", executionID)
-			}
-			return dialers.Fastdialer.Dial(ctx, network, address)
-		},
-		// same as the retryabledns DoH client
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec G402
-	}
-	client.doh = doh.NewWithOptions(doh.Options{
-		HttpClient: &http.Client{
-			Transport: client.transport,
-			Timeout:   options.GetTimeouts().HttpTimeout,
-		},
-	})
-
-	if policyClients[executionID] == nil {
-		policyClients[executionID] = map[string]*Client{}
-	}
-	policyClients[executionID][hash] = client
-	return client, nil
-}
-
-// Do sends a DNS message to the resolvers in turn until one answers with
-// NOERROR, like retryabledns.Client.Do.
-func (c *Client) Do(msg *dns.Msg) (*dns.Msg, error) {
-	if len(c.resolvers) == 0 {
-		return c.Client.Do(msg)
-	}
-	var resp *dns.Msg
-	var err error
-	for i := 0; i < c.retries; i++ {
-		index := atomic.AddUint32(&c.index, 1)
-		r := c.resolvers[index%uint32(len(c.resolvers))]
-		if r.url != "" {
-			resp, err = c.doh.QueryWithDOHMsg(r.method, doh.Resolver{URL: r.url}, msg)
-		} else {
-			resp, err = c.Client.Do(msg)
-		}
-		if err == nil && resp != nil && resp.Rcode == dns.RcodeSuccess {
-			return resp, nil
-		}
-	}
-	return resp, retryabledns.ErrRetriesExceeded
-}
-
-// Close drops the clients with DoH resolvers of an execution.
-func Close(executionID string) {
-	policyMu.Lock()
-	clients := policyClients[executionID]
-	delete(policyClients, executionID)
-	policyMu.Unlock()
-	for _, client := range clients {
-		client.transport.CloseIdleConnections()
-	}
-}

pkg/protocols/dns/dnsclientpool/doh_test.go

@@ -1,97 +0,0 @@
-package dnsclientpool
-
-import (
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"sync/atomic"
-	"testing"
-
-	"github.com/miekg/dns"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/stretchr/testify/require"
-)
-
-// newDohServer answers every DoH query with a TXT record and counts the hits.
-func newDohServer(t *testing.T, hits *atomic.Int32) *httptest.Server {
-	t.Helper()
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		hits.Add(1)
-		m := new(dns.Msg)
-		m.SetQuestion("example.com.", dns.TypeTXT)
-		m.Response = true
-		rr, err := dns.NewRR(`example.com. 60 IN TXT "doh-answer"`)
-		require.NoError(t, err)
-		m.Answer = append(m.Answer, rr)
-		packed, err := m.Pack()
-		require.NoError(t, err)
-		w.Header().Set("Content-Type", "application/dns-message")
-		_, _ = w.Write(packed)
-	}))
-	t.Cleanup(server.Close)
-	return server
-}
-
-func dohOptions(t *testing.T, excludeTargets []string) *types.Options {
-	t.Helper()
-	options := types.DefaultOptions()
-	options.ExecutionId = t.Name()
-	options.ExcludeTargets = excludeTargets
-	require.NoError(t, protocolstate.Init(options))
-	t.Cleanup(func() { protocolstate.Close(options.ExecutionId) })
-	require.NoError(t, Init(options))
-	return options
-}
-
-func txtQuery() *dns.Msg {
-	msg := new(dns.Msg)
-	msg.SetQuestion("example.com.", dns.TypeTXT)
-	return msg
-}
-
-func TestDohResolverObeysNetworkPolicy(t *testing.T) {
-	t.Run("denied address", func(t *testing.T) {
-		var hits atomic.Int32
-		internal := newDohServer(t, &hits)
-		options := dohOptions(t, []string{"127.0.0.1"})
-
-		client, err := Get(options, &Configuration{Retries: 1, Resolvers: []string{"doh:" + internal.URL + "/dns-query:get"}})
-		require.NoError(t, err)
-		_, _ = client.Do(txtQuery())
-		require.Zero(t, hits.Load(), "DoH query reached a denied address")
-	})
-
-	t.Run("redirect to denied address", func(t *testing.T) {
-		var hits atomic.Int32
-		internal := newDohServer(t, &hits)
-		redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			http.Redirect(w, r, internal.URL+"/internal", http.StatusFound)
-		}))
-		t.Cleanup(redirector.Close)
-		// localhost passes the resolver check through ::1, while 127.0.0.1 is denied
-		_, port, err := net.SplitHostPort(strings.TrimPrefix(redirector.URL, "http://"))
-		require.NoError(t, err)
-		options := dohOptions(t, []string{"127.0.0.1"})
-		require.True(t, protocolstate.IsHostAllowed(options.ExecutionId, "localhost"))
-
-		client, err := Get(options, &Configuration{Retries: 1, Resolvers: []string{"doh:http://localhost:" + port + "/dns-query:get"}})
-		require.NoError(t, err)
-		_, _ = client.Do(txtQuery())
-		require.Zero(t, hits.Load(), "DoH redirect reached a denied address")
-	})
-
-	t.Run("allowed address", func(t *testing.T) {
-		var hits atomic.Int32
-		server := newDohServer(t, &hits)
-		options := dohOptions(t, nil)
-
-		client, err := Get(options, &Configuration{Retries: 2, Resolvers: []string{"doh:" + server.URL + "/dns-query:get"}})
-		require.NoError(t, err)
-		resp, err := client.Do(txtQuery())
-		require.NoError(t, err)
-		require.Len(t, resp.Answer, 1)
-		require.Equal(t, int32(1), hits.Load())
-	})
-}

pkg/protocols/dns/request.go

@@ -195,7 +195,7 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad
 	// perform trace if necessary (excluded from duration — only the query RTT is measured)
 	var traceData *retryabledns.TraceData
 	if request.Trace {
-		traceData, err = trace(request.options.Options.ExecutionId, request.dnsClient, domain, request.question, request.TraceMaxRecursion)
+		traceData, err = request.dnsClient.Trace(domain, request.question, request.TraceMaxRecursion)
 		if err != nil {
 			request.options.Output.Request(request.options.TemplatePath, domain, "dns", err)
 		}

pkg/protocols/dns/trace.go

@@ -1,88 +0,0 @@
-package dns
-
-import (
-	"context"
-	"math/rand"
-	"net"
-
-	"github.com/miekg/dns"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool"
-	"github.com/projectdiscovery/retryabledns"
-	sliceutil "github.com/projectdiscovery/utils/slice"
-)
-
-// traceNameServerPort is the port of the name servers that a trace queries.
-var traceNameServerPort = "53"
-
-// trace follows the delegation of host from the root servers in the same way
-// as retryabledns.Client.Trace, but it skips name server addresses that the
-// network policy denies. The name servers come from the answers, so the owner
-// of a zone could otherwise send the queries of a trace to internal addresses.
-func trace(executionID string, client *dnsclientpool.Client, host string, requestType uint16, maxRecursion int) (*retryabledns.TraceData, error) {
-	var traceData retryabledns.TraceData
-	host = dns.CanonicalName(host)
-	servers := retryabledns.RootDNSServersIPv4
-	seenNS := make(map[string]struct{})
-	seenCName := make(map[string]int)
-	for i := 1; i < maxRecursion; i++ {
-		responses, err := client.QueryParallel(host, requestType, servers)
-		if err != nil {
-			return nil, err
-		}
-		for _, server := range servers {
-			seenNS[server] = struct{}{}
-		}
-		if len(responses) == 0 {
-			return &traceData, nil
-		}
-		for _, dnsData := range responses {
-			if dnsData != nil && len(dnsData.Resolver) > 0 {
-				traceData.DNSData = append(traceData.DNSData, dnsData)
-			}
-		}
-
-		var nameServers []string
-		var nextCname string
-		for _, dnsData := range responses {
-			if dnsData == nil {
-				continue
-			}
-			for _, ns := range dnsData.NS {
-				ips, err := net.DefaultResolver.LookupIP(context.Background(), "ip4", ns)
-				if err != nil {
-					continue
-				}
-				for _, ip := range ips {
-					if !protocolstate.IsHostAllowed(executionID, ip.String()) {
-						continue
-					}
-					nameServers = append(nameServers, net.JoinHostPort(ip.String(), traceNameServerPort))
-				}
-			}
-			if nextCname == "" && len(dnsData.CNAME) > 0 {
-				nextCname = dnsData.CNAME[0]
-			}
-		}
-		nameServers = sliceutil.Dedupe(nameServers)
-		if len(nameServers) == 0 {
-			break
-		}
-
-		server := nameServers[rand.Intn(len(nameServers))] // #nosec G404
-		if _, ok := seenNS[server]; ok && nextCname == "" {
-			break
-		}
-		servers = []string{server}
-
-		if nextCname != "" {
-			seenCName[nextCname]++
-			if seenCName[nextCname] > retryabledns.DefaultMaxPerCNAMEFollows {
-				break
-			}
-			host = nextCname
-		}
-	}
-	return &traceData, nil
-}

pkg/protocols/dns/trace_test.go

@@ -1,124 +0,0 @@
-package dns
-
-import (
-	"context"
-	"net"
-	"sync/atomic"
-	"testing"
-
-	"github.com/miekg/dns"
-	"github.com/stretchr/testify/require"
-
-	"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
-	"github.com/projectdiscovery/nuclei/v3/pkg/model"
-	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
-	"github.com/projectdiscovery/nuclei/v3/pkg/output"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/retryabledns"
-)
-
-func serveTraceDNS(t *testing.T, address string, handler dns.HandlerFunc) string {
-	t.Helper()
-	pc, err := net.ListenPacket("udp", address)
-	require.NoError(t, err)
-	server := &dns.Server{PacketConn: pc, Handler: handler}
-	go func() { _ = server.ActivateAndServe() }()
-	t.Cleanup(func() { _ = server.Shutdown() })
-	return pc.LocalAddr().String()
-}
-
-func TestDNSTraceObeysNetworkPolicy(t *testing.T) {
-	// The zone of the traced name delegates to a name server on 127.0.0.1.
-	// The trace must query it when the policy allows the address, so that the
-	// test can see a trace that ignores the policy, and must skip it when the
-	// policy denies the address.
-	t.Run("allowed", func(t *testing.T) {
-		require.NotZero(t, runTraceAgainstLoopbackNameServer(t, nil), "trace did not query the allowed name server")
-	})
-	t.Run("denied", func(t *testing.T) {
-		require.Zero(t, runTraceAgainstLoopbackNameServer(t, []string{"127.0.0.1"}), "trace queried a denied name server")
-	})
-}
-
-func runTraceAgainstLoopbackNameServer(t *testing.T, excludeTargets []string) int32 {
-	t.Helper()
-	var internalHits atomic.Int32
-	internal := serveTraceDNS(t, "127.0.0.1:0", func(w dns.ResponseWriter, q *dns.Msg) {
-		internalHits.Add(1)
-		m := new(dns.Msg)
-		m.SetReply(q)
-		rr, _ := dns.NewRR(q.Question[0].Name + ` 0 IN TXT "internal"`)
-		m.Answer = append(m.Answer, rr)
-		_ = w.WriteMsg(m)
-	})
-	_, port, err := net.SplitHostPort(internal)
-	require.NoError(t, err)
-	previousPort := traceNameServerPort
-	traceNameServerPort = port
-	t.Cleanup(func() { traceNameServerPort = previousPort })
-
-	// public stands in for the root servers, the scan resolvers and the system resolver
-	public := serveTraceDNS(t, "127.0.0.1:0", func(w dns.ResponseWriter, q *dns.Msg) {
-		m := new(dns.Msg)
-		m.SetReply(q)
-		question := q.Question[0]
-		var rr dns.RR
-		switch {
-		case question.Name == "ns.nuclei-test." && question.Qtype == dns.TypeA:
-			rr, _ = dns.NewRR("ns.nuclei-test. 0 IN A 127.0.0.1")
-		case question.Name == "ns.nuclei-test.":
-		case question.Name == "trace.nuclei-test.":
-			rr, _ = dns.NewRR("trace.nuclei-test. 0 IN NS ns.nuclei-test.")
-		default:
-			rr, _ = dns.NewRR(question.Name + ` 0 IN TXT "public"`)
-		}
-		if rr != nil {
-			m.Answer = append(m.Answer, rr)
-		}
-		_ = w.WriteMsg(m)
-	})
-	previousRoots := retryabledns.RootDNSServersIPv4
-	retryabledns.RootDNSServersIPv4 = []string{public}
-	t.Cleanup(func() { retryabledns.RootDNSServersIPv4 = previousRoots })
-	previousResolver := net.DefaultResolver
-	net.DefaultResolver = &net.Resolver{
-		PreferGo: true,
-		Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {
-			var d net.Dialer
-			return d.DialContext(ctx, "udp", public)
-		},
-	}
-	t.Cleanup(func() { net.DefaultResolver = previousResolver })
-
-	options := testutils.DefaultOptions.Copy()
-	options.SetExecutionID(t.Name())
-	options.ExcludeTargets = excludeTargets
-	options.InternalResolversList = []string{public}
-	testutils.Init(options)
-	t.Cleanup(func() { testutils.Cleanup(options) })
-	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
-		ID:   "dns-trace-policy",
-		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}, Name: "test"},
-	})
-
-	request := &Request{
-		RequestType: DNSRequestTypeHolder{DNSRequestType: TXT},
-		Class:       "INET",
-		Name:        "{{FQDN}}",
-		// One retry keeps the client keyed by this test's resolvers, so a
-		// client cached by an earlier run cannot point at a closed server.
-		Retries:           1,
-		Trace:             true,
-		TraceMaxRecursion: 5,
-	}
-	require.NoError(t, request.Compile(executerOpts))
-
-	var event output.InternalEvent
-	input := contextargs.NewWithInput(context.Background(), "trace.nuclei-test")
-	err = request.ExecuteWithResults(input, nil, nil, func(e *output.InternalWrappedEvent) {
-		event = e.InternalEvent
-	})
-	require.NoError(t, err)
-	require.Contains(t, event["trace"], "ns.nuclei-test", "trace did not query the root servers")
-	return internalHits.Load()
-}

pkg/protocols/headless/engine/engine.go

@@ -31,11 +31,6 @@ type Browser struct {
 	// use getHTTPClient to get the http client
 	httpClient     *http.Client
 	httpClientOnce *sync.Once
-
-	// use getScreenshotDir to get the screenshot directory
-	screenshotDir     string
-	screenshotDirErr  error
-	screenshotDirOnce sync.Once
 }
 
 // New creates a new nuclei headless browser module
@@ -241,16 +236,6 @@ func (b *Browser) getHTTPClient() (*http.Client, error) {
 	return b.httpClient, err
 }
 
-// getScreenshotDir returns the directory that holds the screenshots of
-// templates when local file access is off. It is created once per browser
-// and kept after Close, so the screenshots stay available.
-func (b *Browser) getScreenshotDir() (string, error) {
-	b.screenshotDirOnce.Do(func() {
-		b.screenshotDir, b.screenshotDirErr = os.MkdirTemp("", "nuclei-screenshots-*")
-	})
-	return b.screenshotDir, b.screenshotDirErr
-}
-
 // Close closes the browser engine
 //
 // When connected over CDP, it does NOT close the browsers.

pkg/protocols/headless/engine/page.go

@@ -43,7 +43,6 @@ type Page struct {
 	variables          map[string]interface{}
 	lastActionNavigate *Action
 	cancel             context.CancelFunc
-	screenshots        int
 }
 
 // HistoryData contains the page request/response pairs

pkg/protocols/headless/engine/page_actions.go

@@ -526,13 +526,6 @@ func (p *Page) RightClickElement(act *Action, out ActionData) error {
 	return nil
 }
 
-// maxScreenshotsPerPage and maxScreenshotSize bound the disk space that the
-// screenshots of one page can use when local file access is off.
-const (
-	maxScreenshotsPerPage = 10
-	maxScreenshotSize     = 10 * 1024 * 1024
-)
-
 // Screenshot executes screenshot action on a page
 func (p *Page) Screenshot(act *Action, out ActionData) error {
 	to, err := p.getActionArg(act, "to")
@@ -540,52 +533,13 @@ func (p *Page) Screenshot(act *Action, out ActionData) error {
 		return err
 	}
 
-	generatedName := to == ""
-	if generatedName {
+	if to == "" {
 		to = ksuid.New().String()
 		if act.Name != "" {
 			out[act.Name] = to
 		}
 	}
 
-	lfaAllowed := protocolstate.IsLfaAllowed(p.options.Options)
-	if !lfaAllowed {
-		if p.screenshots >= maxScreenshotsPerPage {
-			return errkit.Newf("screenshot limit of %d per page reached, use -allow-local-file-access flag to remove it", maxScreenshotsPerPage)
-		}
-		p.screenshots++
-	}
-
-	var filePath string
-	if lfaAllowed {
-		filePath, err = fileutil.CleanPath(to)
-		if err != nil {
-			return errkit.Newf("could not clean output screenshot path %s", to)
-		}
-		if !strings.HasSuffix(filePath, ".png") {
-			filePath += ".png"
-		}
-	} else {
-		if err := p.isScreenshotPathAllowed(to); err != nil {
-			return err
-		}
-		// Without local file access the template only names a file inside a
-		// directory that nuclei creates, so it cannot choose where files land.
-		baseDir, err := p.instance.browser.getScreenshotDir()
-		if err != nil {
-			return errkit.Wrap(err, "could not create screenshot directory")
-		}
-		filePath = filepath.Join(baseDir, filepath.Clean(to))
-		// Add .png before the containment check, so that a name such as "."
-		// cannot turn into a sibling of the directory.
-		if !strings.HasSuffix(filePath, ".png") {
-			filePath += ".png"
-		}
-		if !isScreenshotPathWithinDirectory(filePath, baseDir) {
-			return ErrLFAccessDenied
-		}
-	}
-
 	var data []byte
 
 	fullpage, err := p.getActionArg(act, "fullpage")
@@ -601,8 +555,23 @@ func (p *Page) Screenshot(act *Action, out ActionData) error {
 	if err != nil {
 		return errors.Wrap(err, "could not take screenshot")
 	}
-	if !lfaAllowed && len(data) > maxScreenshotSize {
-		return errkit.Newf("screenshot of %d bytes is over the limit of %d bytes, use -allow-local-file-access flag to remove it", len(data), maxScreenshotSize)
+
+	to, err = fileutil.CleanPath(to)
+	if err != nil {
+		return errkit.Newf("could not clean output screenshot path %s", to)
+	}
+
+	// Build the final write path (with .png) BEFORE running any containment
+	// gate. Otherwise inputs like "." or a bare directory name pass the gate
+	// against `to` and then the .png suffix moves the actual write outside
+	// cwd (e.g. <cwd>.png is a sibling of cwd and not contained by it).
+	filePath := to
+	if !strings.HasSuffix(filePath, ".png") {
+		filePath += ".png"
+	}
+
+	if err := p.isScreenshotPathAllowed(filePath); err != nil {
+		return err
 	}
 
 	mkdir, err := p.getActionArg(act, "mkdir")
@@ -627,29 +596,25 @@ func (p *Page) Screenshot(act *Action, out ActionData) error {
 	if err != nil {
 		return errors.Wrap(err, "could not write screenshot")
 	}
-	// The generated name alone does not locate a file inside the screenshot directory.
-	if generatedName && act.Name != "" && !lfaAllowed {
-		out[act.Name] = filePath
-	}
 	gologger.Info().Msgf("Screenshot successfully saved at %v\n", filePath)
 	return nil
 }
 
-// isScreenshotPathAllowed checks the path a template gives for a screenshot.
-// Without local file access it must be relative and must not contain "..".
 func (p *Page) isScreenshotPathAllowed(to string) error {
 	if protocolstate.IsLfaAllowed(p.options.Options) {
 		return nil
 	}
 
-	if filepath.IsAbs(to) || filepath.VolumeName(to) != "" || strings.HasPrefix(to, "/") || strings.HasPrefix(to, `\`) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return errkit.Wrap(err, "could not get current working directory")
+	}
+
+	if !isScreenshotPathWithinDirectory(to, cwd) {
+		// writing outside of cwd requires -lfa flag
 		return ErrLFAccessDenied
 	}
-	for _, part := range strings.FieldsFunc(to, func(r rune) bool { return r == '/' || r == '\\' }) {
-		if part == ".." {
-			return ErrLFAccessDenied
-		}
-	}
+
 	return nil
 }
 

pkg/protocols/headless/engine/page_actions_test.go

@@ -449,7 +449,7 @@ func TestActionScreenshotDeniesSiblingPrefixPathWithoutLFA(t *testing.T) {
 	err = page.isScreenshotPathAllowed(filePath)
 	require.ErrorIs(t, err, ErrLFAccessDenied)
 
-	err = page.isScreenshotPathAllowed("test.png")
+	err = page.isScreenshotPathAllowed(filepath.Join(cwd, "test.png"))
 	require.NoError(t, err)
 }
 
@@ -541,8 +541,8 @@ func TestActionScreenshotDeniesPostExtensionEscape(t *testing.T) {
 		"a sibling .png in cwd's parent must be rejected")
 
 	// Sanity: a path inside cwd remains allowed.
-	require.NoError(t, page.isScreenshotPathAllowed("ok.png"),
-		"a relative path must still be allowed")
+	require.NoError(t, page.isScreenshotPathAllowed(filepath.Join(cwd, "ok.png")),
+		"a path inside cwd must still be allowed")
 }
 
 func TestActionScreenshotDeniesSymlinkedParentOutsideCWDWithoutLFA(t *testing.T) {
@@ -1051,10 +1051,6 @@ func TestMain(m *testing.M) {
 	code := m.Run()
 
 	browser.Close()
-	// screenshots of tests without local file access land here
-	if browser.screenshotDir != "" {
-		_ = os.RemoveAll(browser.screenshotDir)
-	}
 	os.Exit(code)
 }
 

pkg/protocols/headless/engine/screenshot_limits_test.go

@@ -1,155 +0,0 @@
-package engine
-
-import (
-	"context"
-	"io/fs"
-	"net/http"
-	"net/http/cookiejar"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-)
-
-// chdirTemp moves the test into a fresh working directory and returns it.
-func chdirTemp(t *testing.T) string {
-	t.Helper()
-	cwd := t.TempDir()
-	originalWd, err := os.Getwd()
-	require.NoError(t, err)
-	require.NoError(t, os.Chdir(cwd))
-	t.Cleanup(func() { require.NoError(t, os.Chdir(originalWd)) })
-	return cwd
-}
-
-// runWithoutLFA opens a test page with local file access off and runs the
-// actions on it.
-func runWithoutLFA(t *testing.T, actions []*Action) error {
-	t.Helper()
-	_, err := runWithoutLFAData(t, actions)
-	return err
-}
-
-// runWithoutLFAData is runWithoutLFA that also returns the action data.
-func runWithoutLFAData(t *testing.T, actions []*Action) (ActionData, error) {
-	t.Helper()
-
-	// the proxy of the shared browser dials with the empty execution id
-	require.NoError(t, protocolstate.Init(&types.Options{}))
-	opts := &types.Options{ExecutionId: t.Name(), AllowLocalFileAccess: false}
-	require.NoError(t, protocolstate.Init(opts))
-	t.Cleanup(func() { protocolstate.Close(opts.ExecutionId) })
-
-	instance, err := sharedBrowser.NewInstance()
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = instance.Close() })
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		_, _ = w.Write([]byte(`<html><body>Nuclei Test Page</body></html>`))
-	}))
-	t.Cleanup(ts.Close)
-
-	input := contextargs.NewWithInput(context.Background(), ts.URL)
-	input.CookieJar, err = cookiejar.New(nil)
-	require.NoError(t, err)
-
-	actions = append([]*Action{
-		{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
-		{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
-	}, actions...)
-	data, page, err := instance.Run(input, actions, nil, &Options{Timeout: 20 * time.Second, Options: opts})
-	if page != nil {
-		page.Close()
-	}
-	return data, err
-}
-
-func screenshotAction(args map[string]string) *Action {
-	return &Action{ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: args}
-}
-
-func requireNoFiles(t *testing.T, dir string) {
-	t.Helper()
-	var files []string
-	require.NoError(t, filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
-		if err == nil && !d.IsDir() {
-			files = append(files, path)
-		}
-		return err
-	}))
-	require.Empty(t, files, "template wrote into the working directory")
-}
-
-func TestScreenshotWithoutLFAWritesOnlyToScreenshotDir(t *testing.T) {
-	cwd := chdirTemp(t)
-	name := "shot-" + strconv.FormatInt(time.Now().UnixNano(), 10)
-	err := runWithoutLFA(t, []*Action{
-		screenshotAction(map[string]string{"to": name + "/deep/dir/marker", "mkdir": "true"}),
-	})
-	require.NoError(t, err)
-	requireNoFiles(t, cwd)
-
-	matches, err := filepath.Glob(filepath.Join(os.TempDir(), "nuclei-screenshots-*", name, "deep", "dir", "marker.png"))
-	require.NoError(t, err)
-	require.Len(t, matches, 1, "screenshot is not in the nuclei screenshot directory")
-}
-
-func TestScreenshotWithoutLFANamedResultIsFullPath(t *testing.T) {
-	chdirTemp(t)
-	action := screenshotAction(map[string]string{})
-	action.Name = "shot"
-	data, err := runWithoutLFAData(t, []*Action{action})
-	require.NoError(t, err)
-
-	path, ok := data["shot"].(string)
-	require.True(t, ok)
-	require.True(t, filepath.IsAbs(path), "named result %q does not locate the screenshot", path)
-	require.FileExists(t, path)
-}
-
-func TestScreenshotWithoutLFARejectsPaths(t *testing.T) {
-	cases := map[string]func(cwd string) string{
-		"absolute inside working directory": func(cwd string) string { return filepath.Join(cwd, "abs") },
-		"parent":                            func(string) string { return "../outside" },
-		"parent after folder":               func(string) string { return "a/../../outside" },
-	}
-	for name, to := range cases {
-		t.Run(name, func(t *testing.T) {
-			cwd := filepath.Join(chdirTemp(t), "work")
-			require.NoError(t, os.Mkdir(cwd, 0700))
-			require.NoError(t, os.Chdir(cwd))
-
-			err := runWithoutLFA(t, []*Action{screenshotAction(map[string]string{"to": to(cwd), "mkdir": "true"})})
-			require.ErrorIs(t, err, ErrLFAccessDenied)
-			requireNoFiles(t, filepath.Dir(cwd))
-		})
-	}
-}
-
-func TestScreenshotWithoutLFAIsLimitedPerPage(t *testing.T) {
-	chdirTemp(t)
-	// one more than the limit of 10 screenshots per page
-	actions := make([]*Action, 0, 11)
-	for range 11 {
-		actions = append(actions, screenshotAction(nil))
-	}
-	require.ErrorContains(t, runWithoutLFA(t, actions), "screenshot limit", "more screenshots than the limit were written for one page")
-}
-
-func TestScreenshotWithoutLFAIsLimitedInSize(t *testing.T) {
-	chdirTemp(t)
-	noise := `() => { document.body.style.margin='0'; const c=document.createElement('canvas'); c.width=2000; c.height=2000; document.body.appendChild(c); const x=c.getContext('2d'); const d=x.createImageData(2000,2000); for(let i=0;i<d.data.length;i++){d.data[i]=(Math.random()*256)|0; if(i%4==3)d.data[i]=255;} x.putImageData(d,0,0); return 'ok'; }`
-	err := runWithoutLFA(t, []*Action{
-		{ActionType: ActionTypeHolder{ActionType: ActionScript}, Data: map[string]string{"code": noise}},
-		screenshotAction(map[string]string{"fullpage": "true"}),
-	})
-	require.ErrorContains(t, err, "over the limit", "a screenshot over the size limit was written")
-}

pkg/protocols/http/build_request.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"net/http"
@@ -271,19 +272,98 @@ func (r *requestGenerator) validateSelfContainedHost(host string) error {
 // selfContained templates do not need/use target data and all values i.e {{Hostname}} , {{BaseURL}} etc are already available
 // in template . makeSelfContainedRequest parses and creates variables map and then creates corresponding http request or raw request
 func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, input *contextargs.Context, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
-	if r.request.isRaw() {
-		return r.makeSelfContainedRawRequest(ctx, input, data, payloads, dynamicValues)
+	isRawRequest := r.request.isRaw()
+
+	optionValues := generators.BuildPayloadFromOptions(r.request.options.Options)
+	values := generators.MergeMaps(optionValues)
+	if r.request.options.HasTemplateCtx(input.MetaInput) {
+		values = generators.MergeMaps(values, r.request.options.GetTemplateCtx(input.MetaInput).GetAll())
+	}
+	values = generators.MergeMaps(values, dynamicValues, payloads)
+	// adds all variables from `variables` section in template
+	scope := r.request.options.NewVariablesScope(optionValues)
+	r.request.options.AddTemplateCtxToVariablesScope(input.MetaInput, scope)
+	scope.AddData(dynamicValues, payloads, r.options.Constants)
+	evaluation := r.request.options.Variables.EvaluateWithInteractshScope(scope, r.request.options.Interactsh)
+
+	variablesMap, interactURLs := evaluation.Values, evaluation.InteractURLs
+	if len(interactURLs) > 0 {
+		r.interactshURLs = append(r.interactshURLs, interactURLs...)
 	}
 
-	values, _, err := r.selfContainedValues(input, payloads, dynamicValues, nil)
-	if err != nil {
-		return nil, err
+	values = generators.MergeMaps(variablesMap, values)
+	signerVars := GetDefaultSignerVars(r.request.Signature.Value)
+
+	// this will ensure that default signer variables are overwritten by other variables
+	values = generators.MergeMaps(signerVars, values, r.options.Constants)
+
+	for payloadName, payloadValue := range payloads {
+		renderedPayload, err := r.renderText(types.ToString(payloadValue), values)
+		if err != nil {
+			return nil, errkit.Wrap(err, "could not evaluate payload helper expressions")
+		}
+
+		payloads[payloadName] = renderedPayload
 	}
 
-	data, err = r.renderText(data, values)
+	values = generators.MergeMaps(values, payloads, r.options.Constants)
+
+	// priority of variables is as follows (from low to high) for self contained templates
+	// default signer vars < variables < cli vars < dynamic values < payload < constants
+
+	// evaluate request
+	data, err := r.renderText(data, values)
 	if err != nil {
 		return nil, errkit.Wrap(err, "could not evaluate helper expressions")
 	}
+
+	// If the request is a raw request, get the URL from the request
+	// header and use it to make the request.
+	if isRawRequest {
+		// Get the hostname from the URL section to build the request.
+		reader := bufio.NewReader(strings.NewReader(data))
+	read_line:
+		s, err := reader.ReadString('\n')
+		if err != nil {
+			return nil, fmt.Errorf("could not read request: %w", err)
+		}
+
+		// ignore all annotations
+		if stringsutil.HasPrefixAny(s, "@") {
+			goto read_line
+		}
+
+		parts := strings.Split(s, " ")
+		if len(parts) < 3 {
+			return nil, fmt.Errorf("malformed request supplied")
+		}
+
+		if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil && !r.request.SkipVariablesCheck {
+			return nil, errkit.Newf("unresolved variables `%v` found in request", parts[1])
+		}
+
+		parsed, err := urlutil.ParseURL(parts[1], true)
+		if err != nil {
+			return nil, fmt.Errorf("could not parse request URL: %w", err)
+		}
+		// self-contained requests choose their own destination URL, so validate
+		// that host against the network policy.
+		if err := r.validateSelfContainedHost(parsed.Hostname()); err != nil {
+			return nil, err
+		}
+		values = generators.MergeMaps(
+			generators.MergeMaps(dynamicValues, protocolutils.GenerateVariables(parsed, false, nil)),
+			values,
+		)
+
+		// Evaluate (replace) variable with final values
+		data, err = r.renderText(data, values)
+		if err != nil {
+			return nil, errkit.Wrap(err, "could not evaluate helper expressions")
+		}
+
+		return r.generateRawRequest(ctx, data, parsed, values, payloads)
+	}
 	if err := expressions.ContainsUnresolvedVariables(data); err != nil && !r.request.SkipVariablesCheck {
 		// early exit: if there are any unresolved variables in `path` after evaluation
 		// then return early since this will definitely fail
@@ -301,227 +381,6 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, input *
 	return r.generateHttpRequest(ctx, urlx, values, payloads)
 }
 
-// makeSelfContainedRawRequest makes a self-contained raw request. Its request
-// line names the URL, and that URL provides variables such as {{Hostname}} to
-// the rest of the request. Each part of the template text is rendered once,
-// so inserted values such as response data are sent unchanged:
-//
-//  1. Render the request line and check its host against the network policy.
-//  2. Evaluate again the template variables and payloads whose template text
-//     names URL variables. The others keep their values, so a random value
-//     is the same in the request line and in the rest of the request.
-//  3. Render the annotations and the rest of the request.
-func (r *requestGenerator) makeSelfContainedRawRequest(ctx context.Context, input *contextargs.Context, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
-	annotations, requestLine, rest, err := splitRequestLine(data)
-	if err != nil {
-		return nil, err
-	}
-
-	// Payloads are rendered in place, so keep their template text for step 2.
-	payloadText := generators.CopyMap(payloads)
-
-	values, authored, err := r.selfContainedValues(input, payloads, dynamicValues, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	requestLine, err = r.renderText(requestLine, values)
-	if err != nil {
-		return nil, errkit.Wrap(err, "could not evaluate helper expressions")
-	}
-
-	// Parse the line that raw.ParseRawRequest will read, even if a value
-	// added a line break.
-	firstLine, _, _ := strings.Cut(requestLine, "\n")
-
-	parsed, err := r.requestLineURL(firstLine)
-	if err != nil {
-		return nil, err
-	}
-
-	// URL variables only fill in names that have no other value.
-	fixed := make(map[string]interface{})
-
-	for name, value := range protocolutils.GenerateVariables(parsed, false, nil) {
-		if _, ok := values[name]; !ok {
-			fixed[name] = value
-		}
-	}
-
-	stale := r.staleValues(authored, payloadText, fixed)
-	if len(stale) == 0 {
-		values = generators.MergeMaps(fixed, values)
-	} else {
-		// Keep every other value, and evaluate the stale template variables
-		// and payloads again from their template text.
-		for name, value := range values {
-			if _, ok := stale[name]; !ok {
-				fixed[name] = value
-			}
-		}
-
-		for name := range stale {
-			if text, ok := payloadText[name]; ok {
-				payloads[name] = text
-			}
-		}
-
-		values, _, err = r.selfContainedValues(input, payloads, dynamicValues, fixed)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	annotations, err = r.renderText(annotations, values)
-	if err != nil {
-		return nil, errkit.Wrap(err, "could not evaluate helper expressions")
-	}
-
-	rest, err = r.renderText(rest, values)
-	if err != nil {
-		return nil, errkit.Wrap(err, "could not evaluate helper expressions")
-	}
-
-	data = annotations + requestLine + rest
-
-	// A value with a line break must not change which line is the request
-	// line, or the request would go to a host that was not checked.
-	if _, line, _, err := splitRequestLine(data); err != nil || line != firstLine {
-		return nil, errkit.New("malformed request supplied: a value changed the request line")
-	}
-
-	return r.generateRawRequest(ctx, data, parsed, values, payloads)
-}
-
-// staleValues returns the names of the authored values whose template text
-// names one of urlValues, directly or through another stale value. It reads
-// template text only, so data inserted into a value cannot mark it stale.
-// Variables are checked in evaluation order, then payloads, which are
-// rendered after the variables.
-func (r *requestGenerator) staleValues(authored, payloadText, urlValues map[string]interface{}) map[string]struct{} {
-	stale := make(map[string]struct{})
-	names := generators.CopyMap(urlValues)
-
-	mark := func(name string, text interface{}) {
-		if _, ok := authored[name]; !ok {
-			return
-		}
-
-		if expressions.ReferencesVariables(types.ToString(text), names) {
-			stale[name] = struct{}{}
-			names[name] = nil
-		}
-	}
-
-	r.request.options.Variables.ForEach(mark)
-
-	for name, text := range payloadText {
-		mark(name, text)
-	}
-
-	return stale
-}
-
-// requestLineURL returns the URL named by a rendered raw request line after
-// checking its host against the network policy.
-func (r *requestGenerator) requestLineURL(line string) (*urlutil.URL, error) {
-	parts := strings.Split(line, " ")
-	if len(parts) < 3 {
-		return nil, fmt.Errorf("malformed request supplied")
-	}
-
-	if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil && !r.request.SkipVariablesCheck {
-		return nil, errkit.Newf("unresolved variables `%v` found in request", parts[1])
-	}
-
-	parsed, err := urlutil.ParseURL(parts[1], true)
-	if err != nil {
-		return nil, fmt.Errorf("could not parse request URL: %w", err)
-	}
-	// self-contained requests choose their own destination URL, so validate
-	// that host against the network policy.
-	if err := r.validateSelfContainedHost(parsed.Hostname()); err != nil {
-		return nil, err
-	}
-
-	return parsed, nil
-}
-
-// selfContainedValues evaluates the template variables and renders the
-// payloads of a self-contained request, and returns the values to render the
-// request with. Payloads are rendered in place. Values in fixed are data:
-// they fill in names that have no other value, and a template variable or
-// payload with the same name keeps that value instead of being evaluated.
-// It also returns the values it produced from template text, that is the
-// evaluated template variables and the rendered payloads.
-func (r *requestGenerator) selfContainedValues(input *contextargs.Context, payloads, dynamicValues, fixed map[string]interface{}) (map[string]interface{}, map[string]interface{}, error) {
-	optionValues := generators.BuildPayloadFromOptions(r.request.options.Options)
-	values := generators.MergeMaps(fixed, optionValues)
-
-	if r.request.options.HasTemplateCtx(input.MetaInput) {
-		values = generators.MergeMaps(values, r.request.options.GetTemplateCtx(input.MetaInput).GetAll())
-	}
-	values = generators.MergeMaps(values, dynamicValues, payloads)
-	// adds all variables from `variables` section in template
-	scope := r.request.options.NewVariablesScope(optionValues)
-	r.request.options.AddTemplateCtxToVariablesScope(input.MetaInput, scope)
-	scope.AddData(dynamicValues, payloads, r.options.Constants, fixed)
-	evaluation := r.request.options.Variables.EvaluateWithInteractshScope(scope, r.request.options.Interactsh)
-
-	variablesMap, interactURLs := evaluation.Values, evaluation.InteractURLs
-	if len(interactURLs) > 0 {
-		r.interactshURLs = append(r.interactshURLs, interactURLs...)
-	}
-
-	values = generators.MergeMaps(variablesMap, values)
-	signerVars := GetDefaultSignerVars(r.request.Signature.Value)
-
-	// this will ensure that default signer variables are overwritten by other variables
-	values = generators.MergeMaps(signerVars, values, r.options.Constants)
-
-	// The evaluation owns TemplateValues, so add the rendered payloads to it.
-	authored := evaluation.TemplateValues
-
-	for payloadName, payloadValue := range payloads {
-		if _, ok := fixed[payloadName]; ok {
-			continue
-		}
-
-		renderedPayload, err := r.renderText(types.ToString(payloadValue), values)
-		if err != nil {
-			return nil, nil, errkit.Wrap(err, "could not evaluate payload helper expressions")
-		}
-
-		payloads[payloadName] = renderedPayload
-		authored[payloadName] = renderedPayload
-	}
-
-	// priority of variables is as follows (from low to high) for self contained templates
-	// default signer vars < variables < cli vars < dynamic values < payload < constants
-	return generators.MergeMaps(values, payloads, r.options.Constants), authored, nil
-}
-
-// splitRequestLine splits raw request template text at its request line, the
-// first line that is not an annotation. It returns the annotation lines, the
-// request line without its line break, and the remaining text starting with
-// that line break. The three parts concatenate back to raw.
-func splitRequestLine(raw string) (annotations, requestLine, rest string, err error) {
-	start := 0
-
-	for {
-		line, _, found := strings.Cut(raw[start:], "\n")
-		if !found {
-			return "", "", "", errkit.New("could not read request: no request line found")
-		}
-
-		if !strings.HasPrefix(line, "@") {
-			return raw[:start], line, raw[start+len(line):], nil
-		}
-
-		start += len(line) + 1
-	}
-}
-
 // generateHttpRequest generates http request from request data from template and variables
 // finalVars = contains all variables including generator and protocol specific variables
 // generatorValues = contains variables used in fuzzing or other generator specific values

pkg/protocols/http/build_request_test.go

@@ -14,14 +14,10 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
-	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
-	"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables"
-	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
 	urlutil "github.com/projectdiscovery/utils/url"
 )
@@ -455,211 +451,6 @@ func TestMakeSelfContainedRequestDoesNotAllocateRuntimeInteractshMarkerFromPaylo
 	require.Equal(t, "{{interactsh-url}}", string(body))
 }
 
-func TestSelfContainedRawRequestSendsResponseDataLiterally(t *testing.T) {
-	const envName = "NUCLEI_TEST_SELF_CONTAINED_ENV"
-
-	t.Setenv(envName, "ENV-VALUE")
-
-	// A fresh Options pointer, because option payloads are cached per pointer.
-	options := testutils.DefaultOptions.Copy()
-	options.EnvironmentVariables = true
-	testutils.Init(options)
-
-	tests := []struct {
-		name string
-		body string
-	}{
-		{name: "template variable", body: "{{secret}}"},
-		{name: "helper expression", body: "{{to_upper(seed)}}"},
-		{name: "environment variable", body: "{{" + envName + "}}"},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			probes := make(chan http.Header, 1)
-			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				switch r.URL.Path {
-				case "/source":
-					_, _ = io.WriteString(w, tt.body)
-				case "/probe":
-					header := r.Header.Clone()
-					header.Set("Host", r.Host)
-
-					probes <- header
-				}
-			}))
-			t.Cleanup(server.Close)
-
-			request := &Request{
-				ID:            "testing-self-contained-raw-response-data",
-				SelfContained: true,
-				// Send the probe even though it holds a literal {{...}}.
-				SkipVariablesCheck: true,
-				Raw: []string{
-					"GET " + server.URL + "/source HTTP/1.1\nHost: {{Hostname}}\n\n",
-					"GET " + server.URL + "/probe HTTP/1.1\nHost: {{Hostname}}\n" +
-						"Probe: [{{tpl}}]\nAuthored: [{{base64(seed)}}-{{" + envName + "}}]\n\n",
-				},
-				Operators: operators.Operators{
-					Extractors: []*extractors.Extractor{{
-						Name:     "tpl",
-						Part:     "body",
-						Type:     extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor},
-						Regex:    []string{".+"},
-						Internal: true,
-					}},
-				},
-			}
-			templateVars := variables.Variable{
-				InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(2),
-			}
-			templateVars.Set("seed", "hello")
-			templateVars.Set("secret", "TOP-SECRET")
-			executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
-				ID:   request.ID,
-				Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
-			})
-			executerOpts.Variables = templateVars
-			require.NoError(t, request.Compile(executerOpts))
-
-			err := request.ExecuteWithResults(
-				contextargs.NewWithInput(context.Background(), ""),
-				make(output.InternalEvent),
-				make(output.InternalEvent),
-				func(*output.InternalWrappedEvent) {},
-			)
-			require.NoError(t, err)
-
-			var probe http.Header
-			select {
-			case probe = <-probes:
-			default:
-				t.Fatal("probe request was not sent")
-			}
-
-			require.Equal(t, "["+tt.body+"]", probe.Get("Probe"), "response data must be sent unchanged")
-			// Template-authored text still renders, including variables derived
-			// from the request URL.
-			require.Equal(t, "[aGVsbG8=-ENV-VALUE]", probe.Get("Authored"))
-			require.Equal(t, strings.TrimPrefix(server.URL, "http://"), probe.Get("Host"))
-		})
-	}
-}
-
-// makeSelfContainedRawRequest compiles request as a self-contained raw request
-// with the given template variables and makes its first request.
-func makeSelfContainedRawRequest(t *testing.T, request *Request, vars [][2]string, dynamicValues map[string]interface{}) (*generatedRequest, error) {
-	t.Helper()
-
-	options := testutils.DefaultOptions
-	testutils.Init(options)
-
-	request.ID = "testing-self-contained-raw"
-	request.SelfContained = true
-	templateVars := variables.Variable{
-		InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(len(vars)),
-	}
-
-	for _, v := range vars {
-		templateVars.Set(v[0], v[1])
-	}
-	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
-		ID:   request.ID,
-		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
-	})
-	executerOpts.Variables = templateVars
-	require.NoError(t, request.Compile(executerOpts))
-
-	generator := request.newGenerator(false)
-	inputData, payloads, ok := generator.nextValue()
-	require.True(t, ok)
-
-	return generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), ""), inputData, payloads, dynamicValues)
-}
-
-func TestMakeSelfContainedRawRequestResolvesURLVariables(t *testing.T) {
-	request := &Request{
-		Raw: []string{"GET http://198.51.100.7/?t={{token}}&d={{data_token}} HTTP/1.1\nHost: {{Hostname}}\n" +
-			"X-Var: {{host_var}}\nX-Upper: {{upper_host}}\nX-Payload: {{user}}\n" +
-			"X-Token: {{token}}\nX-Data-Token: {{data_token}}\nX-Data: {{data}}\n\n"},
-		Payloads: map[string]interface{}{"user": []string{"admin@{{Hostname}}"}},
-		// Send values that hold a literal {{...}} from data.
-		SkipVariablesCheck: true,
-	}
-	got, err := makeSelfContainedRawRequest(t, request, [][2]string{
-		{"token", "{{rand_base(8)}}"},
-		{"data_token", "{{rand_base(8)}}-{{tpl}}"},
-		{"data", "{{tpl}}"},
-		{"host_var", "{{Hostname}}"},
-		{"upper_host", "{{to_upper(host_var)}}"},
-	}, map[string]interface{}{"tpl": "{{Hostname}}"})
-	require.NoError(t, err)
-
-	require.Equal(t, "198.51.100.7", got.request.Header.Get("X-Var"))
-	require.Equal(t, "198.51.100.7", got.request.Header.Get("X-Upper"))
-	require.Equal(t, "[email protected]", got.request.Header.Get("X-Payload"))
-	// Data that names a URL variable is sent unchanged.
-	require.Equal(t, "{{Hostname}}", got.request.Header.Get("X-Data"))
-	// Values whose template text names no URL variable are not evaluated
-	// again, even when data inserted into them does.
-	require.Equal(t, got.request.Params.Get("t"), got.request.Header.Get("X-Token"))
-	require.Equal(t, got.request.Params.Get("d"), got.request.Header.Get("X-Data-Token"))
-	require.True(t, strings.HasSuffix(got.request.Header.Get("X-Data-Token"), "-{{Hostname}}"))
-}
-
-func TestMakeSelfContainedRawRequestRejectsMovedRequestLine(t *testing.T) {
-	request := &Request{
-		Raw: []string{"@timeout: {{wait}}\nGET http://198.51.100.7/ HTTP/1.1\nHost: {{Hostname}}\n\n"},
-	}
-	_, err := makeSelfContainedRawRequest(t, request, nil, map[string]interface{}{
-		"wait": "5s\nGET http://192.0.2.1/secret HTTP/1.1",
-	})
-	require.ErrorContains(t, err, "changed the request line")
-}
-
-func TestSplitRequestLine(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		name                           string
-		raw                            string
-		annotations, requestLine, rest string
-		wantErr                        bool
-	}{
-		{
-			name:        "no annotations",
-			raw:         "GET http://a/ HTTP/1.1\r\nHost: a\r\n\r\n",
-			requestLine: "GET http://a/ HTTP/1.1\r",
-			rest:        "\nHost: a\r\n\r\n",
-		},
-		{
-			name:        "annotations",
-			raw:         "@tls-sni: {{Hostname}}\n@timeout: 5s\nGET http://a/ HTTP/1.1\nHost: a\n",
-			annotations: "@tls-sni: {{Hostname}}\n@timeout: 5s\n",
-			requestLine: "GET http://a/ HTTP/1.1",
-			rest:        "\nHost: a\n",
-		},
-		{name: "no line break", raw: "GET http://a/ HTTP/1.1", wantErr: true},
-		{name: "annotations only", raw: "@timeout: 5s\n", wantErr: true},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
-
-			annotations, requestLine, rest, err := splitRequestLine(tt.raw)
-			if tt.wantErr {
-				require.Error(t, err)
-
-				return
-			}
-
-			require.NoError(t, err)
-			require.Equal(t, tt.annotations, annotations)
-			require.Equal(t, tt.requestLine, requestLine)
-			require.Equal(t, tt.rest, rest)
-		})
-	}
-}
-
 func TestExecuteRequestDoesNotReevaluateGeneratedPayloadMetadata(t *testing.T) {
 	options := testutils.DefaultOptions
 

pkg/protocols/http/http.go

@@ -402,7 +402,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 
 	// If we have request level timeout, ignore http client timeouts
 	for _, req := range request.Raw {
-		if hasTimeoutAnnotation(req) {
+		if reTimeoutAnnotation.MatchString(req) {
 			connectionConfiguration.NoTimeout = true
 		}
 	}

pkg/protocols/http/httpcache/httpcache.go

@@ -94,10 +94,21 @@ func NewTransportWrapper() func(http.RoundTripper) http.RoundTripper {
 			},
 			DisableWarningHeader: true,
 		}
-		return transport
+		return &unwrappingTransport{Transport: transport}
 	}
 }
 
+// unwrappingTransport lets code that must not use the cache, such as reporting
+// clients that need TLS verification, reach the wrapped transport.
+type unwrappingTransport struct {
+	*httpcache.Transport
+}
+
+// Unwrap returns the transport the cache wraps.
+func (t *unwrappingTransport) Unwrap() http.RoundTripper {
+	return t.Transport.Transport
+}
+
 func openDiskCache(dir string, maxBytes int64) (httpcache.Cache, *leveldb.DB, error) {
 	if err := os.MkdirAll(dir, 0o700); err != nil {
 		return nil, nil, fmt.Errorf("create cache dir: %w", err)

pkg/protocols/http/httpcache/httpcache_test.go

@@ -1,6 +1,7 @@
 package httpcache
 
 import (
+	"crypto/tls"
 	"fmt"
 	"io"
 	"net/http"
@@ -10,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
 	libcache "github.com/sandrolain/httpcache"
 )
 
@@ -178,3 +180,29 @@ func TestDiskCacheRespectsSizeCap(t *testing.T) {
 		t.Fatal("stored a response over a 1-byte directory cap")
 	}
 }
+
+// Reporting clients send tokens, so tlsverify must reach the scan transport
+// under the cache wrapper and turn certificate verification on.
+func TestTransportWrapperUnwrapsForTLSVerification(t *testing.T) {
+	cacheMu.Lock()
+	saved := cache
+	cache = libcache.NewMemoryCache()
+	cacheMu.Unlock()
+	t.Cleanup(func() {
+		cacheMu.Lock()
+		cache = saved
+		cacheMu.Unlock()
+	})
+
+	base := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
+	wrapped := NewTransportWrapper()(base)
+	u, ok := wrapped.(interface{ Unwrap() http.RoundTripper })
+	if !ok || u.Unwrap() != base {
+		t.Fatal("the cache wrapper does not unwrap to the scan transport")
+	}
+
+	got, ok := tlsverify.Client(&http.Client{Transport: wrapped}).Transport.(*http.Transport)
+	if !ok || got.TLSClientConfig.InsecureSkipVerify {
+		t.Fatal("tlsverify did not turn verification on under the cache wrapper")
+	}
+}

pkg/protocols/http/httpclientpool/clientpool.go

@@ -166,6 +166,11 @@ func (t *connTrackingTransport) RoundTrip(req *http.Request) (*http.Response, er
 	return resp, err
 }
 
+// Unwrap returns the underlying transport.
+func (t *connTrackingTransport) Unwrap() http.RoundTripper {
+	return t.base
+}
+
 func (t *connTrackingTransport) CloseIdleConnections() {
 	type closeIdler interface{ CloseIdleConnections() }
 	if ci, ok := t.base.(closeIdler); ok {

pkg/protocols/http/raw/raw.go

@@ -14,6 +14,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider/authx"
 	"github.com/projectdiscovery/rawhttp/client"
 	"github.com/projectdiscovery/utils/errkit"
+	stringsutil "github.com/projectdiscovery/utils/strings"
 	urlutil "github.com/projectdiscovery/utils/url"
 )
 
@@ -192,20 +193,23 @@ func readRawRequest(request string, unsafe bool) (*Request, error) {
 		Headers: make(map[string]string),
 	}
 
-	_, request = SplitAnnotations(request)
-
 	// store body if it is unsafe request
 	if unsafe {
-		rawRequest.UnsafeRawBytes = []byte(request)
+		rawRequest.UnsafeRawBytes = stripLeadingAnnotations(request)
 	}
 
 	// parse raw request
 	source := strings.NewReader(request)
 	reader := bufio.NewReader(source)
+read_line:
 	s, err := reader.ReadString('\n')
 	if err != nil {
 		return nil, fmt.Errorf("could not read request: %w", err)
 	}
+	// ignore all annotations
+	if stringsutil.HasPrefixAny(s, "@") {
+		goto read_line
+	}
 
 	parts := strings.Fields(s)
 	if len(parts) > 0 {
@@ -288,18 +292,34 @@ func readRawRequest(request string, unsafe bool) (*Request, error) {
 
 }
 
-// SplitAnnotations splits a raw request into its annotations and the request
-// that follows them. Annotations are the lines starting with "@" before the
-// request line. A line starting with "@" after the request line is part of the
-// request, so text rendered into the headers or the body never becomes an
-// annotation.
-func SplitAnnotations(request string) (annotations, rest string) {
-	rest = request
-	for strings.HasPrefix(rest, "@") {
-		_, rest, _ = strings.Cut(rest, "\n")
+func stripLeadingAnnotations(request string) []byte {
+	reader := bufio.NewReader(strings.NewReader(request))
+	buffer := bufferPool.Get().(*bytes.Buffer)
+	buffer.Reset()
+	defer func() {
+		buffer.Reset()
+		bufferPool.Put(buffer)
+	}()
+
+	requestLineFound := false
+	for {
+		line, err := reader.ReadString('\n')
+		if len(line) > 0 {
+			if requestLineFound || !stringsutil.HasPrefixAny(line, "@") {
+				requestLineFound = true
+				_, _ = buffer.WriteString(line)
+			}
+		}
+
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return []byte(request)
+		}
 	}
 
-	return request[:len(request)-len(rest)], rest
+	return append([]byte(nil), buffer.Bytes()...)
 }
 
 // TryFillCustomHeaders after the Host header

pkg/protocols/http/raw/raw_test.go

@@ -310,51 +310,6 @@ Connection: close`, parseURL(t, "http://scanme.sh"), true, false)
 	require.NotContains(t, string(request.UnsafeRawBytes), "@Host:", "annotation lines must not be present in unsafe raw bytes")
 }
 
-func TestSplitAnnotations(t *testing.T) {
-	tests := []struct {
-		name        string
-		request     string
-		annotations string
-	}{
-		{name: "empty"},
-		{name: "no annotations", request: "GET / HTTP/1.1\r\nHost: a\r\n\r\n"},
-		{
-			name:        "leading annotations",
-			request:     "@Host: b\r\n@timeout: 5s\r\nGET / HTTP/1.1\r\nHost: a\r\n\r\n",
-			annotations: "@Host: b\r\n@timeout: 5s\r\n",
-		},
-		{
-			name:        "annotation lines after the request line",
-			request:     "@once\nPOST / HTTP/1.1\n@Host: c\n\nhello\n@Host: d\n",
-			annotations: "@once\n",
-		},
-		{
-			name:        "annotations without a request line",
-			request:     "@Host: b\n@once",
-			annotations: "@Host: b\n@once",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			annotations, rest := SplitAnnotations(tt.request)
-			require.Equal(t, tt.annotations, annotations)
-			require.Equal(t, tt.request, annotations+rest)
-		})
-	}
-}
-
-func TestParseUnsafeRequestKeepsAnnotationLinesInBody(t *testing.T) {
-	const (
-		head = "POST /foo HTTP/1.1\nHost: {{Hostname}}\n\n"
-		body = "hello\n@Host: decoy.example\n"
-	)
-	request, err := Parse("@Host: honey.scanme.sh\n"+head+body, parseURL(t, "http://scanme.sh"), true, false)
-	require.NoError(t, err)
-	require.Equal(t, head+body, string(request.UnsafeRawBytes))
-	require.Equal(t, body, request.Data)
-}
-
 func TestTryFillCustomHeaders(t *testing.T) {
 	testValue := "GET /manager/html HTTP/1.1\r\nHost: Test\r\n"
 	expected := "GET /test/manager/html HTTP/1.1\r\nHost: Test\r\ntest: test\r\n"

pkg/protocols/http/request.go

@@ -1675,7 +1675,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		}
 
 		responseContentType := respChain.Response().Header.Get("Content-Type")
-		isResponseTruncated := respChain.Body().Len() >= maxBodylimit
+		isResponseTruncated := (request.MaxSize > 0 && respChain.Body().Len() >= request.MaxSize) ||
+			(request.options.Options.ResponseReadSize > 0 && respChain.Body().Len() >= request.options.Options.ResponseReadSize)
 		dumpResponse(event, request, fullResponseStr, formedURL, responseContentType, isResponseTruncated, input.MetaInput.Input)
 
 		callback(event)
@@ -1988,8 +1989,7 @@ func (request *Request) needsFullResponse(generatedRequest *generatedRequest) bo
 	}
 	if request.options != nil && request.options.Options != nil {
 		cliOptions := request.options.Options
-		// -ms writes the response of failed requests too, and a failure never reaches the lazy fill
-		if cliOptions.Debug || cliOptions.DebugResponse || cliOptions.StoreResponse || cliOptions.HTTPStats || cliOptions.MatcherStatus {
+		if cliOptions.Debug || cliOptions.DebugResponse || cliOptions.StoreResponse || cliOptions.HTTPStats {
 			return true
 		}
 	}

pkg/protocols/http/request_annotations.go

@@ -11,7 +11,6 @@ import (
 	"github.com/projectdiscovery/fastdialer/fastdialer"
 	commoninteractsh "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/raw"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
 	"github.com/projectdiscovery/retryablehttp-go"
 	"github.com/projectdiscovery/utils/errkit"
@@ -55,9 +54,8 @@ func parseFlowAnnotations(rawRequest string) (flowMark, bool) {
 	var fm flowMark
 	// parse request for known override annotations
 	var hasFlowOverride bool
-	annotations, _ := raw.SplitAnnotations(rawRequest)
 	// @once
-	if reOnceAnnotation.MatchString(annotations) {
+	if reOnceAnnotation.MatchString(rawRequest) {
 		fm = Once
 		hasFlowOverride = true
 	}
@@ -65,12 +63,6 @@ func parseFlowAnnotations(rawRequest string) (flowMark, bool) {
 	return fm, hasFlowOverride
 }
 
-// hasTimeoutAnnotation reports whether rawRequest has a @timeout annotation.
-func hasTimeoutAnnotation(rawRequest string) bool {
-	annotations, _ := raw.SplitAnnotations(rawRequest)
-	return reTimeoutAnnotation.MatchString(annotations)
-}
-
 type annotationOverrides struct {
 	request        *retryablehttp.Request
 	cancelFunc     context.CancelFunc
@@ -80,12 +72,10 @@ type annotationOverrides struct {
 
 // parseAnnotations and override requests settings
 func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Request) (overrides annotationOverrides, modified bool) {
-	// rawRequest is rendered, so its headers and body can hold values taken
-	// from responses. Only the annotation lines may override the request.
-	annotations, _ := raw.SplitAnnotations(rawRequest)
+	// parse request for known override annotations
 
 	// @Host:target
-	if hosts := reHostAnnotation.FindStringSubmatch(annotations); len(hosts) > 0 {
+	if hosts := reHostAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 {
 		value := strings.TrimSpace(hosts[1])
 		// handle scheme
 		switch {
@@ -111,7 +101,7 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req
 	}
 
 	// @tls-sni:target
-	if hosts := reSniAnnotation.FindStringSubmatch(annotations); len(hosts) > 0 {
+	if hosts := reSniAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 {
 		value := strings.TrimSpace(hosts[1])
 		value = stringsutil.TrimPrefixAny(value, "http://", "https://")
 
@@ -150,7 +140,7 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req
 		modified = true
 		var ctx context.Context
 
-		if duration := reTimeoutAnnotation.FindStringSubmatch(annotations); len(duration) > 0 {
+		if duration := reTimeoutAnnotation.FindStringSubmatch(rawRequest); len(duration) > 0 {
 			value := strings.TrimSpace(duration[1])
 			if parsedTimeout, err := time.ParseDuration(value); err == nil {
 				// Cap at maximum allowed timeout to prevent DoS via extremely large timeout values

pkg/protocols/http/request_annotations_test.go

@@ -2,25 +2,13 @@ package http
 
 import (
 	"context"
-	"fmt"
-	"io"
 	"net/http"
-	"net/http/httptest"
-	"strings"
-	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
-	"github.com/projectdiscovery/nuclei/v3/pkg/model"
-	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
-	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
-	"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
-	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	commoninteractsh "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/retryablehttp-go"
@@ -203,123 +191,3 @@ func TestRequestParseAnnotationsTimeout(t *testing.T) {
 		require.False(t, deadlined, "could not get set request deadline")
 	})
 }
-
-func TestRequestParseAnnotationsIgnoresLinesAfterRequestLine(t *testing.T) {
-	req := &Request{connConfiguration: &httpclientpool.Configuration{}}
-	rawRequest := "POST /probe HTTP/1.1\r\n" +
-		"Host: example.com\r\n" +
-		"\r\n" +
-		"hello\n@Host: http://127.0.0.1:8797\n@tls-sni: decoy.example\n"
-
-	httpReq, err := retryablehttp.NewRequest(http.MethodPost, "https://example.com/probe", nil)
-	require.NoError(t, err)
-
-	overrides, modified := req.parseAnnotations(rawRequest, httpReq)
-	require.False(t, modified)
-	require.Equal(t, "https://example.com/probe", overrides.request.String())
-	require.Nil(t, overrides.request.TLS)
-}
-
-// A response value reused in a raw request must stay request content even when
-// one of its lines looks like an annotation. Annotations are only the lines
-// before the request line.
-func TestRawRequestReusingResponseWithAnnotationLine(t *testing.T) {
-	tests := []struct {
-		name          string
-		unsafe        bool
-		selfContained bool
-	}{
-		{name: "raw"},
-		{name: "unsafe", unsafe: true},
-		{name: "self-contained", selfContained: true},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			var decoyRequests atomic.Int32
-			decoy := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
-				decoyRequests.Add(1)
-			}))
-			defer decoy.Close()
-
-			responseBody := fmt.Sprintf("hello\n@Host: %s\n", decoy.URL)
-
-			var mu sync.Mutex
-			var probes []string
-			target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				if r.URL.Path == "/source" {
-					_, _ = io.WriteString(w, responseBody)
-					return
-				}
-				body, _ := io.ReadAll(r.Body)
-				mu.Lock()
-				probes = append(probes, r.Method+" "+r.URL.Path+" "+string(body))
-				mu.Unlock()
-			}))
-			defer target.Close()
-
-			baseURL, host, input := "", "{{Hostname}}", target.URL
-			if tt.selfContained {
-				// A self-contained request names its own target.
-				baseURL, host, input = target.URL, strings.TrimPrefix(target.URL, "http://"), ""
-			}
-			probeHeaders := "Host: " + host + "\nContent-Type: text/plain\n"
-			if tt.unsafe {
-				// An unsafe request is sent as written, so it states its own length.
-				probeHeaders += "Content-Length: {{len(tpl)}}\n"
-			}
-			request := &Request{
-				ID:            "testing-http-response-annotation-line",
-				Unsafe:        tt.unsafe,
-				SelfContained: tt.selfContained,
-				Raw: []string{
-					"GET " + baseURL + "/source HTTP/1.1\nHost: " + host + "\n\n",
-					"POST " + baseURL + "/probe HTTP/1.1\n" + probeHeaders + "\n{{tpl}}",
-				},
-				Operators: operators.Operators{
-					Extractors: []*extractors.Extractor{{
-						Name:     "tpl",
-						Part:     "body",
-						Type:     extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor},
-						Regex:    []string{`(?s).+`},
-						Internal: true,
-					}},
-				},
-			}
-
-			options := testutils.DefaultOptions
-			testutils.Init(options)
-			executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
-				ID:   request.ID,
-				Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}, Name: "test"},
-			})
-			require.NoError(t, request.Compile(executerOpts))
-
-			err := request.ExecuteWithResults(
-				contextargs.NewWithInput(context.Background(), input),
-				make(output.InternalEvent),
-				make(output.InternalEvent),
-				func(*output.InternalWrappedEvent) {},
-			)
-			require.NoError(t, err)
-
-			require.Zero(t, decoyRequests.Load(), "a response line must not redirect the request")
-			mu.Lock()
-			defer mu.Unlock()
-			require.Equal(t, []string{"POST /probe " + responseBody}, probes)
-		})
-	}
-}
-
-func TestTemplateAnnotationsAfterRequestLine(t *testing.T) {
-	const request = "GET / HTTP/1.1\r\nHost: {{Hostname}}\r\n\r\n@timeout: 5s\r\n@once\r\n"
-
-	require.False(t, hasTimeoutAnnotation(request))
-	require.True(t, hasTimeoutAnnotation("@timeout: 5s\r\n"+request))
-
-	_, hasFlowOverride := parseFlowAnnotations(request)
-	require.False(t, hasFlowOverride)
-	mark, hasFlowOverride := parseFlowAnnotations("@once\r\n" + request)
-	require.True(t, hasFlowOverride)
-	require.Equal(t, Once, mark)
-}

pkg/protocols/http/request_test.go

@@ -1,7 +1,6 @@
 package http
 
 import (
-	"cmp"
 	"context"
 	"crypto/tls"
 	"fmt"
@@ -1866,84 +1865,6 @@ func TestHTTPResponseReadSizeHonoursTemplateMaxSize(t *testing.T) {
 	}
 }
 
-// storedResponseWriter keeps the stored response dumps.
-type storedResponseWriter struct {
-	*testutils.MockOutputWriter
-
-	responses []string
-}
-
-func (w *storedResponseWriter) WriteStoreDebugData(_, _, _, data string) {
-	if strings.Contains(data, "Dumped HTTP response") {
-		w.responses = append(w.responses, data)
-	}
-}
-
-// TestHTTPResponseTruncatedUsesEffectiveReadLimit checks that a stored response
-// is marked truncated exactly when its body reached the limit it was read with.
-func TestHTTPResponseTruncatedUsesEffectiveReadLimit(t *testing.T) {
-	previous := MaxBodyRead
-
-	t.Cleanup(func() { MaxBodyRead = previous })
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		w.Header().Set("Content-Type", "text/plain")
-		_, _ = w.Write([]byte(strings.Repeat("A", 2000)))
-	}))
-	defer server.Close()
-
-	tests := []struct {
-		name             string
-		defaultLimit     int // replaces MaxBodyRead when set
-		templateMaxSize  int
-		responseReadSize int
-		wantTruncated    bool
-	}{
-		{name: "default limit", defaultLimit: 1000, wantTruncated: true},
-		{name: "default limit, negative response-size-read", defaultLimit: 1000, responseReadSize: -1, wantTruncated: true},
-		{name: "body under default limit", wantTruncated: false},
-		{name: "template max-size", templateMaxSize: 500, wantTruncated: true},
-		{name: "response-size-read", responseReadSize: 800, wantTruncated: true},
-		{name: "max-size below response-size-read", templateMaxSize: 500, responseReadSize: 3000, wantTruncated: true},
-		{name: "limits above body length", templateMaxSize: 4000, responseReadSize: 3000, wantTruncated: false},
-	}
-
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			MaxBodyRead = cmp.Or(tc.defaultLimit, previous)
-
-			opts := testutils.DefaultOptions.Copy()
-			opts.StoreResponse = true
-			opts.ResponseReadSize = tc.responseReadSize
-			testutils.Init(opts)
-			t.Cleanup(func() { testutils.Cleanup(opts) })
-
-			request := &Request{
-				ID:      "test-truncated",
-				MaxSize: tc.templateMaxSize,
-				Raw:     []string{"GET / HTTP/1.1\nHost: {{Hostname}}\n\n"},
-			}
-			executerOptions := testutils.NewMockExecuterOptions(opts, &testutils.TemplateInfo{
-				ID:   request.ID,
-				Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
-			})
-			writer := &storedResponseWriter{MockOutputWriter: testutils.NewMockOutputWriter(opts.OmitTemplate)}
-			executerOptions.Output = writer
-			require.NoError(t, request.Compile(executerOptions))
-
-			err := request.ExecuteWithResults(
-				contextargs.NewWithInput(context.Background(), server.URL),
-				make(output.InternalEvent),
-				make(output.InternalEvent),
-				func(*output.InternalWrappedEvent) {},
-			)
-			require.NoError(t, err)
-			require.Len(t, writer.responses, 1)
-			require.Equal(t, tc.wantTruncated, strings.Contains(writer.responses[0], "(Truncated)"))
-		})
-	}
-}
-
 const httpToHTTPSPage = "<html><title>400 The plain HTTP request was sent to HTTPS port</title><center>The plain HTTP request was sent to HTTPS port</center></html>"
 
 // servePlainOrTLS accepts HTTP and TLS on one port. A plaintext request is

pkg/protocols/http/response_defer_test.go

@@ -248,26 +248,6 @@ func TestNeedsFullResponse(t *testing.T) {
 		require.NoError(t, req.Compile(execOpts))
 		require.True(t, req.needsFullResponse(nil))
 	})
-
-	t.Run("matcher status option requires full response", func(t *testing.T) {
-		msOpts := testutils.DefaultOptions.Copy()
-		msOpts.MatcherStatus = true
-
-		req := &Request{
-			ID:     "matcher-status-flag",
-			Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
-			Operators: operators.Operators{
-				Matchers: []*matchers.Matcher{{
-					Part:  "body",
-					Type:  matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
-					Words: []string{"test"},
-				}},
-			},
-		}
-		execOpts := testutils.NewMockExecuterOptions(msOpts, &testutils.TemplateInfo{ID: "matcher-status-flag"})
-		require.NoError(t, req.Compile(execOpts))
-		require.True(t, req.needsFullResponse(nil))
-	})
 }
 
 func TestExecuteRequestDeferredResponse_NoMatch(t *testing.T) {
@@ -315,49 +295,6 @@ func TestExecuteRequestDeferredResponse_NoMatch(t *testing.T) {
 	require.Contains(t, types.ToString(capturedEvent.InternalEvent["all_headers"]), "X-Custom-Header")
 }
 
-// With -ms a failed match is written out too, so its event must carry the
-// response even though no result is created.
-func TestExecuteRequestDeferredResponse_NoMatchWithMatcherStatus(t *testing.T) {
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		_, _ = fmt.Fprintf(w, "response body content")
-	}))
-	defer ts.Close()
-
-	options := testutils.DefaultOptions.Copy()
-	options.MatcherStatus = true
-	testutils.Init(options)
-
-	req := &Request{
-		ID:     "defer-no-match-ms",
-		Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
-		Path:   []string{"{{BaseURL}}"},
-		Operators: operators.Operators{
-			Matchers: []*matchers.Matcher{{
-				Part:  "body",
-				Type:  matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
-				Words: []string{"nonexistent-needle"},
-			}},
-		},
-	}
-
-	execOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
-		ID:   "defer-no-match-ms",
-		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}, Name: "test"},
-	})
-	require.NoError(t, req.Compile(execOpts))
-
-	var capturedEvent *output.InternalWrappedEvent
-	ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
-	err := req.ExecuteWithResults(ctxArgs, make(output.InternalEvent), make(output.InternalEvent), func(event *output.InternalWrappedEvent) {
-		capturedEvent = event
-	})
-	require.NoError(t, err)
-	require.NotNil(t, capturedEvent)
-	require.False(t, capturedEvent.HasResults())
-	require.Contains(t, types.ToString(capturedEvent.InternalEvent["response"]), "HTTP/1.1 200")
-	require.Contains(t, types.ToString(capturedEvent.InternalEvent["response"]), "response body content")
-}
-
 func TestExecuteRequestDeferredResponse_WithMatch(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("X-Server", "nuclei-test")

pkg/protocols/javascript/js.go

@@ -11,7 +11,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/alecthomas/chroma/v2/quick"
+	"github.com/alecthomas/chroma/quick"
 	"github.com/ditashi/jsbeautifier-go/jsbeautifier"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/goja"

pkg/protocols/whois/rdapclientpool/clientpool.go

@@ -1,57 +1,39 @@
 package rdapclientpool
 
 import (
-	"context"
-	"fmt"
-	"net"
-	"net/http"
 	"sync"
 
-	"github.com/openrdap/rdap/bootstrap"
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/rdap"
 )
 
-// clients holds one client per execution, because each one dials with the
-// network policy of its execution.
-var clients = map[string]*rdap.Client{}
+var normalClient *rdap.Client
 var m sync.Mutex
 
 // Init initializes the client pool implementation
 func Init(options *types.Options) error {
-	_, err := Get(options, nil)
-	return err
-}
+	m.Lock()
+	defer m.Unlock()
 
-// newClient returns an rdap client whose requests, bootstrap lookups and
-// redirects all dial through the fastdialer of the execution, so every
-// address it connects to is checked against the network policy.
-func newClient(options *types.Options) *rdap.Client {
-	executionID := options.ExecutionId
-	httpClient := &http.Client{
-		Transport: &http.Transport{
-			DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
-				dialers := protocolstate.GetDialersWithId(executionID)
-				if dialers == nil || dialers.Fastdialer == nil {
-					return nil, fmt.Errorf("dialers not initialized for %s", executionID)
-				}
-				return dialers.Fastdialer.Dial(ctx, network, address)
-			},
-		},
-		Timeout: options.GetTimeouts().HttpTimeout,
-	}
-	client := &rdap.Client{
-		HTTP:      httpClient,
-		Bootstrap: &bootstrap.Client{HTTP: httpClient},
+	// Don't create clients if already created in the past.
+	if normalClient != nil {
+		return nil
 	}
+
+	normalClient = &rdap.Client{}
 	if options.Verbose || options.Debug || options.DebugRequests || options.DebugResponse {
-		client.Verbose = func(text string) {
+		normalClient.Verbose = func(text string) {
 			gologger.Debug().Msgf("rdap: %s", text)
 		}
 	}
-	return client
+	return nil
+}
+
+func getNormalClient() *rdap.Client {
+	m.Lock()
+	defer m.Unlock()
+	return normalClient
 }
 
 // Configuration contains the custom configuration options for a client - placeholder
@@ -62,25 +44,7 @@ func (c *Configuration) Hash() string {
 	return ""
 }
 
-// Get creates or gets the client of the execution
+// Get creates or gets a client for the protocol based on custom configuration
 func Get(options *types.Options, configuration *Configuration) (*rdap.Client, error) {
-	m.Lock()
-	defer m.Unlock()
-	if client, ok := clients[options.ExecutionId]; ok {
-		return client, nil
-	}
-	client := newClient(options)
-	clients[options.ExecutionId] = client
-	return client, nil
-}
-
-// Close drops the client of an execution.
-func Close(executionID string) {
-	m.Lock()
-	client := clients[executionID]
-	delete(clients, executionID)
-	m.Unlock()
-	if client != nil {
-		client.HTTP.CloseIdleConnections()
-	}
+	return getNormalClient(), nil
 }

pkg/protocols/whois/rdapclientpool/clientpool_test.go

@@ -1,66 +0,0 @@
-package rdapclientpool
-
-import (
-	"net/http"
-	"net/http/httptest"
-	"net/url"
-	"sync/atomic"
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/projectdiscovery/rdap"
-	"github.com/stretchr/testify/require"
-)
-
-func newRdapServer(t *testing.T, hits *atomic.Int32) *httptest.Server {
-	t.Helper()
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		hits.Add(1)
-		w.Header().Set("Content-Type", "application/rdap+json")
-		_, _ = w.Write([]byte(`{"objectClassName":"domain","ldhName":"example.com"}`))
-	}))
-	t.Cleanup(server.Close)
-	return server
-}
-
-func rdapOptions(t *testing.T, executionID string, excludeTargets []string) *types.Options {
-	t.Helper()
-	options := types.DefaultOptions()
-	options.ExecutionId = executionID
-	options.ExcludeTargets = excludeTargets
-	require.NoError(t, protocolstate.Init(options))
-	t.Cleanup(func() { protocolstate.Close(options.ExecutionId) })
-	require.NoError(t, Init(options))
-	return options
-}
-
-func domainRequest(t *testing.T, server string) *rdap.Request {
-	t.Helper()
-	serverURL, err := url.Parse(server)
-	require.NoError(t, err)
-	request := rdap.NewAutoRequest("example.com")
-	request.Server = serverURL
-	return request
-}
-
-// Each execution gets a client that dials with its own network policy.
-func TestClientObeysExecutionNetworkPolicy(t *testing.T) {
-	var hits atomic.Int32
-	server := newRdapServer(t, &hits)
-
-	open := rdapOptions(t, t.Name()+"-open", nil)
-	restricted := rdapOptions(t, t.Name()+"-restricted", []string{"127.0.0.1"})
-
-	openClient, err := Get(open, nil)
-	require.NoError(t, err)
-	_, err = openClient.Do(domainRequest(t, server.URL))
-	require.NoError(t, err)
-	require.Equal(t, int32(1), hits.Load())
-
-	restrictedClient, err := Get(restricted, nil)
-	require.NoError(t, err)
-	_, err = restrictedClient.Do(domainRequest(t, server.URL))
-	require.Error(t, err)
-	require.Equal(t, int32(1), hits.Load(), "rdap request reached a denied address")
-}

pkg/protocols/whois/whois_policy_test.go

@@ -1,56 +0,0 @@
-package whois
-
-import (
-	"context"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"sync/atomic"
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
-	"github.com/projectdiscovery/nuclei/v3/pkg/output"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
-	"github.com/stretchr/testify/require"
-)
-
-func TestWhoisServerRedirectObeysNetworkPolicy(t *testing.T) {
-	var hits atomic.Int32
-	internal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		hits.Add(1)
-		w.Header().Set("Content-Type", "application/rdap+json")
-		_, _ = w.Write([]byte(`{"objectClassName":"domain","ldhName":"example.com"}`))
-	}))
-	t.Cleanup(internal.Close)
-	// the redirector listens on ::1 so that it is reached while 127.0.0.1 is denied
-	var redirects atomic.Int32
-	redirector := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		redirects.Add(1)
-		http.Redirect(w, r, internal.URL+r.URL.Path, http.StatusFound)
-	}))
-	listener, err := net.Listen("tcp", "[::1]:0")
-	require.NoError(t, err)
-	redirector.Listener = listener
-	redirector.Start()
-	t.Cleanup(redirector.Close)
-	_, port, err := net.SplitHostPort(listener.Addr().String())
-	require.NoError(t, err)
-
-	options := testutils.DefaultOptions.Copy()
-	options.SetExecutionID(t.Name())
-	options.ExcludeTargets = []string{"127.0.0.1"}
-	testutils.Init(options)
-	t.Cleanup(func() { protocolinit.Close(options.ExecutionId) })
-	require.True(t, protocolstate.IsHostAllowed(options.ExecutionId, "::1"))
-
-	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "whois-redirect"})
-	request := &Request{ID: "whois-redirect", Query: "example.com", Server: "http://[::1]:" + port + "/rdap/"}
-	require.NoError(t, request.Compile(executerOpts))
-
-	input := contextargs.NewWithInput(context.Background(), "example.com")
-	_ = request.ExecuteWithResults(input, nil, nil, func(*output.InternalWrappedEvent) {})
-	require.NotZero(t, redirects.Load(), "the redirector was never reached")
-	require.Zero(t, hits.Load(), "whois redirect reached a denied address")
-}

pkg/reporting/exporters/es/elasticsearch.go

@@ -71,10 +71,9 @@ func New(option *Options) (*Exporter, error) {
 			Transport: &http.Transport{
 				MaxIdleConns:        10,
 				MaxIdleConnsPerHost: 10,
-				// No DialTLSContext: fastdialer DialTLS ignores TLSClientConfig and
-				// skips verification; net/http does the handshake with TLSClientConfig.
-				DialContext:     dialers.Fastdialer.Dial,
-				TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification},
+				DialContext:         dialers.Fastdialer.Dial,
+				DialTLSContext:      dialers.Fastdialer.DialTLS,
+				TLSClientConfig:     &tls.Config{InsecureSkipVerify: option.SSLVerification},
 			},
 		}
 	}

pkg/reporting/exporters/es/tls_test.go

@@ -1,46 +0,0 @@
-package es
-
-import (
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"strconv"
-	"sync/atomic"
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/output"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/stretchr/testify/require"
-)
-
-func TestExporterVerifiesTLS(t *testing.T) {
-	var got atomic.Int32
-	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		got.Add(1)
-		w.WriteHeader(http.StatusCreated)
-	}))
-	defer srv.Close()
-	host, port, err := net.SplitHostPort(srv.Listener.Addr().String())
-	require.NoError(t, err)
-	p, err := strconv.Atoi(port)
-	require.NoError(t, err)
-
-	opts := types.DefaultOptions()
-	opts.SetExecutionID("es-tls-verify")
-	require.NoError(t, protocolstate.Init(opts))
-	defer protocolstate.Close(opts.ExecutionId)
-
-	newExporter := func(skipVerify bool) *Exporter {
-		e, err := New(&Options{IP: host, Port: p, SSL: true, SSLVerification: skipVerify, Username: "elastic", Password: "secret", IndexName: "nuclei", ExecutionId: opts.ExecutionId})
-		require.NoError(t, err)
-		return e
-	}
-
-	err = newExporter(false).Export(&output.ResultEvent{})
-	require.ErrorContains(t, err, "certificate")
-	require.Zero(t, got.Load(), "the password went to an untrusted server")
-
-	require.NoError(t, newExporter(true).Export(&output.ResultEvent{}))
-	require.EqualValues(t, 1, got.Load())
-}

pkg/reporting/exporters/splunk/splunkhec.go

@@ -63,10 +63,9 @@ func New(option *Options) (*Exporter, error) {
 			Transport: &http.Transport{
 				MaxIdleConns:        10,
 				MaxIdleConnsPerHost: 10,
-				// No DialTLSContext: fastdialer DialTLS ignores TLSClientConfig and
-				// skips verification; net/http does the handshake with TLSClientConfig.
-				DialContext:     dialers.Fastdialer.Dial,
-				TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification},
+				DialContext:         dialers.Fastdialer.Dial,
+				DialTLSContext:      dialers.Fastdialer.DialTLS,
+				TLSClientConfig:     &tls.Config{InsecureSkipVerify: option.SSLVerification},
 			},
 		}
 	}

pkg/reporting/exporters/splunk/tls_test.go

@@ -1,46 +0,0 @@
-package splunk
-
-import (
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"strconv"
-	"sync/atomic"
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/output"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/stretchr/testify/require"
-)
-
-func TestExporterVerifiesTLS(t *testing.T) {
-	var got atomic.Int32
-	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		got.Add(1)
-		w.WriteHeader(http.StatusCreated)
-	}))
-	defer srv.Close()
-	host, port, err := net.SplitHostPort(srv.Listener.Addr().String())
-	require.NoError(t, err)
-	p, err := strconv.Atoi(port)
-	require.NoError(t, err)
-
-	opts := types.DefaultOptions()
-	opts.SetExecutionID("splunk-tls-verify")
-	require.NoError(t, protocolstate.Init(opts))
-	defer protocolstate.Close(opts.ExecutionId)
-
-	newExporter := func(skipVerify bool) *Exporter {
-		e, err := New(&Options{Host: host, Port: p, SSL: true, SSLVerification: skipVerify, Token: "secret", IndexName: "nuclei", ExecutionId: opts.ExecutionId})
-		require.NoError(t, err)
-		return e
-	}
-
-	err = newExporter(false).Export(&output.ResultEvent{})
-	require.ErrorContains(t, err, "certificate")
-	require.Zero(t, got.Load(), "the token went to an untrusted server")
-
-	require.NoError(t, newExporter(true).Export(&output.ResultEvent{}))
-	require.EqualValues(t, 1, got.Load())
-}

pkg/reporting/reporting.go

@@ -20,6 +20,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/pdf"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
@@ -27,6 +28,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
+	"github.com/projectdiscovery/retryablehttp-go"
 	"github.com/projectdiscovery/utils/errkit"
 	fileutil "github.com/projectdiscovery/utils/file"
 	"go.uber.org/multierr"
@@ -76,6 +78,17 @@ type IssueTrackerStats struct {
 func New(options *Options, db string, doNotDedupe bool) (Client, error) {
 	client := &ReportingClient{options: options}
 
+	// The caller can pass the scan client, which skips TLS verification;
+	// trackers and exporters send tokens and passwords, so they get a copy
+	// that verifies certificates.
+	if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
+		if hc := tlsverify.Client(options.HttpClient.HTTPClient); hc != options.HttpClient.HTTPClient {
+			opts := retryablehttp.DefaultOptionsSingle
+			opts.Timeout = hc.Timeout // keep the caller's timeout, also when it is 0
+			options.HttpClient = retryablehttp.NewWithHTTPClient(hc, opts)
+		}
+	}
+
 	if options.GitHub != nil {
 		options.GitHub.HttpClient = options.HttpClient
 		options.GitHub.OmitRaw = options.OmitRaw

pkg/reporting/reporting_tls_test.go

@@ -0,0 +1,98 @@
+package reporting_test
+
+import (
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/model"
+	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/es"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitlab"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/stretchr/testify/require"
+)
+
+// connectProxy tunnels every CONNECT to target, whatever host the client asked
+// for, so no name is resolved and nothing leaves 127.0.0.1.
+func connectProxy(t *testing.T, target string) *httptest.Server {
+	p := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodConnect {
+			http.Error(w, "connect only", http.StatusMethodNotAllowed)
+			return
+		}
+		up, err := net.Dial("tcp", target)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadGateway)
+			return
+		}
+		c, _, err := w.(http.Hijacker).Hijack()
+		if err != nil {
+			_ = up.Close()
+			return
+		}
+		_, _ = c.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n"))
+		go func() { _, _ = io.Copy(up, c); _ = up.Close() }()
+		go func() { _, _ = io.Copy(c, up); _ = c.Close() }()
+	}))
+	t.Cleanup(p.Close)
+	return p
+}
+
+// With -pi and a proxy, the runner gives reporting the scan client, which skips
+// TLS verification. No tracker or exporter may send its secret to a server with
+// an untrusted certificate.
+func TestNewVerifiesTLSWithScanClient(t *testing.T) {
+	var hits atomic.Int32
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		w.WriteHeader(http.StatusUnauthorized)
+	}))
+	t.Cleanup(srv.Close)
+	proxy := connectProxy(t, srv.Listener.Addr().String())
+
+	opts := types.DefaultOptions()
+	opts.SetExecutionID("reporting-tls-test")
+	opts.ProxyInternal = true
+	opts.AliveHttpProxy = proxy.URL
+	require.NoError(t, protocolstate.Init(opts))
+	t.Cleanup(func() { protocolstate.Close(opts.ExecutionId) })
+	scanClient, err := httpclientpool.Get(opts, &httpclientpool.Configuration{}, "")
+	require.NoError(t, err)
+
+	event := &output.ResultEvent{
+		TemplateID: "test-template",
+		Host:       "https://target.example",
+		Matched:    "https://target.example/",
+		Info:       model.Info{Name: "test", SeverityHolder: severity.Holder{Severity: severity.High}},
+	}
+	configs := map[string]*reporting.Options{
+		"github":        {GitHub: &github.Options{BaseURL: "https://github.victim.example/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token"}},
+		"gitlab":        {GitLab: &gitlab.Options{BaseURL: "https://gitlab.victim.example/", Username: "u", Token: "test-token", ProjectName: "1"}},
+		"gitea":         {Gitea: &gitea.Options{BaseURL: "https://gitea.victim.example/", Token: "test-token", ProjectOwner: "o", ProjectName: "r"}},
+		"jira":          {Jira: &jira.Options{Cloud: true, URL: "https://jira.victim.example", AccountID: "a", Email: "[email protected]", Token: "test-token", ProjectName: "P", IssueType: "Bug"}},
+		"elasticsearch": {ElasticsearchExporter: &es.Options{Host: "es.victim.example", Port: 443, SSL: true, Username: "u", Password: "test-password", IndexName: "n"}},
+		"splunk":        {SplunkExporter: &splunk.Options{Host: "splunk.victim.example", Port: 443, SSL: true, Token: "test-token", IndexName: "n"}},
+	}
+	for name, ro := range configs {
+		ro.HttpClient = scanClient
+		ro.ExecutionId = opts.ExecutionId
+		client, err := reporting.New(ro, "", true)
+		if err == nil {
+			_ = client.CreateIssue(event)
+			client.Close()
+		}
+		require.Zero(t, hits.Load(), "%s sent a request to a server with an untrusted certificate", name)
+	}
+}

pkg/reporting/tlsverify/tlsverify.go

@@ -0,0 +1,68 @@
+// Package tlsverify gives trackers and exporters an HTTP client that verifies
+// TLS certificates, since they send tokens and passwords.
+package tlsverify
+
+import (
+	"crypto/tls"
+	"errors"
+	"net/http"
+)
+
+// errTLSDialerOnly is returned when a transport dials HTTPS only through a
+// custom TLS dialer: dropping that dialer would also drop its route or address
+// policy, so the copy refuses requests instead.
+var errTLSDialerOnly = errors.New("tlsverify: the transport has a custom TLS dialer and no DialContext, so certificate verification cannot be enforced")
+
+type refusingTransport struct{}
+
+func (refusingTransport) RoundTrip(*http.Request) (*http.Response, error) {
+	return nil, errTLSDialerOnly
+}
+
+// Client returns c, or a copy of c whose transport verifies TLS certificates
+// when c can skip verification (the scan client can). The copy keeps the proxy
+// and the dialer, so the address policy still applies, but it drops a custom
+// TLS dialer, because that dialer can use the scan TLS config. So the token
+// never goes over an unverified connection.
+//
+// A transport with an Unwrap() http.RoundTripper method declares that its
+// RoundTrip only observes or caches requests and enforces no policy; the copy
+// uses the unwrapped transport. Other transports are returned unchanged.
+func Client(c *http.Client) *http.Client {
+	rt := c.Transport
+	for {
+		u, ok := rt.(interface{ Unwrap() http.RoundTripper })
+		if !ok {
+			break
+		}
+		rt = u.Unwrap()
+	}
+	t, ok := rt.(*http.Transport)
+	if !ok {
+		return c
+	}
+	insecure := t.TLSClientConfig != nil && t.TLSClientConfig.InsecureSkipVerify
+	tlsDialer := t.DialTLSContext != nil || t.DialTLS != nil //nolint:staticcheck // DialTLS is deprecated but still used
+	if !insecure && !tlsDialer {
+		return c
+	}
+	copied := *c
+	if tlsDialer && t.DialContext == nil && t.Dial == nil { //nolint:staticcheck // Dial is deprecated but still used
+		copied.Transport = refusingTransport{}
+		return &copied
+	}
+	verifying := t.Clone()
+	verifying.DialTLSContext = nil
+	verifying.DialTLS = nil //nolint:staticcheck // DialTLS is deprecated but still used
+	if insecure {
+		// Keep the caller's roots, client certificates and checks; the server
+		// name of the scan config belongs to a scan target, not to this host.
+		verifying.TLSClientConfig.InsecureSkipVerify = false
+		verifying.TLSClientConfig.ServerName = ""
+		if verifying.TLSClientConfig.MinVersion < tls.VersionTLS12 {
+			verifying.TLSClientConfig.MinVersion = tls.VersionTLS12
+		}
+	}
+	copied.Transport = verifying
+	return &copied
+}

pkg/reporting/tlsverify/tlsverify_test.go

@@ -0,0 +1,67 @@
+package tlsverify
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"net"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// The scan client can skip TLS verification; the tracker must not send the
+// token over such a connection, and must not change the caller's client.
+func TestClientTurnsOnTLSVerification(t *testing.T) {
+	insecure := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
+	caller := &http.Client{Transport: insecure}
+
+	got := Client(caller)
+	gotTransport, ok := got.Transport.(*http.Transport)
+	require.True(t, ok)
+	require.False(t, gotTransport.TLSClientConfig.InsecureSkipVerify)
+	require.True(t, insecure.TLSClientConfig.InsecureSkipVerify, "the caller's transport changed")
+
+	secure := &http.Client{Transport: &http.Transport{}}
+	require.Same(t, secure, Client(secure))
+}
+
+func TestClientKeepsCallerTLSSettings(t *testing.T) {
+	pinned := func(tls.ConnectionState) error { return nil }
+	insecure := &http.Transport{
+		DialTLSContext: func(context.Context, string, string) (net.Conn, error) { return nil, errors.New("scan dialer") },
+		DialContext:    (&net.Dialer{}).DialContext,
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: true,
+			ServerName:         "scan-target.example",
+			MinVersion:         tls.VersionTLS10,
+			RootCAs:            x509.NewCertPool(),
+			VerifyConnection:   pinned,
+		},
+	}
+	got := Client(&http.Client{Transport: insecure}).Transport.(*http.Transport)
+	require.Nil(t, got.DialTLSContext)
+	require.NotNil(t, got.DialContext)
+	cfg := got.TLSClientConfig
+	require.False(t, cfg.InsecureSkipVerify)
+	require.Empty(t, cfg.ServerName)
+	require.Equal(t, uint16(tls.VersionTLS12), cfg.MinVersion)
+	require.Same(t, insecure.TLSClientConfig.RootCAs, cfg.RootCAs)
+	require.NotNil(t, cfg.VerifyConnection)
+	require.True(t, insecure.TLSClientConfig.InsecureSkipVerify, "the caller's config changed")
+}
+
+// A custom TLS dialer may carry the only route or address policy, so a
+// transport without DialContext cannot be rebuilt and must refuse requests.
+func TestClientRefusesTLSDialerWithoutDialContext(t *testing.T) {
+	var dialed bool
+	tlsOnly := &http.Transport{DialTLSContext: func(context.Context, string, string) (net.Conn, error) {
+		dialed = true
+		return nil, errors.New("policy dialer")
+	}}
+	_, err := Client(&http.Client{Transport: tlsOnly}).Get("https://tracker.example/")
+	require.ErrorIs(t, err, errTLSDialerOnly)
+	require.False(t, dialed)
+}

pkg/reporting/trackers/github/github.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"net/http"
 	"strconv"
 
 	"github.com/google/go-github/v92/github"
@@ -12,6 +11,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
 	"github.com/projectdiscovery/retryablehttp-go"
 	"golang.org/x/oauth2"
@@ -55,17 +55,16 @@ type Options struct {
 // New creates a new issue tracker integration client based on options.
 func New(options *Options) (*Integration, error) {
 	ctx := context.Background()
+	// Build the oauth2 client on top of the caller's client, so its transport
+	// (proxy, address policy) and redirect rules apply to every request.
+	if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
+		ctx = context.WithValue(ctx, oauth2.HTTPClient, tlsverify.Client(options.HttpClient.HTTPClient))
+	}
 	ts := oauth2.StaticTokenSource(
 		&oauth2.Token{AccessToken: options.Token},
 	)
 	tc := oauth2.NewClient(ctx, ts)
 
-	if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
-		if tcTransport, ok := tc.Transport.(*http.Transport); ok {
-			tcTransport.Proxy = options.HttpClient.HTTPClient.Transport.(*http.Transport).Proxy
-		}
-	}
-
 	clientOptions := []github.ClientOptionsFunc{github.WithHTTPClient(tc)}
 	if options.BaseURL != "" {
 		clientOptions = append(clientOptions, github.WithURLs(&options.BaseURL, nil))

pkg/reporting/trackers/github/github_test.go

@@ -0,0 +1,126 @@
+package github
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/model"
+	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	"github.com/projectdiscovery/retryablehttp-go"
+	"github.com/stretchr/testify/require"
+)
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+func testEvent() *output.ResultEvent {
+	return &output.ResultEvent{
+		TemplateID: "test-template",
+		Host:       "https://target.example",
+		Info:       model.Info{Name: "test", SeverityHolder: severity.Holder{Severity: severity.High}},
+	}
+}
+
+func clientWithTransport(rt http.RoundTripper) *retryablehttp.Client {
+	c := retryablehttp.NewClient(retryablehttp.DefaultOptionsSingle)
+	c.HTTPClient = &http.Client{Transport: rt}
+	return c
+}
+
+// The tracker must send every request through the caller's client, so a
+// caller's address policy also covers the GitHub API base URL.
+func TestNewUsesCallerHTTPClient(t *testing.T) {
+	var serverHits atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		serverHits.Add(1)
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusCreated)
+		_, _ = w.Write([]byte(`{"number":1,"html_url":"https://github.example/o/r/issues/1"}`))
+	}))
+	defer srv.Close()
+
+	t.Run("requests go through the caller's transport", func(t *testing.T) {
+		var seen atomic.Int32
+		rt := roundTripFunc(func(r *http.Request) (*http.Response, error) {
+			seen.Add(1)
+			require.Equal(t, "Bearer test-token", r.Header.Get("Authorization"))
+			return http.DefaultTransport.RoundTrip(r)
+		})
+		tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(rt)})
+		require.NoError(t, err)
+		_, err = tracker.CreateIssue(testEvent())
+		require.NoError(t, err)
+		require.Positive(t, seen.Load())
+	})
+
+	t.Run("a refusing transport stops the request", func(t *testing.T) {
+		before := serverHits.Load()
+		denied := errors.New("denied by address policy")
+		rt := roundTripFunc(func(*http.Request) (*http.Response, error) { return nil, denied })
+		tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(rt)})
+		require.NoError(t, err)
+		_, err = tracker.CreateIssue(testEvent())
+		require.ErrorContains(t, err, "denied by address policy")
+		require.Equal(t, before, serverHits.Load(), "the request reached the server")
+	})
+}
+
+func TestNewRefusesUntrustedCertificateFromInsecureCallerClient(t *testing.T) {
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusCreated)
+		_, _ = w.Write([]byte(`{"number":1}`))
+	}))
+	defer srv.Close()
+
+	insecure := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
+	tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(insecure)})
+	require.NoError(t, err)
+	_, err = tracker.CreateIssue(testEvent())
+	require.ErrorContains(t, err, "certificate")
+}
+
+type wrappingTransport struct{ base http.RoundTripper }
+
+func (w *wrappingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	return w.base.RoundTrip(r)
+}
+func (w *wrappingTransport) Unwrap() http.RoundTripper { return w.base }
+
+// The scan client wraps its transport and, with a SOCKS proxy, does TLS in its
+// own dialer with the scan TLS config, which skips verification.
+func TestNewRefusesUntrustedCertificateFromInsecureTLSDialer(t *testing.T) {
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusCreated)
+		_, _ = w.Write([]byte(`{"number":1}`))
+	}))
+	defer srv.Close()
+
+	var dialed atomic.Int32
+	var dialer net.Dialer
+	base := &http.Transport{
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			dialed.Add(1)
+			return dialer.DialContext(ctx, network, addr)
+		},
+		DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			conn, err := dialer.DialContext(ctx, network, addr)
+			if err != nil {
+				return nil, err
+			}
+			return tls.Client(conn, &tls.Config{InsecureSkipVerify: true}), nil
+		},
+	}
+	tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(&wrappingTransport{base: base})})
+	require.NoError(t, err)
+	_, err = tracker.CreateIssue(testEvent())
+	require.ErrorContains(t, err, "certificate")
+	require.Positive(t, dialed.Load(), "the caller's dialer was not used")
+}