Compare commits

..
Author SHA1 Message Date
Mzack9999 17d5bd4272 merge dev 2026-11-04 00:56:00 +01:00
Mzack9999 3aa5af8fd8 merge dev 2026-11-03 12:45:46 +01:00
Mzack9999 8d4960ad60 discard runtimes 2026-10-29 21:25:29 +01:00
97 changed files with 618 additions and 7203 deletions

No files matched your search

+1
View File
@@ -21,6 +21,7 @@ jobs:
# in parallel once it passes instead of queueing behind the unit tests
lint:
name: "Lint"
if: ${{ !endsWith(github.actor, '[bot]') }}
runs-on: ubuntu-latest
steps:
- uses: actions/[email protected]
+20 -19
View File
@@ -1,19 +1,20 @@
name: 🔤 Typos
on:
push:
branches: ["dev"]
pull_request:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
typos:
name: "Spell Check"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: crate-ci/[email protected]
name: 🔤 Typos
on:
push:
branches: ["dev"]
pull_request:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
typos:
name: "Spell Check"
if: ${{ !endsWith(github.actor, '[bot]') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: crate-ci/[email protected]
-1
View File
@@ -256,7 +256,6 @@ CONFIGURATIONS:
-rsr, -response-size-read int max response size to read in bytes
-rss, -response-size-save int max response size to read in bytes (default 1048576)
-rqss, -request-size-save int max request body size to save in bytes (0 to save the whole body) (default 32768)
-mrbi, -max-request-body-inflight int max request body bytes held by in-flight http requests; requests with bodies of 1MB or more queue past it (0 disables)
-reset reset removes all nuclei configuration and data files (including nuclei-templates)
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
-hae, -http-api-endpoint string experimental http api endpoint
-57
View File
@@ -1,57 +0,0 @@
package main
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
const headerChildEnv = "NUCLEI_HEADER_CHILD_ARGS"
// -H values can contain "#", on the command line, in a header file and in the
// config file, while lines of a header file that start with "#" are skipped as
// comments.
func TestHeaderValuesKeepHash(t *testing.T) {
if args := os.Getenv(headerChildEnv); args != "" {
options = &types.Options{Logger: gologger.DefaultLogger}
os.Args = append([]string{os.Args[0]}, strings.Split(args, "\n")...)
readConfig()
fmt.Printf("headers=%s\n", strings.Join(options.CustomHeaders, "|"))
os.Exit(0)
}
root := t.TempDir()
headerFile := filepath.Join(root, "headers.txt")
require.NoError(t, os.WriteFile(headerFile, []byte("# auth headers\nX-Token: abc#def\nReferer: https://app.example.com/#/dashboard\n"), 0o600))
configFile := filepath.Join(root, "config.yaml")
require.NoError(t, os.WriteFile(configFile, []byte("header:\n - \"X-Config: cfg#value\"\n"), 0o600))
tests := []struct {
name string
args []string
want string
}{
{"cli and header file", []string{"-H", "Cookie: sid=1; color=#ff0000", "-H", headerFile}, "Cookie: sid=1; color=#ff0000|X-Token: abc#def|Referer: https://app.example.com/#/dashboard"},
{"config file", []string{"-config", configFile}, "X-Config: cfg#value"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tempRoot := filepath.Join(t.TempDir(), "tmp")
require.NoError(t, os.Mkdir(tempRoot, 0o700))
args := append(tt.args, "-disable-update-check")
cmd := exec.Command(os.Args[0], "-test.run=^TestHeaderValuesKeepHash$", "-test.count=1")
cmd.Env = isolatedNucleiStorageEnv(root, tempRoot, headerChildEnv+"="+strings.Join(args, "\n"))
output, err := cmd.CombinedOutput()
require.NoError(t, err, string(output))
require.Contains(t, string(output), "headers="+tt.want+"\n")
})
}
}
+1 -18
View File
@@ -55,17 +55,6 @@ var (
inlineSecretsTempMu sync.Mutex
)
// normalizeHeaderValue trims a -H value and drops it when the whole line is a
// "#" comment. The goflags default strips inline comments, which cut values
// such as "Cookie: color=#fff" at the "#".
func normalizeHeaderValue(s string) string {
s = strings.TrimSpace(s)
if strings.HasPrefix(s, "#") {
return ""
}
return s
}
func main() {
options.Logger = gologger.DefaultLogger
@@ -266,11 +255,6 @@ func readConfig() *goflags.FlagSet {
var pdcpAuth string
var fuzzFlag bool
// FileStringSliceOptions as set up by runner.ConfigureOptions, but keeping
// "#" inside -H values
headerSliceOptions := goflags.FileStringSliceOptions
headerSliceOptions.Normalize = normalizeHeaderValue
flagSet := goflags.NewFlagSet()
flagSet.CaseSensitive = true
flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing
@@ -383,7 +367,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "mr", 10, "max number of redirects to follow for http templates"),
flagSet.BoolVarP(&options.DisableRedirects, "disable-redirects", "dr", false, "disable redirects for http templates"),
flagSet.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking
flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", nil, "custom header/cookie to include in all http request in header:value format (cli, file)", headerSliceOptions),
flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", nil, "custom header/cookie to include in all http request in header:value format (cli, file)", goflags.FileStringSliceOptions),
flagSet.RuntimeMapVarP(&options.Vars, "var", "V", nil, "custom vars in key=value format"),
flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"),
flagSet.BoolVarP(&options.SystemResolvers, "system-resolvers", "sr", false, "use system DNS resolving as error fallback"),
@@ -407,7 +391,6 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 0, "max response size to read in bytes"),
flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", unitutils.Mega, "max response size to read in bytes"),
flagSet.IntVarP(&options.RequestSaveSize, "request-size-save", "rqss", 32*unitutils.Kilo, "max request body size to save in bytes (0 to save the whole body)"),
flagSet.IntVarP(&options.MaxRequestBodyInFlight, "max-request-body-inflight", "mrbi", 0, "max request body bytes held by in-flight http requests; requests with bodies of 1MB or more queue past it (0 disables)"),
flagSet.CallbackVar(func() { resetRequested = true }, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"),
+5 -3
View File
@@ -15,7 +15,7 @@ require (
github.com/Mzack9999/go-rsync v0.0.0-20250821180103-81ffa574ef4d
github.com/RedTeamPentesting/adauth v0.5.4-0.20260511073005-3d18e8a5a687
github.com/adrg/xdg v0.5.3
github.com/alecthomas/chroma/v2 v2.27.0
github.com/alecthomas/chroma v0.10.0
github.com/alexsnet/go-vnc v0.1.0
github.com/alitto/pond v1.9.2
github.com/andybalholm/brotli v1.2.0
@@ -74,7 +74,6 @@ require (
github.com/miekg/dns v1.1.73
github.com/oiweiwei/go-msrpc v1.2.12
github.com/olekukonko/tablewriter v1.0.8
github.com/openrdap/rdap v0.9.1
github.com/ory/dockertest/v3 v3.12.0
github.com/pkg/errors v0.9.1
github.com/praetorian-inc/fingerprintx v1.1.15
@@ -176,6 +175,7 @@ require (
github.com/STARRY-S/zip v0.2.3 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/akrylysov/pogreb v0.10.2 // indirect
github.com/alecthomas/chroma/v2 v2.20.0 // indirect
github.com/alecthomas/kingpin/v2 v2.4.0 // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
@@ -237,6 +237,7 @@ require (
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/djherbis/times v1.6.0 // indirect
github.com/dlclark/regexp2 v1.11.5 // indirect
github.com/dlclark/regexp2/v2 v2.2.1 // indirect
github.com/docker/cli v29.2.0+incompatible // indirect
github.com/docker/go-connections v0.7.0 // indirect
@@ -263,7 +264,7 @@ require (
github.com/go-fed/httpsig v1.1.0 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.9.0 // indirect
github.com/go-jose/go-jose/v4 v4.1.6 // indirect
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
github.com/go-logr/logr v1.4.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
@@ -366,6 +367,7 @@ require (
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/opencontainers/runc v1.3.6 // indirect
github.com/openrdap/rdap v0.9.1 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/pierrec/lz4/v4 v4.1.23 // indirect
github.com/pjbgf/sha1cd v0.6.0 // indirect
+9 -6
View File
@@ -147,13 +147,15 @@ github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi
github.com/akrylysov/pogreb v0.10.2/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
github.com/alecthomas/chroma/v2 v2.27.0 h1:FodwmyOBgJULFYmDqibcp9pvfDLWdtPRh9v/r5BXYZs=
github.com/alecthomas/chroma/v2 v2.27.0/go.mod h1:NjJ3ciIgrqBNeIkWZ4e46nseoLDslxU1LmfCoL+wcY8=
github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
github.com/alecthomas/chroma/v2 v2.20.0 h1:sfIHpxPyR07/Oylvmcai3X/exDlE8+FA820NTz+9sGw=
github.com/alecthomas/chroma/v2 v2.20.0/go.mod h1:e7tViK0xh/Nf4BYHl00ycY6rV7b8iXBksI9E359yNmA=
github.com/alecthomas/kingpin/v2 v2.3.1/go.mod h1:oYL5vtsvEHZGHxU7DMp32Dvx+qL+ptGn6lWaot2vCNE=
github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY=
github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/alecthomas/repr v0.5.1 h1:E3G4t2QbHTSNpPKBgMTln5KLkZHLOcU7r37J4pXBuIg=
github.com/alecthomas/repr v0.5.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -356,6 +358,7 @@ github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c h1:+Zo5Ca9
github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c/go.mod h1:HJGU9ULdREjOcVGZVPB5s6zYmHi1RxzT71l2wQyLmnE=
github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2/v2 v2.2.1 h1:mf4KkFUj0gJuarK8P+LgiS+Lit7m9N1yAwEfPbee7R0=
@@ -452,8 +455,8 @@ github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuw
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-jose/go-jose/v4 v4.1.6 h1:sNRkftxFJxwsILiQukqp5z83QBSmntVwmgRY7AqBDO4=
github.com/go-jose/go-jose/v4 v4.1.6/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+5 -29
View File
@@ -4,8 +4,6 @@ import (
"bufio"
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"net/http"
@@ -89,7 +87,11 @@ func NewUploadWriter(ctx context.Context, logger *gologger.Logger, creds *pdcpau
tmp.Update()
u.uploadURL = tmp.URL
u.client = newUploadClient()
// create http client
opts := retryablehttp.DefaultOptionsSingle
opts.NoAdjustTimeout = true
opts.Timeout = time.Duration(3) * time.Minute
u.client = retryablehttp.NewClient(opts)
// create context
ctx, u.cancel = context.WithCancel(ctx)
@@ -99,32 +101,6 @@ func NewUploadWriter(ctx context.Context, logger *gologger.Logger, creds *pdcpau
return u, nil
}
// newUploadClient returns the client for cloud uploads. The upload sends the
// API key and the scan results, so the client verifies the server certificate.
// The retryablehttp default clients skip verification.
func newUploadClient() *retryablehttp.Client {
transport := retryablehttp.DefaultReusePooledTransport()
// fastdialer DialTLS does not verify certificates, so net/http does the
// TLS handshake over the fastdialer connection.
transport.DialTLSContext = nil
transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12}
opts := retryablehttp.DefaultOptionsSingle
opts.NoAdjustTimeout = true
opts.Timeout = time.Duration(3) * time.Minute
opts.HttpClient = &http.Client{Transport: transport}
// a certificate error does not go away on retry
defaultRetry := retryablehttp.DefaultRetryPolicy()
opts.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) {
var certErr *tls.CertificateVerificationError
if errors.As(err, &certErr) {
return false, nil
}
return defaultRetry(ctx, resp, err)
}
return retryablehttp.NewClient(opts)
}
// SetScanID sets the scan id for the upload writer
func (u *UploadWriter) SetScanID(id string) error {
if !xidRegex.MatchString(id) {
-55
View File
@@ -1,55 +0,0 @@
package pdcp
import (
"context"
"crypto/tls"
"crypto/x509"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/projectdiscovery/gologger"
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
"github.com/stretchr/testify/require"
)
func newTestUploadWriter(t *testing.T, gotKey *atomic.Value) (*UploadWriter, *httptest.Server) {
t.Helper()
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotKey.Store(r.Header.Get(pdcpauth.ApiKeyHeaderName))
_, _ = w.Write([]byte(`{"id":"abcdefghij0123456789"}`))
}))
t.Cleanup(srv.Close)
creds := &pdcpauth.PDCPCredentials{APIKey: "test-key", Server: srv.URL}
u, err := NewUploadWriter(context.Background(), gologger.DefaultLogger, creds)
require.NoError(t, err)
t.Cleanup(u.Close)
return u, srv
}
func TestUploadRejectsUntrustedCertificate(t *testing.T) {
var gotKey atomic.Value
u, _ := newTestUploadWriter(t, &gotKey)
err := u.upload([]byte("{}\n"))
require.Error(t, err)
var unknownAuthority x509.UnknownAuthorityError
require.ErrorAs(t, err, &unknownAuthority)
require.Nil(t, gotKey.Load(), "API key must not reach a server with an untrusted certificate")
}
func TestUploadTrustedCertificate(t *testing.T) {
var gotKey atomic.Value
u, srv := newTestUploadWriter(t, &gotKey)
pool := x509.NewCertPool()
pool.AddCert(srv.Certificate())
transport, ok := u.client.HTTPClient.Transport.(*http.Transport)
require.True(t, ok)
transport.TLSClientConfig = &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}
require.NoError(t, u.upload([]byte("{}\n")))
require.Equal(t, "test-key", gotKey.Load())
require.Equal(t, "abcdefghij0123456789", u.scanID)
}
-2
View File
@@ -73,7 +73,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
"github.com/projectdiscovery/retryablehttp-go"
@@ -677,7 +676,6 @@ func (r *Runner) RunEnumeration() error {
Catalog: r.catalog,
IssuesClient: r.issuesClient,
RateLimiter: r.rateLimiter,
RequestBodyBudget: httpbody.NewBudget(int64(r.options.MaxRequestBodyInFlight)),
Interactsh: r.interactsh,
ProjectFile: r.projectFile,
Browser: r.browser,
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"sort"
"strings"
"github.com/alecthomas/chroma/v2/quick"
"github.com/alecthomas/chroma/quick"
"github.com/logrusorgru/aurora/v4"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
@@ -1,89 +0,0 @@
//go:build integration
// +build integration
package integration_test
import (
"fmt"
"net"
"time"
"github.com/ory/dockertest/v3"
)
// ipmiSimSetup runs OpenIPMI's BMC simulator with an anonymous user,
// ADMIN/ADMIN and root/calvin, and cipher suite 0 enabled.
const ipmiSimSetup = `apt-get update -qq && apt-get install -y -qq --no-install-recommends openipmi >/dev/null && mkdir -p /etc/ipmi && cat > /etc/ipmi/lan.conf <<'EOF'
name "bmc"
set_working_mc 0x20
startlan 1
addr 0.0.0.0 623
priv_limit admin
allowed_auths_callback none md2 md5 straight
allowed_auths_user none md2 md5 straight
allowed_auths_operator none md2 md5 straight
allowed_auths_admin none md2 md5 straight
guid a123456789abcdefa123456789abcdef
endlan
user 1 true "" "" user 10 none md2 md5 straight
user 2 true "ADMIN" "ADMIN" admin 10 none md2 md5 straight
user 3 true "root" "calvin" admin 10 none md2 md5 straight
EOF
printf 'mc_setbmc 0x20\nmc_add 0x20 0 no-device-sdrs 0x23 9 8 0x9f 0x1291 0xf02 persist_sdr\nmc_enable 0x20\n' > /etc/ipmi/emu.cmd
exec ipmi_sim -c /etc/ipmi/lan.conf -f /etc/ipmi/emu.cmd -n`
type javascriptIPMI struct{}
func (j *javascriptIPMI) Execute(filePath string) error {
pool, err := dockertest.NewPool("")
if err != nil {
return fmt.Errorf("could not create docker pool: %w", err)
}
if err := pool.Client.Ping(); err != nil {
return fmt.Errorf("could not connect to Docker: %w", err)
}
resource, err := pool.RunWithOptions(&dockertest.RunOptions{
Repository: "debian",
Tag: "bookworm-slim",
Platform: "linux/amd64",
Cmd: []string{"sh", "-c", ipmiSimSetup},
ExposedPorts: []string{"623/udp"},
})
if err != nil {
return fmt.Errorf("could not start ipmi simulator: %w", err)
}
defer purge(pool, resource)
address := net.JoinHostPort("127.0.0.1", resource.GetPort("623/udp"))
if err := waitForServiceCheck(address, 3*time.Minute, rmcpPingCheck); err != nil {
return err
}
results, err := runSignedNucleiTemplateAndGetResults(filePath, address, debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
// rmcpPingCheck waits for an answer to an RMCP presence ping, since the
// port is udp and cannot be probed with a connect.
func rmcpPingCheck(address string) error {
conn, err := net.Dial("udp", address)
if err != nil {
return err
}
defer func() { _ = conn.Close() }()
if _, err := conn.Write([]byte{0x06, 0x00, 0xff, 0x06, 0x00, 0x00, 0x11, 0xbe, 0x80, 0x00, 0x00, 0x00}); err != nil {
return err
}
_ = conn.SetReadDeadline(time.Now().Add(time.Second))
reply := make([]byte, 64)
n, err := conn.Read(reply)
if err != nil {
return err
}
if n < 9 || reply[8] != 0x40 {
return fmt.Errorf("unexpected rmcp reply")
}
return nil
}
@@ -46,7 +46,6 @@ var jsTestcases = []integrationCase{
{Path: "protocols/javascript/http-get.yaml", TestCase: &javascriptHTTPGet{}},
{Path: "protocols/javascript/http-client-flow.yaml", TestCase: &javascriptHTTPClientFlow{}},
{Path: "protocols/javascript/http-denied.yaml", TestCase: &javascriptHTTPDenied{}},
{Path: "protocols/javascript/ipmi.yaml", TestCase: &javascriptIPMI{}, DisableOn: javascriptDockerDisabled, Serial: true},
{Path: "protocols/javascript/rsync-test.yaml", TestCase: &javascriptRsyncTest{}, DisableOn: javascriptDockerDisabled, Serial: true},
{Path: "protocols/javascript/vnc-pass-brute.yaml", TestCase: &javascriptVncPassBrute{}, DisableOn: javascriptDockerDisabled, Serial: true},
{Path: "protocols/javascript/postgres-pass-brute.yaml", TestCase: &javascriptPostgresPassBrute{}, DisableOn: javascriptDockerDisabled, Serial: true},
@@ -1,46 +0,0 @@
id: ipmi-checks
info:
name: IPMI Checks
author: pdteam
severity: info
javascript:
- code: |
const ipmi = require('nuclei/ipmi');
const caps = ipmi.GetAuthCapabilities(Host, Port);
const root = ipmi.GetRAKPHash(Host, Port, 'root');
// cipher suite 0 lets a wrong password in
const anyPassword = new ipmi.Client();
const cipherZero = anyPassword.Login(Host, Port, 'ADMIN', 'wrong', 0);
anyPassword.Close();
// an encrypted, signed session, then Get Device ID and admin privilege
const client = new ipmi.Client();
client.Login(Host, Port, 'ADMIN', 'ADMIN', 3);
const device = client.Command(0x06, 0x01, []);
const admin = client.Command(0x06, 0x3b, [0x04]);
client.Close();
const manufacturer = device.Data[6] | device.Data[7] << 8 | device.Data[8] << 16;
let wrongRejected = false;
try { new ipmi.Client().Login(Host, Port, 'ADMIN', 'wrong', 3); } catch (e) { wrongRejected = true; }
ipmi.Ping(Host, Port) && caps.IPMI20 && caps.AnonymousLogin &&
ipmi.GetCipherSuites(Host, Port).includes(0) &&
ipmi.GetSystemGUID(Host, Port).length == 32 &&
ipmi.FindRAKPPassword(root, ['admin', 'calvin']) == 'calvin' &&
!ipmi.CheckRAKPPassword(root, 'admin') &&
cipherZero && wrongRejected &&
device.CompletionCode == 0 && manufacturer == 4753 &&
admin.CompletionCode == 0;
args:
Host: "{{Host}}"
Port: "623"
matchers:
- type: dsl
dsl:
- "success == true"
+6 -8
View File
@@ -39,14 +39,12 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
Catalog: base.catalog,
IssuesClient: base.rc,
RateLimiter: base.rateLimiter,
RequestBodyBudget: base.executerOpts.RequestBodyBudget,
Interactsh: base.interactshClient,
Colorizer: aurora.New(aurora.WithColors(true)),
ResumeCfg: types.NewResumeCfg(),
Parser: base.parser,
Browser: base.browserInstance,
DoNotCache: opts.DoNotCacheTemplates,
Interactsh: base.interactshClient,
Colorizer: aurora.New(aurora.WithColors(true)),
ResumeCfg: types.NewResumeCfg(),
Parser: base.parser,
Browser: base.browserInstance,
DoNotCache: opts.DoNotCacheTemplates,
}
if base.interactshClient != nil {
u.executerOpts.InteractshScope = base.interactshClient.NewRequestScope()
-2
View File
@@ -36,7 +36,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
nucleiUtils "github.com/projectdiscovery/nuclei/v3/pkg/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
"github.com/projectdiscovery/ratelimit"
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
)
@@ -280,7 +279,6 @@ func (e *NucleiEngine) init(ctx context.Context) error {
Catalog: e.catalog,
IssuesClient: e.rc,
RateLimiter: e.rateLimiter,
RequestBodyBudget: httpbody.NewBudget(int64(e.opts.MaxRequestBodyInFlight)),
Interactsh: e.interactshClient,
Colorizer: aurora.New(aurora.WithColors(true)),
ResumeCfg: types.NewResumeCfg(),
+4 -12
View File
@@ -7,22 +7,18 @@ import (
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/alecthomas/chroma/v2/quick"
"github.com/alecthomas/chroma/quick"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/retryablehttp-go"
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
"github.com/projectdiscovery/utils/errkit"
)
var (
const (
aiTemplateGeneratorAPIEndpoint = "https://api.projectdiscovery.io/v1/template/ai"
// aiTemplateHTTPClient verifies TLS, since the request carries the PDCP API key.
aiTemplateHTTPClient = &http.Client{}
// aiTemplateIDRegex keeps the template file name inside the pdcp directory.
aiTemplateIDRegex = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
)
type AITemplateResponse struct {
@@ -116,7 +112,7 @@ func generateAITemplate(prompt string) (string, string, error) {
req.Header.Set("Content-Type", "application/json")
req.Header.Set(pdcpauth.ApiKeyHeaderName, creds.APIKey)
resp, err := aiTemplateHTTPClient.Do(req)
resp, err := retryablehttp.DefaultClient().Do(req)
if err != nil {
return "", "", errkit.Newf("Failed to send HTTP request: %v", err)
}
@@ -142,9 +138,5 @@ func generateAITemplate(prompt string) (string, string, error) {
return "", "", errkit.Newf("Failed to generate template")
}
if !aiTemplateIDRegex.MatchString(result.TemplateID) {
return "", "", errkit.Newf("Invalid template ID in API response: %q", result.TemplateID)
}
return result.Completion, result.TemplateID, nil
}
-55
View File
@@ -1,55 +0,0 @@
package loader
import (
"encoding/json"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/stretchr/testify/require"
)
func newAITemplateServer(t *testing.T, templateID string, gotKey *atomic.Value) {
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotKey.Store(r.Header.Get("X-Api-Key"))
_ = json.NewEncoder(w).Encode(map[string]string{
"completion": "id: test\n",
"template_id": templateID,
})
}))
t.Cleanup(srv.Close)
oldEndpoint, oldClient := aiTemplateGeneratorAPIEndpoint, aiTemplateHTTPClient
t.Cleanup(func() { aiTemplateGeneratorAPIEndpoint, aiTemplateHTTPClient = oldEndpoint, oldClient })
aiTemplateGeneratorAPIEndpoint = srv.URL
t.Setenv("PDCP_API_KEY", "test-key")
if templateID != "" {
// Trust the test certificate, to check the template_id only.
aiTemplateHTTPClient = srv.Client()
}
}
func TestAITemplateVerifiesTLS(t *testing.T) {
var gotKey atomic.Value
newAITemplateServer(t, "", &gotKey)
_, _, err := generateAITemplate("check for exposed admin panels")
require.ErrorContains(t, err, "certificate")
require.Nil(t, gotKey.Load(), "the API key went to an untrusted server")
}
func TestAITemplateRejectsUnsafeID(t *testing.T) {
for _, id := range []string{"../../.config/nuclei/config", "a/b", `a\b`, "a.b", "a b"} {
var gotKey atomic.Value
newAITemplateServer(t, id, &gotKey)
_, _, err := generateAITemplate("check for exposed admin panels")
require.Error(t, err, id)
}
var gotKey atomic.Value
newAITemplateServer(t, "Abc-123_x", &gotKey)
_, templateID, err := generateAITemplate("check for exposed admin panels")
require.NoError(t, err)
require.Equal(t, "Abc-123_x", templateID)
}
+7 -47
View File
@@ -3,7 +3,6 @@ package core
import (
"fmt"
"net/http/cookiejar"
"sync"
"sync/atomic"
"github.com/projectdiscovery/gologger"
@@ -33,45 +32,22 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
templateThreads++
}
swg, _ := syncutil.New(syncutil.WithSize(templateThreads))
// pending tracks every started step until it returns. swg only limits
// concurrency and its slots can be released early, so completion is
// tracked here instead.
var pending sync.WaitGroup
for _, template := range w.Workflows {
newCtx := scan.NewScanContext(ctx.Context(), ctx.Input.Clone())
// Root steps do not own a waitgroup slot.
if err := e.runWorkflowStep(template, newCtx, results, swg, &pending, w, nil); err != nil {
if err := e.runWorkflowStep(template, newCtx, results, swg, w); err != nil {
gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
}
}
pending.Wait()
swg.Wait()
return results.Load()
}
// slotRelease returns a func that gives back one swg slot. It is safe to call
// from several goroutines and several times; only the first call releases.
func slotRelease(swg *syncutil.AdaptiveWaitGroup) func() {
var once sync.Once
return func() {
once.Do(swg.Done)
}
}
// runWorkflowStep runs a workflow step for the workflow. It executes the workflow
// in a recursive manner running all subtemplates and matchers.
//
// releaseSlot, when non-nil, releases the AdaptiveWaitGroup slot held by the
// goroutine that is running this step. It must be called before this step
// blocks on swg.AddWithContext for child steps; otherwise every in-flight
// parent can hold a slot while waiting for another, which deadlocks when the
// number of matching nested steps reaches TemplateThreads (see #8074).
//
// pending is incremented for every child step before it starts, and each child
// calls Done when it returns, so the root can wait for the whole tree.
func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *syncutil.AdaptiveWaitGroup, pending *sync.WaitGroup, w *workflows.Workflow, releaseSlot func()) error {
func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *syncutil.AdaptiveWaitGroup, w *workflows.Workflow) error {
var firstMatched bool
var err error
var mainErr error
@@ -155,25 +131,18 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
continue
}
// Free this step's slot before blocking on child acquires.
if releaseSlot != nil {
releaseSlot()
}
for _, subtemplate := range matcher.Subtemplates {
if err := swg.AddWithContext(ctx.Context()); err != nil {
return
}
pending.Add(1)
go func(subtemplate *workflows.WorkflowTemplate) {
defer pending.Done()
release := slotRelease(swg)
defer release()
defer swg.Done()
// create a new context with the same input but with unset callbacks
// clone the Input so that other parallel executions won't overwrite the shared variables when subsequent templates are running
subCtx := scan.NewScanContext(ctx.Context(), ctx.Input.Clone())
if err := e.runWorkflowStep(subtemplate, subCtx, results, swg, pending, w, release); err != nil {
if err := e.runWorkflowStep(subtemplate, subCtx, results, swg, w); err != nil {
gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err)
}
}(subtemplate)
@@ -193,27 +162,18 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
return mainErr
}
if len(template.Subtemplates) > 0 && firstMatched {
// Free this step's slot before blocking on child acquires so nested
// matching steps cannot exhaust TemplateThreads and deadlock (#8074).
if releaseSlot != nil {
releaseSlot()
}
for _, subtemplate := range template.Subtemplates {
if err := swg.AddWithContext(ctx.Context()); err != nil {
return err
}
pending.Add(1)
go func(template *workflows.WorkflowTemplate) {
defer pending.Done()
release := slotRelease(swg)
defer release()
// create a new context with the same input but with unset callbacks
subCtx := scan.NewScanContext(ctx.Context(), ctx.Input.Clone())
if err := e.runWorkflowStep(template, subCtx, results, swg, pending, w, release); err != nil {
if err := e.runWorkflowStep(template, subCtx, results, swg, w); err != nil {
gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
}
swg.Done()
}(subtemplate)
}
}
-83
View File
@@ -303,89 +303,6 @@ func TestWorkflowsSameStepExecutersRefreshTemplateContext(t *testing.T) {
require.Equal(t, "foo", secondWhoami, "could not refresh workflow context into later executers in the same step")
}
func TestWorkflowNestedSubtemplatesDoNotDeadlockAtConcurrencyLimit(t *testing.T) {
// Repro for #8074: with TemplateThreads equal to the number of matching
// nested parents, each parent used to hold a waitgroup slot while calling
// AddWithContext for its children, so every slot was occupied by a parent
// waiting for another slot and the workflow never finished.
progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
matchOutputs := []*output.InternalWrappedEvent{{
OperatorsResult: &operators.Result{},
Results: []*output.ResultEvent{{}},
}}
const parents = 3
var entered atomic.Int32
allHolding := make(chan struct{})
releaseParents := make(chan struct{})
leafRuns := &atomic.Int32{}
leaf := &mockExecuter{result: true, executeScanHook: func(ctx *scan.ScanContext) {
leafRuns.Add(1)
}}
parentExec := &mockExecuter{
result: true,
outputs: matchOutputs,
executeScanHook: func(ctx *scan.ScanContext) {
if entered.Add(1) == parents {
close(allHolding)
}
// Stay inside ExecuteWithResults so the parent still owns its
// waitgroup slot while every peer also holds one.
<-releaseParents
},
}
mkParent := func() *workflows.WorkflowTemplate {
return &workflows.WorkflowTemplate{
Executers: []*workflows.ProtocolExecuterPair{{
Executer: parentExec,
Options: &protocols.ExecutorOptions{Progress: progressBar},
}},
Subtemplates: []*workflows.WorkflowTemplate{{
Executers: []*workflows.ProtocolExecuterPair{{
Executer: leaf,
Options: &protocols.ExecutorOptions{Progress: progressBar},
}},
}},
}
}
workflow := &workflows.Workflow{
Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: parents}},
Workflows: []*workflows.WorkflowTemplate{{
Executers: []*workflows.ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, outputs: matchOutputs},
Options: &protocols.ExecutorOptions{Progress: progressBar},
}},
Subtemplates: []*workflows.WorkflowTemplate{mkParent(), mkParent(), mkParent()},
}},
}
engine := &Engine{}
input := contextargs.NewWithInput(context.Background(), "https://test.com")
ctx := scan.NewScanContext(context.Background(), input)
done := make(chan bool, 1)
go func() {
done <- engine.executeWorkflow(ctx, workflow)
}()
select {
case <-allHolding:
case <-time.After(2 * time.Second):
t.Fatal("nested parents did not all acquire TemplateThreads slots")
}
close(releaseParents)
select {
case matched := <-done:
require.True(t, matched, "nested workflow should report a match")
require.EqualValues(t, parents, leafRuns.Load(), "each nested parent should run its leaf")
case <-time.After(3 * time.Second):
t.Fatal("nested workflow deadlocked at TemplateThreads == matching nested parents")
}
}
func TestWorkflowSubtemplatesUnblockCancelledAcquire(t *testing.T) {
progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
var started atomic.Int32
+18 -14
View File
@@ -155,14 +155,23 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) {
baseInteractURLs := append([]string(nil), input.InteractURLs...)
if rule.generator == nil {
for _, component := range finalComponentList {
var dataKeys map[string]struct{}
optionVars := rule.options.Options.Vars.AsMap()
constants := rule.options.Constants
dataValues := rule.options.NewVariablesScope(baseValues, optionVars, constants)
// Render variables once, allocating their interactsh URLs. The
// output may contain runtime data, so it is never rendered again.
// get vars from variables while replacing interactsh urls
evaluation := rule.options.Variables.EvaluateWithInteractshScope(dataValues, rule.options.Interactsh)
input.Values = generators.MergeMaps(evaluation.Values, baseValues, optionVars, constants)
input.InteractURLs = mergeInteractURLs(baseInteractURLs, evaluation.InteractURLs)
evaluatedValues, interactURLs := evaluation.Values, evaluation.InteractURLs
input.Values, dataKeys = mergeFuzzValueLayers(
fuzzValueLayer{values: evaluatedValues, kind: fuzzValueTemplate},
fuzzValueLayer{values: baseValues, kind: fuzzValueData},
fuzzValueLayer{values: optionVars, kind: fuzzValueData},
fuzzValueLayer{values: constants, kind: fuzzValueData},
)
// Replace template-text interactsh markers without evaluating runtime values.
input.Values, interactURLs = rule.evaluateVarsWithInteractsh(input.Values, interactURLs, dataKeys)
input.InteractURLs = mergeInteractURLs(baseInteractURLs, interactURLs)
err := rule.executeRuleValues(input, component)
if err != nil {
@@ -231,17 +240,15 @@ func (rule *Rule) prepareGeneratorValues(values, baseValues map[string]interface
evaluation := rule.options.Variables.EvaluateWithInteractshScope(evaluationValues, rule.options.Interactsh)
evaluatedValues, interactURLs := evaluation.Values, evaluation.InteractURLs
inputValues, dataKeys := mergeFuzzValueLayers(
fuzzValueLayer{values: evaluatedValues, kind: fuzzValueData},
fuzzValueLayer{values: values, kind: fuzzValueTemplate},
fuzzValueLayer{values: evaluatedValues, kind: fuzzValueTemplate},
fuzzValueLayer{values: baseValues, kind: fuzzValueData},
fuzzValueLayer{values: optionVars, kind: fuzzValueData},
fuzzValueLayer{values: constants, kind: fuzzValueData},
)
// Generator values are template-authored payload definitions until this
// render. Rendered variables, runtime/base, CLI option, and constant
// values remain data. Payloads layer over variables so a payload that
// shares a variable's name is still rendered.
// render. Runtime/base, CLI option, and constant values remain data.
return rule.evaluateVarsWithInteractsh(inputValues, interactURLs, dataKeys)
}
@@ -319,7 +326,7 @@ func (rule *Rule) evaluateVars(input string) (string, error) {
func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, interactshUrls []string, dataKeys map[string]struct{}) (map[string]interface{}, []string) {
// Check if Interactsh options are configured
if rule.options.Interactsh != nil {
rendered := maps.Clone(data)
data = maps.Clone(data)
interactshUrlsMap := make(map[string]struct{})
for _, url := range interactshUrls {
@@ -328,8 +335,7 @@ func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, intera
interactshUrls = mapsutil.GetKeys(interactshUrlsMap)
// Render each template-text value against the unrendered input, so
// the result does not depend on map iteration order.
// Iterate through template-text data to replace Interactsh URL markers.
for k, v := range data {
if _, ok := dataKeys[k]; ok {
continue
@@ -343,7 +349,7 @@ func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, intera
InteractURLs: interactshUrls,
})
if err == nil {
rendered[k] = got.Text
data[k] = got.Text
for _, url := range got.InteractURLs {
if _, ok := interactshUrlsMap[url]; ok {
continue
@@ -354,8 +360,6 @@ func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, intera
}
}
}
data = rendered
}
return data, interactshUrls
-224
View File
@@ -1,224 +0,0 @@
package fuzz
import (
"context"
"fmt"
"io"
"net/http"
"strings"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/component"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/stretchr/testify/require"
)
const (
// requestBodyWithDSL is request data that looks like a DSL expression.
requestBodyWithDSL = `PLAIN-{{to_upper("abc")}}-END`
queryRuleTarget = "http://127.0.0.1/echo?p=SEED"
)
// fakeInteractshClient hands out numbered URLs without contacting a server.
// Fuzz rules only allocate URLs through NewURLWithData, so the embedded nil
// IClient is never called.
type fakeInteractshClient struct {
interactsh.IClient
issued int
}
func (c *fakeInteractshClient) NewURLWithData(string) (string, error) {
c.issued++
return fmt.Sprintf("oast%d.example", c.issued), nil
}
// newQueryRule compiles a single-mode rule that replaces the p query value
// with fuzzValue.
func newQueryRule(
t *testing.T,
fuzzValue string,
generator *generators.PayloadGenerator,
templateVars variables.Variable,
client interactsh.IClient,
) *Rule {
t.Helper()
rule := &Rule{
Part: component.RequestQueryComponent,
Type: "replace",
Mode: "single",
Fuzz: SliceOrMapSlice{Value: []string{fuzzValue}},
}
err := rule.Compile(generator, &protocols.ExecutorOptions{
Variables: templateVars,
Options: &types.Options{},
Interactsh: client,
})
require.NoError(t, err)
return rule
}
// executeQueryRule runs rule against queryRuleTarget and returns the generated
// requests.
func executeQueryRule(
t *testing.T,
rule *Rule,
method string,
body io.Reader,
values map[string]any,
) []GeneratedRequest {
t.Helper()
req, err := retryablehttp.NewRequest(method, queryRuleTarget, body)
require.NoError(t, err)
var got []GeneratedRequest
err = rule.Execute(&ExecuteRuleInput{
Input: contextargs.NewWithInput(context.Background(), queryRuleTarget),
BaseRequest: req,
Values: values,
Callback: func(gr GeneratedRequest) bool {
got = append(got, gr)
return true
},
})
require.NoError(t, err)
return got
}
// newRequestDataVariableRule returns a query rule whose template declares
// b: "{{body}}", so b captures request data.
func newRequestDataVariableRule(t *testing.T, fuzzValue string, payloads map[string]any) *Rule {
t.Helper()
templateVars := newFuzzTestVariables()
templateVars.Set("b", "{{body}}")
var generator *generators.PayloadGenerator
if payloads != nil {
var err error
generator, err = generators.New(
payloads, generators.BatteringRamAttack, "", disk.NewCatalog(""), "", types.DefaultOptions(),
)
require.NoError(t, err)
}
// The CLI creates an Interactsh client even with -no-interactsh.
return newQueryRule(t, fuzzValue, generator, templateVars, &interactsh.Client{})
}
// fuzzedQueryValues runs rule against a POST request whose body is
// requestBodyWithDSL and returns the fuzzed p query value of each generated
// request.
func fuzzedQueryValues(t *testing.T, rule *Rule) []string {
t.Helper()
generated := executeQueryRule(t, rule, http.MethodPost, strings.NewReader(requestBodyWithDSL), map[string]any{
"body": requestBodyWithDSL,
})
got := make([]string, 0, len(generated))
for _, gr := range generated {
got = append(got, gr.Request.URL.Query().Get("p"))
}
return got
}
func TestExecuteKeepsRequestDataInTemplateVariableLiteral(t *testing.T) {
t.Parallel()
t.Run("fuzz value", func(t *testing.T) {
t.Parallel()
rule := newRequestDataVariableRule(t, "{{b}}", nil)
require.Equal(t, []string{requestBodyWithDSL}, fuzzedQueryValues(t, rule))
})
t.Run("payload value", func(t *testing.T) {
t.Parallel()
rule := newRequestDataVariableRule(t, "{{inj}}", map[string]any{
"inj": []any{"{{b}}"},
})
// Each run iterates a fresh value map. Repeat so the result
// cannot depend on map iteration order.
for range 256 {
require.Equal(t, []string{requestBodyWithDSL}, fuzzedQueryValues(t, rule))
}
})
}
func TestExecuteRendersTemplateVariableInteractshMarker(t *testing.T) {
t.Parallel()
templateVars := newFuzzTestVariables()
templateVars.Set("marker", "{{interactsh-url}}")
rule := newQueryRule(t, "{{marker}}", nil, templateVars, &fakeInteractshClient{})
got := executeQueryRule(t, rule, http.MethodGet, nil, nil)
require.Len(t, got, 1)
require.Equal(t, []string{"oast1.example"}, got[0].InteractURLs)
require.Equal(t, "oast1.example", got[0].Request.URL.Query().Get("p"))
}
func TestPrepareGeneratorValuesRendersPayloadSharingVariableName(t *testing.T) {
t.Parallel()
templateVars := newFuzzTestVariables()
templateVars.Set("inj", "variable")
rule := &Rule{
options: &protocols.ExecutorOptions{
Variables: templateVars,
Options: &types.Options{},
Interactsh: &fakeInteractshClient{},
},
}
values, _ := rule.prepareGeneratorValues(map[string]any{
"inj": `{{to_upper("payload")}}`,
}, nil)
require.Equal(t, "PAYLOAD", values["inj"])
}
func TestPrepareGeneratorValuesRendersPayloadsAgainstUnrenderedInput(t *testing.T) {
t.Parallel()
rule := &Rule{
options: &protocols.ExecutorOptions{
Variables: newFuzzTestVariables(),
Options: &types.Options{},
Interactsh: &interactsh.Client{},
},
}
for range 256 {
values, _ := rule.prepareGeneratorValues(map[string]any{
"upper": `{{to_upper("x")}}`,
"ref": "{{upper}}",
}, nil)
require.Equal(t, "X", values["upper"])
// ref sees the unrendered definition of upper as data.
require.Equal(t, `{{to_upper("x")}}`, values["ref"])
}
}
-53
View File
@@ -1,53 +0,0 @@
package compiler_test
//nolint:depguard // No depguard rules are configured, so its default list allows only the standard library.
import (
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
)
// TestCloseWithoutConnection calls Close on clients that never connected.
// The script must keep running past the call.
func TestCloseWithoutConnection(t *testing.T) {
t.Parallel()
scripts := map[string]string{
"ssh": `
const ssh = require('nuclei/ssh');
ExportAs('closed', new ssh.SSHClient().Close());
true;
`,
"net": `
const net = require('nuclei/net');
new net.NetConn().Close();
ExportAs('closed', true);
true;
`,
}
for name, script := range scripts {
t.Run(name, func(t *testing.T) {
t.Parallel()
program, err := compiler.SourceAutoMode(script, false)
if err != nil {
t.Fatal(err)
}
opts := new(compiler.ExecuteOptions)
opts.ExecutionId = t.Name()
opts.Source = &script // ExportAs is defined only for sources that use it.
opts.TimeoutVariants = types.NewTimeoutVariant(5)
result, err := compiler.New().ExecuteWithOptions(t.Context(), program, compiler.NewExecuteArgs(), opts)
if err != nil {
t.Fatal(err)
}
if got := result["closed"]; got != "true" {
t.Errorf(`result["closed"] = %v, want "true"`, got)
}
})
}
}
-2
View File
@@ -24,14 +24,12 @@ import (
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libhash"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libhttp"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libikev2"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libipmi"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libkerberos"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libkrbforge"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libkrbroast"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libldap"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libmssql"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libmysql"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libmqtt"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libnet"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/liboracle"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libpop3"
-39
View File
@@ -1,39 +0,0 @@
package ipmi
import (
lib_ipmi "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/ipmi"
"github.com/projectdiscovery/goja"
"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
)
var (
module = gojs.NewGojaModule("nuclei/ipmi")
)
func init() {
module.Set(
gojs.Objects{
// Functions
"CheckRAKPPassword": lib_ipmi.CheckRAKPPassword,
"FindRAKPPassword": lib_ipmi.FindRAKPPassword,
"GetAuthCapabilities": lib_ipmi.GetAuthCapabilities,
"GetCipherSuites": lib_ipmi.GetCipherSuites,
"GetRAKPHash": lib_ipmi.GetRAKPHash,
"GetSystemGUID": lib_ipmi.GetSystemGUID,
"Ping": lib_ipmi.Ping,
// Var and consts
// Objects / Classes
"AuthCapabilities": gojs.GetClassConstructor[lib_ipmi.AuthCapabilities](&lib_ipmi.AuthCapabilities{}),
"Client": gojs.GetClassConstructor[lib_ipmi.Client](&lib_ipmi.Client{}),
"CommandResponse": gojs.GetClassConstructor[lib_ipmi.CommandResponse](&lib_ipmi.CommandResponse{}),
"RAKPHash": gojs.GetClassConstructor[lib_ipmi.RAKPHash](&lib_ipmi.RAKPHash{}),
},
).Register()
}
func Enable(runtime *goja.Runtime) {
module.Enable(runtime)
}
-34
View File
@@ -1,34 +0,0 @@
package mqtt
import (
lib_mqtt "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/mqtt"
"github.com/projectdiscovery/goja"
"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
)
var (
module = gojs.NewGojaModule("nuclei/mqtt")
)
func init() {
module.Set(
gojs.Objects{
// Functions
"IsMQTT": lib_mqtt.IsMQTT,
"IsAnonymousAllowed": lib_mqtt.IsAnonymousAllowed,
// Var and consts
// Objects / Classes
"Client": gojs.GetClassConstructor[lib_mqtt.Client](&lib_mqtt.Client{}),
"ConnectOptions": gojs.GetClassConstructor[lib_mqtt.ConnectOptions](&lib_mqtt.ConnectOptions{}),
"ConnectResult": gojs.GetClassConstructor[lib_mqtt.ConnectResult](&lib_mqtt.ConnectResult{}),
"Message": gojs.GetClassConstructor[lib_mqtt.Message](&lib_mqtt.Message{}),
},
).Register()
}
func Enable(runtime *goja.Runtime) {
module.Enable(runtime)
}
-2
View File
@@ -8,14 +8,12 @@ export * as grpc from './grpc';
export * as hash from './hash';
export * as http from './http';
export * as ikev2 from './ikev2';
export * as ipmi from './ipmi';
export * as kerberos from './kerberos';
export * as krbforge from './krbforge';
export * as krbroast from './krbroast';
export * as ldap from './ldap';
export * as mssql from './mssql';
export * as mysql from './mysql';
export * as mqtt from './mqtt';
export * as net from './net';
export * as oracle from './oracle';
export * as pop3 from './pop3';
-304
View File
@@ -1,304 +0,0 @@
/**
* CheckRAKPPassword reports whether password produced hash. It runs offline,
* so it can test any number of passwords without touching the BMC.
* @example
* ```javascript
* const ipmi = require('nuclei/ipmi');
* const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
* log(ipmi.CheckRAKPPassword(hash, 'ADMIN'));
* ```
*/
export function CheckRAKPPassword(hash: RAKPHash, password: string): boolean {
return false;
}
/**
* FindRAKPPassword returns the first of passwords that produced hash, or an
* empty string.
* @example
* ```javascript
* const ipmi = require('nuclei/ipmi');
* const hash = ipmi.GetRAKPHash('acme.com', 623, 'root');
* log(ipmi.FindRAKPPassword(hash, ['calvin', 'root', 'admin']));
* ```
*/
export function FindRAKPPassword(hash: RAKPHash, passwords: string[]): string {
return "";
}
/**
* GetAuthCapabilities returns the authentication capabilities the BMC
* advertises on the current channel.
* @example
* ```javascript
* const ipmi = require('nuclei/ipmi');
* const caps = ipmi.GetAuthCapabilities('acme.com', 623);
* log(caps.AnonymousLogin);
* ```
*/
export function GetAuthCapabilities(ctx: any, host: string, port: number): AuthCapabilities | null {
return null;
}
/**
* GetCipherSuites returns the IDs of the RMCP+ cipher suites the BMC
* offers. Cipher suite 0 authenticates nothing, so any password logs in.
* @example
* ```javascript
* const ipmi = require('nuclei/ipmi');
* const suites = ipmi.GetCipherSuites('acme.com', 623);
* log(suites.includes(0));
* ```
*/
export function GetCipherSuites(ctx: any, host: string, port: number): number[] | null {
return null;
}
/**
* GetRAKPHash starts an RMCP+ session as username and returns the HMAC the
* BMC answers with. The session is never completed, so no login is
* attempted; the BMC drops the half-open session after its timeout. It fails
* when the BMC rejects the username.
* @example
* ```javascript
* const ipmi = require('nuclei/ipmi');
* const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
* log(ipmi.CheckRAKPPassword(hash, 'ADMIN'));
* ```
*/
export function GetRAKPHash(ctx: any, host: string, port: number, username: string): RAKPHash | null {
return null;
}
/**
* GetSystemGUID returns the BMC system GUID, which vendors derive from
* serial numbers or MAC addresses.
* @example
* ```javascript
* const ipmi = require('nuclei/ipmi');
* const guid = ipmi.GetSystemGUID('acme.com', 623);
* log(guid);
* ```
*/
export function GetSystemGUID(ctx: any, host: string, port: number): string | null {
return null;
}
/**
* Ping reports whether the host answers an RMCP presence ping and advertises
* IPMI support.
* @example
* ```javascript
* const ipmi = require('nuclei/ipmi');
* const isIPMI = ipmi.Ping('acme.com', 623);
* log(isIPMI);
* ```
*/
export function Ping(ctx: any, host: string, port: number): boolean | null {
return null;
}
/**
* Client holds one RMCP+ session, for commands that need a login.
* Close it when done: BMCs allow only a few sessions at a time.
* @example
* ```javascript
* const ipmi = require('nuclei/ipmi');
* const client = new ipmi.Client();
* client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
* const id = client.Command(0x06, 0x01, []);
* client.Close();
* ```
*/
export class Client {
// Constructor of Client
constructor() {}
/**
* Login opens an RMCP+ session as username with the given cipher suite
* (0, 1, 2, 3 or 6). Cipher suite 0 skips authentication, so a BMC
* that allows it accepts any password. A rejected login is an error.
* @example
* ```javascript
* const ipmi = require('nuclei/ipmi');
* const client = new ipmi.Client();
* client.Login('acme.com', 623, 'ADMIN', 'not-the-password', 0);
* ```
*/
public Login(ctx: any, host: string, port: number, username: string, password: string, cipherSuite: number): boolean | null {
return null;
}
/**
* Command sends an IPMI request in the session, such as Get Device ID
* (netFn 0x06, cmd 0x01) or Set Session Privilege Level (0x06, 0x3b).
* @example
* ```javascript
* const ipmi = require('nuclei/ipmi');
* const client = new ipmi.Client();
* client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
* const admin = client.Command(0x06, 0x3b, [0x04]).CompletionCode == 0;
* client.Close();
* ```
*/
public Command(netFn: number, cmd: number, data: Uint8Array): CommandResponse | null {
return null;
}
/**
* Close ends the session on the BMC, freeing its slot.
* @example
* ```javascript
* const ipmi = require('nuclei/ipmi');
* const client = new ipmi.Client();
* client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
* client.Close();
* ```
*/
public Close(): void {
return;
}
}
/**
* AuthCapabilities is what a BMC advertises before any login, from Get
* Channel Authentication Capabilities.
* @example
* ```javascript
* const ipmi = require('nuclei/ipmi');
* const caps = ipmi.GetAuthCapabilities('acme.com', 623);
* log(toJSON(caps));
* ```
*/
export interface AuthCapabilities {
/**
* IPMI20 is true when the BMC supports IPMI 2.0 (RMCP+).
*/
IPMI20?: boolean,
/**
* AuthTypes are the IPMI 1.5 authentication types the channel
* accepts: none, md2, md5, straight (cleartext password), oem.
*/
AuthTypes?: string[],
/**
* AnonymousLogin is true when a null username with a null password
* is accepted.
*/
AnonymousLogin?: boolean,
/**
* NullUsernames is true when accounts with an empty name are enabled.
*/
NullUsernames?: boolean,
/**
* NonNullUsernames is true when named accounts are enabled.
*/
NonNullUsernames?: boolean,
/**
* PerMessageAuth is false when the BMC skips authenticating
* messages after login.
*/
PerMessageAuth?: boolean,
/**
* UserLevelAuth is false when user-level commands skip
* authentication.
*/
UserLevelAuth?: boolean,
/**
* OEMID is the IANA enterprise number of the BMC vendor, or 0.
*/
OEMID?: number,
}
/**
* CommandResponse is the reply to an IPMI command.
*/
export interface CommandResponse {
/**
* CompletionCode is 0 on success.
*/
CompletionCode?: number,
Data?: Uint8Array,
}
/**
* RAKPHash is the HMAC a BMC returns for a user during RMCP+ session
* setup (CVE-2013-4786). It can be checked offline against candidate
* passwords, which tests credentials without a login attempt.
* @example
* ```javascript
* const ipmi = require('nuclei/ipmi');
* const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
* log(hash.Hashcat);
* ```
*/
export interface RAKPHash {
Username?: string,
/**
* Algorithm is the HMAC the BMC used: sha1 or md5.
*/
Algorithm?: string,
/**
* Salt and HMAC are hex encoded; Salt is everything the HMAC covers.
*/
Salt?: string,
HMAC?: string,
/**
* Hashcat is the hash in hashcat mode 7300 format, for sha1 only.
*/
Hashcat?: string,
}
-190
View File
@@ -1,190 +0,0 @@
/**
* IsMQTT checks whether the target host:port is running an MQTT broker.
* @example
* ```javascript
* const mqtt = require('nuclei/mqtt');
* const isMqtt = mqtt.IsMQTT('acme.com', 1883);
* log(isMqtt);
* ```
*/
export function IsMQTT(host: string, port: number): boolean | null {
return null;
}
/**
* IsAnonymousAllowed checks whether an MQTT broker permits unauthenticated / anonymous login.
* @example
* ```javascript
* const mqtt = require('nuclei/mqtt');
* const allowed = mqtt.IsAnonymousAllowed('acme.com', 1883);
* log(allowed);
* ```
*/
export function IsAnonymousAllowed(host: string, port: number): boolean | null {
return null;
}
/**
* Message represents an MQTT published message.
* @example
* ```javascript
* const mqtt = require('nuclei/mqtt');
* const client = new mqtt.Client();
* client.Connect('broker.hivemq.com', 1883);
* const msgs = client.Subscribe('test/topic', 2);
* log(msgs[0].Topic, msgs[0].Payload);
* ```
*/
export interface Message {
Topic?: string;
Payload?: string;
QoS?: number;
Retain?: boolean;
}
/**
* ConnectOptions contains connection options for an MQTT broker.
* @example
* ```javascript
* const mqtt = require('nuclei/mqtt');
* const opts = new mqtt.ConnectOptions();
* opts.Username = 'admin';
* opts.Password = 'admin';
* opts.CleanSession = true;
* ```
*/
export interface ConnectOptions {
Username?: string;
Password?: string;
ClientID?: string;
CleanSession?: boolean;
KeepAlive?: number;
TLS?: boolean;
InsecureSkipVerify?: boolean;
Timeout?: number;
}
/**
* ConnectResult is the response returned by an MQTT connection attempt.
* @example
* ```javascript
* const mqtt = require('nuclei/mqtt');
* const client = new mqtt.Client();
* const res = client.Connect('acme.com', 1883);
* log(res.Success, res.ReturnCode, res.Reason);
* ```
*/
export interface ConnectResult {
Success?: boolean;
ReturnCode?: number;
Reason?: string;
SessionPresent?: boolean;
}
/**
* Client is an MQTT client for testing MQTT broker misconfigurations and authentication.
* @example
* ```javascript
* const mqtt = require('nuclei/mqtt');
* const client = new mqtt.Client();
* const res = client.Connect('acme.com', 1883);
* if (res.Success) {
* const msgs = client.Subscribe('#', 2);
* client.Publish('nuclei/probe', 'test');
* }
* client.Close();
* ```
*/
export class Client {
constructor() {}
/**
* Connect connects to an MQTT broker using optional credentials and settings.
* @example
* ```javascript
* const mqtt = require('nuclei/mqtt');
* const client = new mqtt.Client();
* const res = client.Connect('acme.com', 1883);
* log(res.Success);
* ```
*/
public Connect(host: string, port: number, options?: ConnectOptions): ConnectResult | null {
return null;
}
/**
* ConnectAuth connects to an MQTT broker with a username and password.
* @example
* ```javascript
* const mqtt = require('nuclei/mqtt');
* const client = new mqtt.Client();
* const res = client.ConnectAuth('acme.com', 1883, 'admin', 'admin');
* log(res.Success);
* ```
*/
public ConnectAuth(host: string, port: number, username: string, password: string): ConnectResult | null {
return null;
}
/**
* Subscribe subscribes to a topic filter and collects received messages up to timeoutSec.
* @example
* ```javascript
* const mqtt = require('nuclei/mqtt');
* const client = new mqtt.Client();
* client.Connect('acme.com', 1883);
* const msgs = client.Subscribe('#', 3);
* for (const m of msgs) {
* log(m.Topic, m.Payload);
* }
* ```
*/
public Subscribe(topic: string, timeoutSec?: number): Message[] | null {
return null;
}
/**
* Publish publishes a message payload to a topic.
* @example
* ```javascript
* const mqtt = require('nuclei/mqtt');
* const client = new mqtt.Client();
* client.Connect('acme.com', 1883);
* const ok = client.Publish('nuclei/test', 'probe payload');
* log(ok);
* ```
*/
public Publish(topic: string, payload: string): boolean | null {
return null;
}
/**
* Ping sends an MQTT PINGREQ and awaits PINGRESP.
* @example
* ```javascript
* const mqtt = require('nuclei/mqtt');
* const client = new mqtt.Client();
* client.Connect('acme.com', 1883);
* const alive = client.Ping();
* log(alive);
* ```
*/
public Ping(): boolean | null {
return null;
}
/**
* Close cleanly disconnects and closes the connection.
* @example
* ```javascript
* const mqtt = require('nuclei/mqtt');
* const client = new mqtt.Client();
* client.Connect('acme.com', 1883);
* client.Close();
* ```
*/
public Close(): error | null {
return null;
}
}
-1
View File
@@ -46,7 +46,6 @@ export class NetConn {
constructor() {}
/**
* Close closes the connection.
* Close does nothing if no connection is open.
* @example
* ```javascript
* const net = require('nuclei/net');
+1 -1
View File
@@ -2,7 +2,7 @@
/**
* PGClient is a client for Postgres database.
* Internally client uses lib/pq driver.
* Internally client uses go-pg/pg driver.
* @example
* ```javascript
* const postgres = require('nuclei/postgres');
+1 -2
View File
@@ -106,8 +106,7 @@ export class SSHClient {
/**
* Close closes the SSH connection and destroys the client
* Returns the success state and error. If error is not nil,
* state will be false. Close does nothing and returns true
* if no connection is open.
* state will be false
* @example
* ```javascript
* const ssh = require('nuclei/ssh');
+2 -6
View File
@@ -128,13 +128,9 @@ func (p *GojaModule) Register() Module {
return p
}
// GetClassConstructor returns a constructor for any given go struct type for goja runtime.
// Each object it constructs starts as a shallow copy of instance, so objects
// do not share state.
// GetClassConstructor returns a constructor for any given go struct type for goja runtime
func GetClassConstructor[T any](instance *T) func(call goja.ConstructorCall, runtime *goja.Runtime) *goja.Object {
return func(call goja.ConstructorCall, runtime *goja.Runtime) *goja.Object {
obj := *instance
return utils.LinkConstructor(call, runtime, &obj)
return utils.LinkConstructor[*T](call, runtime, instance)
}
}
-46
View File
@@ -1,46 +0,0 @@
package gojs_test
//nolint:depguard // No depguard rules are configured, so its default list allows only the standard library.
import (
"testing"
"github.com/projectdiscovery/goja"
"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
)
type counter struct{ n int }
func (c *counter) Inc() int {
c.n++
return c.n
}
func TestGetClassConstructorReturnsNewObjects(t *testing.T) {
t.Parallel()
prototype := new(counter)
runtime := goja.New()
err := runtime.Set("Counter", gojs.GetClassConstructor(prototype))
if err != nil {
t.Fatal(err)
}
value, err := runtime.RunString(`
const first = new Counter();
first.Inc();
new Counter().Inc();
`)
if err != nil {
t.Fatal(err)
}
if got := value.ToInteger(); got != 1 {
t.Errorf("new Counter().Inc() = %d, want 1", got)
}
if prototype.n != 0 {
t.Errorf("prototype.n = %d, want 0", prototype.n)
}
}
-394
View File
@@ -1,394 +0,0 @@
package ipmi
import (
"context"
"crypto/hmac"
"crypto/md5"
"crypto/sha1"
"encoding/hex"
"errors"
"fmt"
)
type (
// AuthCapabilities is what a BMC advertises before any login, from Get
// Channel Authentication Capabilities.
// @example
// ```javascript
// const ipmi = require('nuclei/ipmi');
// const caps = ipmi.GetAuthCapabilities('acme.com', 623);
// log(toJSON(caps));
// ```
AuthCapabilities struct {
// IPMI20 is true when the BMC supports IPMI 2.0 (RMCP+).
IPMI20 bool
// AuthTypes are the IPMI 1.5 authentication types the channel
// accepts: none, md2, md5, straight (cleartext password), oem.
AuthTypes []string
// AnonymousLogin is true when a null username with a null password
// is accepted.
AnonymousLogin bool
// NullUsernames is true when accounts with an empty name are enabled.
NullUsernames bool
// NonNullUsernames is true when named accounts are enabled.
NonNullUsernames bool
// PerMessageAuth is false when the BMC skips authenticating
// messages after login.
PerMessageAuth bool
// UserLevelAuth is false when user-level commands skip
// authentication.
UserLevelAuth bool
// OEMID is the IANA enterprise number of the BMC vendor, or 0.
OEMID int
}
// RAKPHash is the HMAC a BMC returns for a user during RMCP+ session
// setup (CVE-2013-4786). It can be checked offline against candidate
// passwords, which tests credentials without a login attempt.
// @example
// ```javascript
// const ipmi = require('nuclei/ipmi');
// const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
// log(hash.Hashcat);
// ```
RAKPHash struct {
Username string
// Algorithm is the HMAC the BMC used: sha1 or md5.
Algorithm string
// Salt and HMAC are hex encoded; Salt is everything the HMAC covers.
Salt string
HMAC string
// Hashcat is the hash in hashcat mode 7300 format, for sha1 only.
Hashcat string
}
// Client holds one RMCP+ session, for commands that need a login.
// Close it when done: BMCs allow only a few sessions at a time.
// @example
// ```javascript
// const ipmi = require('nuclei/ipmi');
// const client = new ipmi.Client();
// client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
// const id = client.Command(0x06, 0x01, []);
// client.Close();
// ```
Client struct {
session *established
}
// CommandResponse is the reply to an IPMI command.
CommandResponse struct {
// CompletionCode is 0 on success.
CompletionCode int
Data []byte
}
)
var authTypes = []struct {
bit byte
name string
}{{0x01, "none"}, {0x02, "md2"}, {0x04, "md5"}, {0x10, "straight"}, {0x20, "oem"}}
// Ping reports whether the host answers an RMCP presence ping and advertises
// IPMI support.
// @example
// ```javascript
// const ipmi = require('nuclei/ipmi');
// const isIPMI = ipmi.Ping('acme.com', 623);
// log(isIPMI);
// ```
func Ping(ctx context.Context, host string, port int) (bool, error) {
s, err := dial(ctx, host, port)
if err != nil {
return false, err
}
defer s.close()
return s.ping()
}
// GetAuthCapabilities returns the authentication capabilities the BMC
// advertises on the current channel.
// @example
// ```javascript
// const ipmi = require('nuclei/ipmi');
// const caps = ipmi.GetAuthCapabilities('acme.com', 623);
// log(caps.AnonymousLogin);
// ```
func GetAuthCapabilities(ctx context.Context, host string, port int) (*AuthCapabilities, error) {
s, err := dial(ctx, host, port)
if err != nil {
return nil, err
}
defer s.close()
// bit 7 asks for the IPMI 2.0 extended data; an IPMI 1.5 only BMC
// rejects it, so ask again without
data, err := s.command(cmdGetChannelAuthCaps, []byte{0x80 | currentChannel, privAdmin})
if err != nil {
if data, err = s.command(cmdGetChannelAuthCaps, []byte{currentChannel, privAdmin}); err != nil {
return nil, err
}
}
return parseAuthCapabilities(data)
}
// parseAuthCapabilities decodes the Get Channel Authentication Capabilities
// response data (IPMI v2.0 table 22-15).
func parseAuthCapabilities(data []byte) (*AuthCapabilities, error) {
if len(data) < 8 {
return nil, fmt.Errorf("short channel authentication capabilities response")
}
caps := &AuthCapabilities{
IPMI20: data[1]&0x80 != 0 && data[3]&0x02 != 0,
AnonymousLogin: data[2]&0x01 != 0,
NullUsernames: data[2]&0x02 != 0,
NonNullUsernames: data[2]&0x04 != 0,
UserLevelAuth: data[2]&0x08 == 0,
PerMessageAuth: data[2]&0x10 == 0,
OEMID: int(data[4]) | int(data[5])<<8 | int(data[6])<<16,
}
for _, authType := range authTypes {
if data[1]&authType.bit != 0 {
caps.AuthTypes = append(caps.AuthTypes, authType.name)
}
}
return caps, nil
}
// GetSystemGUID returns the BMC system GUID, which vendors derive from
// serial numbers or MAC addresses.
// @example
// ```javascript
// const ipmi = require('nuclei/ipmi');
// const guid = ipmi.GetSystemGUID('acme.com', 623);
// log(guid);
// ```
func GetSystemGUID(ctx context.Context, host string, port int) (string, error) {
s, err := dial(ctx, host, port)
if err != nil {
return "", err
}
defer s.close()
data, err := s.command(cmdGetSystemGUID, nil)
if err != nil {
return "", err
}
if len(data) < 16 {
return "", fmt.Errorf("short system guid response")
}
return hex.EncodeToString(data[:16]), nil
}
// GetCipherSuites returns the IDs of the RMCP+ cipher suites the BMC
// offers. Cipher suite 0 authenticates nothing, so any password logs in.
// @example
// ```javascript
// const ipmi = require('nuclei/ipmi');
// const suites = ipmi.GetCipherSuites('acme.com', 623);
// log(suites.includes(0));
// ```
func GetCipherSuites(ctx context.Context, host string, port int) ([]int, error) {
s, err := dial(ctx, host, port)
if err != nil {
return nil, err
}
defer s.close()
// the records come in pages of 16 bytes; a shorter page is the last
var records []byte
for index := byte(0); index < 64; index++ {
data, err := s.command(cmdGetCipherSuites, []byte{currentChannel, 0x00, 0x80 | index})
if err != nil {
return nil, err
}
if len(data) > 0 {
records = append(records, data[1:]...)
}
if len(data) < 17 {
break
}
}
return parseCipherSuites(records), nil
}
// parseCipherSuites returns the suite IDs in Get Channel Cipher Suites
// record data. A record starts with 0xc0 (standard) or 0xc1 (oem, followed
// by a 3 byte IANA); algorithm bytes are tagged below 0xc0, so they never
// look like a record start.
func parseCipherSuites(records []byte) []int {
suites := []int{}
for i := 0; i < len(records); i++ {
switch records[i] {
case 0xc0:
if i+1 < len(records) {
suites = append(suites, int(records[i+1]))
i++
}
case 0xc1:
if i+1 < len(records) {
suites = append(suites, int(records[i+1]))
i += 4
}
}
}
return suites
}
// GetRAKPHash starts an RMCP+ session as username and returns the HMAC the
// BMC answers with. The session is never completed, so no login is
// attempted; the BMC drops the half-open session after its timeout. It fails
// when the BMC rejects the username.
// @example
// ```javascript
// const ipmi = require('nuclei/ipmi');
// const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
// log(ipmi.CheckRAKPPassword(hash, 'ADMIN'));
// ```
func GetRAKPHash(ctx context.Context, host string, port int, username string) (*RAKPHash, error) {
s, err := dial(ctx, host, port)
if err != nil {
return nil, err
}
defer s.close()
// the session is never completed, so no login is attempted; md5 is for
// BMCs that refuse sha1
var h *handshake
for _, auth := range []byte{authHMACSHA1, authHMACMD5} {
if h, err = s.startSession(algorithms{auth: auth}, username, privAdmin); err == nil {
break
}
var refused *rakpError
if !errors.As(err, &refused) || refused.step != "the session" {
return nil, err
}
}
if err != nil {
return nil, err
}
hash := &RAKPHash{Username: username, Algorithm: "sha1", Salt: hex.EncodeToString(h.rakp2Salt()), HMAC: hex.EncodeToString(h.bmcMAC)}
if h.auth == authHMACMD5 {
hash.Algorithm = "md5"
} else {
hash.Hashcat = hash.Salt + ":" + hash.HMAC
}
return hash, nil
}
// CheckRAKPPassword reports whether password produced hash. It runs offline,
// so it can test any number of passwords without touching the BMC.
// @example
// ```javascript
// const ipmi = require('nuclei/ipmi');
// const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
// log(ipmi.CheckRAKPPassword(hash, 'ADMIN'));
// ```
func CheckRAKPPassword(hash *RAKPHash, password string) bool {
if hash == nil {
return false
}
salt, err := hex.DecodeString(hash.Salt)
if err != nil {
return false
}
want, err := hex.DecodeString(hash.HMAC)
if err != nil {
return false
}
newHash := sha1.New
if hash.Algorithm == "md5" {
newHash = md5.New
}
return hmac.Equal(hmacOf(newHash, []byte(password), salt), want)
}
// FindRAKPPassword returns the first of passwords that produced hash, or an
// empty string.
// @example
// ```javascript
// const ipmi = require('nuclei/ipmi');
// const hash = ipmi.GetRAKPHash('acme.com', 623, 'root');
// log(ipmi.FindRAKPPassword(hash, ['calvin', 'root', 'admin']));
// ```
func FindRAKPPassword(hash *RAKPHash, passwords []string) string {
for _, password := range passwords {
if CheckRAKPPassword(hash, password) {
return password
}
}
return ""
}
// Login opens an RMCP+ session as username with the given cipher suite
// (0, 1, 2, 3 or 6). Cipher suite 0 skips authentication, so a BMC
// that allows it accepts any password. A rejected login is an error.
// @example
// ```javascript
// const ipmi = require('nuclei/ipmi');
// const client = new ipmi.Client();
// client.Login('acme.com', 623, 'ADMIN', 'not-the-password', 0);
// ```
func (c *Client) Login(ctx context.Context, host string, port int, username string, password string, cipherSuite int) (bool, error) {
alg, ok := cipherSuites[cipherSuite]
if !ok {
return false, fmt.Errorf("unsupported cipher suite %d", cipherSuite)
}
c.Close()
s, err := dial(ctx, host, port)
if err != nil {
return false, err
}
h, err := s.startSession(alg, username, privAdmin)
if err != nil {
s.close()
return false, err
}
k, err := s.finishSession(h, password)
if err != nil {
s.close()
return false, err
}
c.session = &established{session: s, algorithms: alg, keys: k, bmcSessionID: h.bmcSessionID, consoleSessionID: h.consoleSessionID}
return true, nil
}
// Command sends an IPMI request in the session, such as Get Device ID
// (netFn 0x06, cmd 0x01) or Set Session Privilege Level (0x06, 0x3b).
// @example
// ```javascript
// const ipmi = require('nuclei/ipmi');
// const client = new ipmi.Client();
// client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
// const admin = client.Command(0x06, 0x3b, [0x04]).CompletionCode == 0;
// client.Close();
// ```
func (c *Client) Command(netFn int, cmd int, data []byte) (*CommandResponse, error) {
if c.session == nil {
return nil, fmt.Errorf("not logged in")
}
code, reply, err := c.session.command(byte(netFn), byte(cmd), data)
if err != nil {
return nil, err
}
return &CommandResponse{CompletionCode: int(code), Data: reply}, nil
}
// Close ends the session on the BMC, freeing its slot.
// @example
// ```javascript
// const ipmi = require('nuclei/ipmi');
// const client = new ipmi.Client();
// client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
// client.Close();
// ```
func (c *Client) Close() {
if c.session == nil {
return
}
_, _, _ = c.session.command(netFnApp, cmdCloseSession, c.session.bmcSessionID)
c.session.close()
c.session = nil
}
-178
View File
@@ -1,178 +0,0 @@
package ipmi
import (
"bytes"
"context"
"crypto/sha1"
"encoding/hex"
"net"
"testing"
"time"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
// The fixtures were captured from OpenIPMI's ipmi_sim with an enabled user
// with an empty name and password, ADMIN/ADMIN and root/calvin.
func mustHex(t *testing.T, s string) []byte {
t.Helper()
data, err := hex.DecodeString(s)
require.NoError(t, err)
return data
}
func TestParseAuthCapabilities(t *testing.T) {
caps, err := parseAuthCapabilities(mustHex(t, "0197050300000000"))
require.NoError(t, err)
require.Equal(t, &AuthCapabilities{
IPMI20: true,
AuthTypes: []string{"none", "md2", "md5", "straight"},
AnonymousLogin: true,
NonNullUsernames: true,
PerMessageAuth: true,
UserLevelAuth: true,
}, caps)
_, err = parseAuthCapabilities(mustHex(t, "0197"))
require.Error(t, err)
}
func TestParseCipherSuites(t *testing.T) {
records := mustHex(t, "c000004080c001014080c002014180c003014181c004014182c005014183c006024080c007024280c008024281c009024282c00a024283c00b024380c00c024381c00d024382c00e024383")
require.Equal(t, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, parseCipherSuites(records))
// an oem record carries a 3 byte IANA before its algorithms
require.Equal(t, []int{1, 7}, parseCipherSuites(mustHex(t, "c0010140c107d70200014080")))
}
func TestRAKPPasswordChecksAreOffline(t *testing.T) {
// hashcat -m 7300 cracks this hash to calvin as well
hash := &RAKPHash{
Username: "root",
Salt: "988200cd0402000069b30c7a5e3deb30504c5b76f7b1840d5e36439998d9e0f59ae49b09119a10b2a123456789abcdefa123456789abcdef1404726f6f74",
HMAC: "23c711e5c7a737c3c0beb9f4680ef31fa6aad398",
}
require.True(t, CheckRAKPPassword(hash, "calvin"))
require.False(t, CheckRAKPPassword(hash, "ADMIN"))
require.Equal(t, "calvin", FindRAKPPassword(hash, []string{"admin", "root", "calvin"}))
require.Empty(t, FindRAKPPassword(hash, []string{"admin"}))
require.False(t, CheckRAKPPassword(nil, "calvin"))
require.False(t, CheckRAKPPassword(&RAKPHash{Salt: "zz", HMAC: "00"}, "calvin"))
}
func TestRequestsHonourNetworkPolicy(t *testing.T) {
execID := "ipmi-deny"
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID, ExcludeTargets: []string{"203.0.113.50"}}))
t.Cleanup(func() { protocolstate.Close(execID) })
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
_, err := GetRAKPHash(ctx, "203.0.113.50", 623, "ADMIN")
require.ErrorContains(t, err, "denied")
require.NotErrorIs(t, err, errNoResponse, "denied before sending, not timed out")
_, err = Ping(ctx, "203.0.113.50", 623)
require.ErrorContains(t, err, "denied")
}
func TestPayloadEncryptionRoundTrips(t *testing.T) {
key := mustHex(t, "000102030405060708090a0b0c0d0e0f")
for size := range 40 {
payload := make([]byte, size)
for i := range payload {
payload[i] = byte(i)
}
sealed, err := encryptPayload(key, payload)
require.NoError(t, err)
require.Zero(t, len(sealed)%16, "iv and padded blocks")
opened, err := decryptPayload(key, sealed)
require.NoError(t, err)
require.Equal(t, payload, opened)
}
_, err := decryptPayload(key, []byte{1, 2, 3})
require.Error(t, err)
}
func TestSignAlignsTheIntegrityPad(t *testing.T) {
e := &established{algorithms: cipherSuites[3], keys: &keys{integrity: make([]byte, 20)}}
for size := range 8 {
packet := append(rmcpHeader(rmcpClassIPMI), make([]byte, 12+size)...)
signed := e.sign(packet)
// everything from the auth type through the next header is a
// multiple of 4, then 12 bytes of HMAC-SHA1-96
require.Zero(t, (len(signed)-12-4)%4)
require.Equal(t, byte(0x07), signed[len(signed)-13])
}
}
func TestLoginRejectsUnsupportedCipherSuites(t *testing.T) {
_, err := (&Client{}).Login(context.Background(), "127.0.0.1", 623, "ADMIN", "ADMIN", 17)
require.ErrorContains(t, err, "unsupported cipher suite 17")
_, err = (&Client{}).Command(0x06, 0x01, nil)
require.ErrorContains(t, err, "not logged in")
}
// capturedHandshake is a cipher suite 3 session setup with ipmi_sim for
// ADMIN/ADMIN, up to and including RAKP message 4.
func capturedHandshake(t *testing.T) (*handshake, []byte) {
return &handshake{
algorithms: cipherSuites[3],
consoleSessionID: mustHex(t, "244a5ac8"),
bmcSessionID: mustHex(t, "04170000"),
consoleRandom: mustHex(t, "de862025bb0c5ee6c30a58d383d44cb5"),
bmcRandom: mustHex(t, "b56fef2ee0fcab6664a019a4c88df4f6"),
bmcGUID: mustHex(t, "a123456789abcdefa123456789abcdef"),
role: 0x14,
username: "ADMIN",
bmcMAC: mustHex(t, "7fac6f776553c58df508b9ff656b5eacb6c2def9"),
}, mustHex(t, "00000000244a5ac8231219c5fc212cdd58c09424")
}
func TestBMCProofsMatchOnlyTheRealPassword(t *testing.T) {
h, rakp4 := capturedHandshake(t)
sik := func(password string) []byte {
return hmacOf(sha1.New, []byte(password), h.consoleRandom, h.bmcRandom, []byte{h.role, byte(len(h.username))}, []byte(h.username))
}
require.Equal(t, h.bmcMAC, hmacOf(sha1.New, []byte("ADMIN"), h.rakp2Salt()), "rakp 2 code")
require.NotEqual(t, h.bmcMAC, hmacOf(sha1.New, []byte("wrong"), h.rakp2Salt()))
require.True(t, rakp4Valid(sha1.New, sik("ADMIN"), h, rakp4))
require.False(t, rakp4Valid(sha1.New, sik("wrong"), h, rakp4))
tampered := bytes.Clone(rakp4)
tampered[len(tampered)-1] ^= 0xff
require.False(t, rakp4Valid(sha1.New, sik("ADMIN"), h, tampered), "a forged success must not pass")
require.False(t, rakp4Valid(sha1.New, sik("ADMIN"), h, rakp4[:10]))
}
func TestShortCommandReplyIsRejectedNotPanicked(t *testing.T) {
execID := "ipmi-short"
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
t.Cleanup(func() { protocolstate.Close(execID) })
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
// a reply that ends at the completion code, without the checksum
bmc, err := net.ListenPacket("udp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = bmc.Close() })
go func() {
buf := make([]byte, 512)
for {
n, addr, err := bmc.ReadFrom(buf)
if err != nil {
return
}
reply := bytes.Clone(buf[:21])
reply[19] = buf[n-2] // echo the command
reply[20] = 0x00
_, _ = bmc.WriteTo(reply, addr)
}
}()
port := bmc.LocalAddr().(*net.UDPAddr).Port
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
_, err = GetSystemGUID(ctx, "127.0.0.1", port)
require.ErrorIs(t, err, errNoResponse)
}
-173
View File
@@ -1,173 +0,0 @@
package ipmi
import (
"bytes"
"context"
"encoding/binary"
"errors"
"fmt"
"net"
"time"
libnet "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/net"
)
// Wire constants from the IPMI v2.0 specification and DSP0136 (ASF).
const (
rmcpVersion = 0x06
rmcpNoAck = 0xff
rmcpClassASF = 0x06
rmcpClassIPMI = 0x07
asfIANA = 0x000011be
asfTypePing = 0x80
asfTypePong = 0x40
asfEntitiesIPMI = 0x80
authTypeNone = 0x00
authTypeRMCPP = 0x06
netFnApp = 0x06
cmdGetSystemGUID = 0x37
cmdGetChannelAuthCaps = 0x38
cmdGetCipherSuites = 0x54
bmcAddress = 0x20
consoleAddress = 0x81
currentChannel = 0x0e
privAdmin = 0x04
payloadOpenSessionRequest = 0x10
payloadOpenSessionResponse = 0x11
payloadRAKP1 = 0x12
payloadRAKP2 = 0x13
payloadRAKP3 = 0x14
payloadRAKP4 = 0x15
// rakpNameOnlyLookup asks the BMC to find the user by name alone, so the
// role does not have to match the one the account was created with.
rakpNameOnlyLookup = 0x10
attempts = 2
)
var errNoResponse = errors.New("no ipmi response")
// session is one UDP exchange with a BMC. UDP is lossy, so each request is
// sent up to attempts times.
type session struct {
conn net.Conn
timeout time.Duration
}
func dial(ctx context.Context, host string, port int) (*session, error) {
conn, err := libnet.Dial(ctx, "udp", net.JoinHostPort(host, fmt.Sprint(port)))
if err != nil {
return nil, err
}
timeout := 3 * time.Second
if deadline, ok := ctx.Deadline(); ok {
timeout = min(timeout, time.Until(deadline)/attempts)
}
return &session{conn: conn, timeout: timeout}, nil
}
func (s *session) close() { _ = s.conn.Close() }
// exchange sends packet and returns the first reply accepted by match.
func (s *session) exchange(packet []byte, match func([]byte) bool) ([]byte, error) {
buf := make([]byte, 1024)
for range attempts {
if _, err := s.conn.Write(packet); err != nil {
return nil, err
}
_ = s.conn.SetReadDeadline(time.Now().Add(s.timeout))
for {
n, err := s.conn.Read(buf)
if err != nil {
break
}
if reply := buf[:n]; match(reply) {
return bytes.Clone(reply), nil
}
}
}
return nil, errNoResponse
}
func rmcpHeader(class byte) []byte {
return []byte{rmcpVersion, 0x00, rmcpNoAck, class}
}
// ping sends an ASF presence ping and reports whether the pong advertises
// IPMI support.
func (s *session) ping() (bool, error) {
packet := append(rmcpHeader(rmcpClassASF), 0, 0, 0, 0, asfTypePing, 0x00, 0x00, 0x00)
binary.BigEndian.PutUint32(packet[4:8], asfIANA)
reply, err := s.exchange(packet, func(reply []byte) bool {
return len(reply) >= 12 && reply[3] == rmcpClassASF && reply[8] == asfTypePong
})
if err != nil {
return false, err
}
// pong data: IANA (4), OEM (4), supported entities (1), interactions (1)
if len(reply) < 21 {
return false, nil
}
return reply[20]&asfEntitiesIPMI != 0, nil
}
func checksum(data []byte) byte {
var sum byte
for _, b := range data {
sum += b
}
return -sum
}
// command sends a session-less IPMI 1.5 request and returns the response
// data after a successful completion code.
func (s *session) command(cmd byte, data []byte) ([]byte, error) {
header := []byte{bmcAddress, netFnApp << 2}
body := append([]byte{consoleAddress, 0x00, cmd}, data...)
message := append(append(header, checksum(header)), body...)
message = append(message, checksum(body))
packet := append(rmcpHeader(rmcpClassIPMI), authTypeNone, 0, 0, 0, 0, 0, 0, 0, 0, byte(len(message)))
packet = append(packet, message...)
reply, err := s.exchange(packet, func(reply []byte) bool {
// rmcp (4), session header (10), then rqAddr, netFn, checksum,
// rsAddr, seq, cmd, completion code
// ... completion code, then at least the trailing checksum
return len(reply) >= 22 && reply[3] == rmcpClassIPMI && reply[4] == authTypeNone && reply[19] == cmd
})
if err != nil {
return nil, err
}
if code := reply[20]; code != 0 {
return nil, fmt.Errorf("ipmi command 0x%02x failed with completion code 0x%02x", cmd, code)
}
// drop the trailing checksum
return reply[21 : len(reply)-1], nil
}
// rmcpPlus sends an IPMI 2.0 session-setup payload and returns the reply
// payload of the expected type.
func (s *session) rmcpPlus(payloadType byte, payload []byte, replyType byte) ([]byte, error) {
packet := append(rmcpHeader(rmcpClassIPMI), authTypeRMCPP, payloadType, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
binary.LittleEndian.PutUint16(packet[14:16], uint16(len(payload)))
packet = append(packet, payload...)
reply, err := s.exchange(packet, func(reply []byte) bool {
return len(reply) >= 16 && reply[3] == rmcpClassIPMI && reply[4] == authTypeRMCPP && reply[5]&0x3f == replyType
})
if err != nil {
return nil, err
}
length := int(binary.LittleEndian.Uint16(reply[14:16]))
if len(reply) < 16+length {
return nil, fmt.Errorf("truncated ipmi payload")
}
return reply[16 : 16+length], nil
}
-398
View File
@@ -1,398 +0,0 @@
package ipmi
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/hmac"
"crypto/md5"
"crypto/rand"
"crypto/sha1"
"encoding/binary"
"errors"
"fmt"
"hash"
)
// RMCP+ algorithm numbers (IPMI v2.0 tables 13-17 to 13-19).
const (
authNone = 0x00
authHMACSHA1 = 0x01
authHMACMD5 = 0x02
integrityNone = 0x00
integrityHMACSHA196 = 0x01
confidentialityNone = 0x00
confidentialityAESCBC128 = 0x01
payloadIPMI = 0x00
payloadEncrypted = 0x80
payloadSigned = 0x40
cmdCloseSession = 0x3c
// rakp status codes worth naming in errors
rakpInvalidRole = 0x09
rakpUnauthorizedName = 0x0d
)
type algorithms struct {
auth, integrity, confidentiality byte
}
// cipherSuites lists the suites a session can be opened with. Suites with
// SHA-256 or MD5 message integrity are left out until they can be checked
// against a BMC that implements them correctly.
var cipherSuites = map[int]algorithms{
0: {authNone, integrityNone, confidentialityNone},
1: {authHMACSHA1, integrityNone, confidentialityNone},
2: {authHMACSHA1, integrityHMACSHA196, confidentialityNone},
3: {authHMACSHA1, integrityHMACSHA196, confidentialityAESCBC128},
6: {authHMACMD5, integrityNone, confidentialityNone},
}
func authHash(auth byte) func() hash.Hash {
switch auth {
case authHMACSHA1:
return sha1.New
case authHMACMD5:
return md5.New
}
return nil
}
func hmacOf(newHash func() hash.Hash, key []byte, parts ...[]byte) []byte {
mac := hmac.New(newHash, key)
for _, part := range parts {
_, _ = mac.Write(part)
}
return mac.Sum(nil)
}
// handshake is the state of an RMCP+ session setup up to RAKP message 2.
type handshake struct {
algorithms
consoleSessionID, bmcSessionID []byte
consoleRandom, bmcRandom []byte
bmcGUID []byte
role byte
username string
// bmcMAC is the RAKP message 2 key exchange code, empty without
// authentication.
bmcMAC []byte
}
// rakpError is a non-zero status in an RMCP+ session setup reply.
type rakpError struct {
step string
status byte
}
func (e *rakpError) Error() string {
switch e.status {
case rakpUnauthorizedName:
return fmt.Sprintf("bmc rejected the username in %s (status 0x%02x)", e.step, e.status)
case rakpInvalidRole:
return fmt.Sprintf("bmc rejected the requested privilege in %s (status 0x%02x)", e.step, e.status)
}
return fmt.Sprintf("bmc refused %s with status 0x%02x", e.step, e.status)
}
// startSession opens an RMCP+ session and runs RAKP messages 1 and 2.
func (s *session) startSession(alg algorithms, username string, privilege byte) (*handshake, error) {
if len(username) > 16 {
return nil, fmt.Errorf("ipmi usernames are at most 16 bytes")
}
h := &handshake{
algorithms: alg,
consoleSessionID: make([]byte, 4),
consoleRandom: make([]byte, 16),
role: privilege | rakpNameOnlyLookup,
username: username,
}
if _, err := rand.Read(h.consoleSessionID); err != nil {
return nil, err
}
if _, err := rand.Read(h.consoleRandom); err != nil {
return nil, err
}
open := []byte{0x00, privilege, 0x00, 0x00}
open = append(open, h.consoleSessionID...)
open = append(open,
0x00, 0x00, 0x00, 0x08, alg.auth, 0x00, 0x00, 0x00,
0x01, 0x00, 0x00, 0x08, alg.integrity, 0x00, 0x00, 0x00,
0x02, 0x00, 0x00, 0x08, alg.confidentiality, 0x00, 0x00, 0x00,
)
opened, err := s.rmcpPlus(payloadOpenSessionRequest, open, payloadOpenSessionResponse)
if err != nil {
return nil, err
}
if len(opened) < 2 {
return nil, fmt.Errorf("short open session response")
}
if opened[1] != 0 {
return nil, &rakpError{step: "the session", status: opened[1]}
}
if len(opened) < 12 {
return nil, fmt.Errorf("short open session response")
}
h.bmcSessionID = bytes.Clone(opened[8:12])
rakp1 := []byte{0x00, 0x00, 0x00, 0x00}
rakp1 = append(rakp1, h.bmcSessionID...)
rakp1 = append(rakp1, h.consoleRandom...)
rakp1 = append(rakp1, h.role, 0x00, 0x00, byte(len(username)))
rakp1 = append(rakp1, username...)
rakp2, err := s.rmcpPlus(payloadRAKP1, rakp1, payloadRAKP2)
if err != nil {
return nil, err
}
if len(rakp2) < 2 {
return nil, fmt.Errorf("short rakp message 2")
}
if rakp2[1] != 0 {
return nil, &rakpError{step: "rakp message 1", status: rakp2[1]}
}
if len(rakp2) < 40 {
return nil, fmt.Errorf("short rakp message 2")
}
h.bmcRandom = bytes.Clone(rakp2[8:24])
h.bmcGUID = bytes.Clone(rakp2[24:40])
h.bmcMAC = bytes.Clone(rakp2[40:])
return h, nil
}
// rakp2Salt is everything the RAKP message 2 code covers.
func (h *handshake) rakp2Salt() []byte {
salt := make([]byte, 0, 58+len(h.username))
salt = append(salt, h.consoleSessionID...)
salt = append(salt, h.bmcSessionID...)
salt = append(salt, h.consoleRandom...)
salt = append(salt, h.bmcRandom...)
salt = append(salt, h.bmcGUID...)
salt = append(salt, h.role, byte(len(h.username)))
return append(salt, h.username...)
}
// keys are the session keys derived from the session integrity key.
type keys struct {
integrity []byte // K1
aes []byte // first 16 bytes of K2
}
// finishSession runs RAKP messages 3 and 4, which is where a wrong password
// is rejected, and derives the session keys. Both directions are proven:
// the BMC's RAKP message 2 code and RAKP message 4 integrity value must
// match the password, so a peer that only answers with success statuses
// cannot pass for a login.
func (s *session) finishSession(h *handshake, password string) (*keys, error) {
if len(password) > 20 {
return nil, fmt.Errorf("ipmi passwords are at most 20 bytes")
}
newHash := authHash(h.auth)
if newHash != nil && !hmac.Equal(h.bmcMAC, hmacOf(newHash, []byte(password), h.rakp2Salt())) {
return nil, errBMCProof
}
rakp3 := []byte{0x00, 0x00, 0x00, 0x00}
rakp3 = append(rakp3, h.bmcSessionID...)
if newHash != nil {
rakp3 = append(rakp3, hmacOf(newHash, []byte(password),
h.bmcRandom, h.consoleSessionID, []byte{h.role, byte(len(h.username))}, []byte(h.username))...)
}
rakp4, err := s.rmcpPlus(payloadRAKP3, rakp3, payloadRAKP4)
if err != nil {
return nil, err
}
if len(rakp4) < 2 {
return nil, fmt.Errorf("short rakp message 4")
}
if rakp4[1] != 0 {
return nil, &rakpError{step: "the login", status: rakp4[1]}
}
if newHash == nil {
return &keys{}, nil
}
sik := hmacOf(newHash, []byte(password),
h.consoleRandom, h.bmcRandom, []byte{h.role, byte(len(h.username))}, []byte(h.username))
if !rakp4Valid(newHash, sik, h, rakp4) {
return nil, errBMCProof
}
return &keys{
integrity: hmacOf(newHash, sik, bytes.Repeat([]byte{0x01}, 20)),
aes: hmacOf(newHash, sik, bytes.Repeat([]byte{0x02}, 20))[:16],
}, nil
}
// errBMCProof means the peer's authentication code does not match the
// password: a wrong password, or something other than the BMC answering.
var errBMCProof = errors.New("bmc authentication code does not match the password")
// rakp4Valid checks the RAKP message 4 integrity value, the session key's
// HMAC over the console random, BMC session ID and BMC GUID, truncated to
// 12 bytes for sha1 and 16 for md5.
func rakp4Valid(newHash func() hash.Hash, sik []byte, h *handshake, rakp4 []byte) bool {
size := 12
if h.auth == authHMACMD5 {
size = 16
}
if len(rakp4) < 8+size {
return false
}
want := hmacOf(newHash, sik, h.consoleRandom, h.bmcSessionID, h.bmcGUID)[:size]
return hmac.Equal(rakp4[8:8+size], want)
}
// established is an authenticated RMCP+ session.
type established struct {
*session
algorithms
keys *keys
bmcSessionID []byte
consoleSessionID []byte
sequence uint32
requestSeq byte
}
// command sends an IPMI request inside the session and returns its
// completion code and response data.
func (e *established) command(netFn, cmd byte, data []byte) (byte, []byte, error) {
// the request sequence is 6 bits on the wire
e.requestSeq = (e.requestSeq + 1) & 0x3f
header := []byte{bmcAddress, netFn << 2}
body := append([]byte{consoleAddress, e.requestSeq << 2, cmd}, data...)
message := append(append(header, checksum(header)), body...)
message = append(message, checksum(body))
payloadType := byte(payloadIPMI)
if e.confidentiality == confidentialityAESCBC128 {
payloadType |= payloadEncrypted
var err error
if message, err = encryptPayload(e.keys.aes, message); err != nil {
return 0, nil, err
}
}
if e.integrity != integrityNone {
payloadType |= payloadSigned
}
e.sequence++
packet := append(rmcpHeader(rmcpClassIPMI), authTypeRMCPP, payloadType)
packet = append(packet, e.bmcSessionID...)
packet = binary.LittleEndian.AppendUint32(packet, e.sequence)
packet = binary.LittleEndian.AppendUint16(packet, uint16(len(message)))
packet = append(packet, message...)
if e.integrity != integrityNone {
packet = e.sign(packet)
}
// a reply must be for this session, pass the integrity check when the
// session is signed, and answer this request, not an earlier one that
// arrived late
var payload []byte
_, err := e.exchange(packet, func(reply []byte) bool {
payload = e.open(reply)
return len(payload) >= 8 && payload[4]>>2 == e.requestSeq && payload[5] == cmd
})
if err != nil {
return 0, nil, err
}
// rqAddr, netFn, checksum, rsAddr, seq, cmd, completion code, data,
// checksum
return payload[6], bytes.Clone(payload[7 : len(payload)-1]), nil
}
// open returns the IPMI message in a session reply, or nil when the reply
// is not for this session or fails its integrity check.
func (e *established) open(reply []byte) []byte {
if len(reply) < 16 || reply[3] != rmcpClassIPMI || reply[4] != authTypeRMCPP || reply[5]&0x3f != payloadIPMI {
return nil
}
if !bytes.Equal(reply[6:10], e.consoleSessionID) {
return nil
}
length := int(binary.LittleEndian.Uint16(reply[14:16]))
if len(reply) < 16+length {
return nil
}
if e.integrity != integrityNone {
if reply[5]&payloadSigned == 0 || !e.verify(reply, 16+length) {
return nil
}
}
payload := reply[16 : 16+length]
if reply[5]&payloadEncrypted != 0 {
var err error
if payload, err = decryptPayload(e.keys.aes, payload); err != nil {
return nil
}
}
return payload
}
// verify checks a reply's HMAC-SHA1-96 trailer: the pad after the payload,
// pad length and next header, then the code over everything from the auth
// type on.
func (e *established) verify(reply []byte, payloadEnd int) bool {
const codeSize = 12
if len(reply) < payloadEnd+2+codeSize {
return false
}
codeStart := len(reply) - codeSize
if reply[codeStart-1] != 0x07 || int(reply[codeStart-2]) != codeStart-2-payloadEnd {
return false
}
want := hmacOf(sha1.New, e.keys.integrity, reply[4:codeStart])[:codeSize]
return hmac.Equal(reply[codeStart:], want)
}
// sign appends the HMAC-SHA1-96 integrity trailer: pad to a multiple of 4 from the auth
// type on, pad length, next header, and the truncated HMAC over it all.
func (e *established) sign(packet []byte) []byte {
pad := (4 - (len(packet)-4+2)%4) % 4
packet = append(packet, bytes.Repeat([]byte{0xff}, pad)...)
packet = append(packet, byte(pad), 0x07)
return append(packet, hmacOf(sha1.New, e.keys.integrity, packet[4:])[:12]...)
}
// encryptPayload is AES-CBC-128 with a random IV in front and the
// confidentiality pad (1, 2, 3, ... then its length) at the end.
func encryptPayload(key, payload []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
pad := (aes.BlockSize - (len(payload)+1)%aes.BlockSize) % aes.BlockSize
plain := bytes.Clone(payload)
for i := 1; i <= pad; i++ {
plain = append(plain, byte(i))
}
plain = append(plain, byte(pad))
out := make([]byte, aes.BlockSize+len(plain))
if _, err := rand.Read(out[:aes.BlockSize]); err != nil {
return nil, err
}
cipher.NewCBCEncrypter(block, out[:aes.BlockSize]).CryptBlocks(out[aes.BlockSize:], plain)
return out, nil
}
func decryptPayload(key, payload []byte) ([]byte, error) {
if len(payload) < 2*aes.BlockSize || len(payload)%aes.BlockSize != 0 {
return nil, fmt.Errorf("malformed encrypted payload")
}
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
plain := make([]byte, len(payload)-aes.BlockSize)
cipher.NewCBCDecrypter(block, payload[:aes.BlockSize]).CryptBlocks(plain, payload[aes.BlockSize:])
pad := int(plain[len(plain)-1])
if pad+1 > len(plain) {
return nil, fmt.Errorf("malformed encrypted payload")
}
return plain[:len(plain)-pad-1], nil
}
-596
View File
@@ -1,596 +0,0 @@
package mqtt
import (
"context"
"crypto/tls"
"errors"
"fmt"
"net"
"strconv"
"sync"
"time"
libnet "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/net"
)
var (
defaultTimeout = 10 * time.Second
)
const (
maxSubscribedMessages = 1000
maxSubscribedBytes = 10 * 1024 * 1024 // 10MB
)
type (
// Message represents an MQTT published message.
// @example
// ```javascript
// const mqtt = require('nuclei/mqtt');
// const client = new mqtt.Client();
// client.Connect('broker.hivemq.com', 1883);
// const msgs = client.Subscribe('test/topic', 2);
// log(msgs[0].Topic, msgs[0].Payload);
// ```
Message struct {
Topic string `json:"topic"`
Payload string `json:"payload"`
QoS int `json:"qos"`
Retain bool `json:"retain"`
}
// ConnectOptions contains connection options for an MQTT broker.
// @example
// ```javascript
// const mqtt = require('nuclei/mqtt');
// const opts = new mqtt.ConnectOptions();
// opts.Username = 'admin';
// opts.Password = 'admin';
// ```
ConnectOptions struct {
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
ClientID string `json:"client_id,omitempty"`
CleanSession *bool `json:"clean_session,omitempty"`
KeepAlive int `json:"keep_alive,omitempty"`
TLS bool `json:"tls,omitempty"`
InsecureSkipVerify *bool `json:"insecure_skip_verify,omitempty"`
Timeout int `json:"timeout,omitempty"`
}
// ConnectResult is the response returned by an MQTT connection attempt.
// @example
// ```javascript
// const mqtt = require('nuclei/mqtt');
// const client = new mqtt.Client();
// const res = client.Connect('acme.com', 1883);
// log(res.Success, res.ReturnCode, res.Reason);
// ```
ConnectResult struct {
Success bool `json:"success"`
ReturnCode int `json:"return_code"`
Reason string `json:"reason"`
SessionPresent bool `json:"session_present"`
}
// Client is an MQTT client for testing MQTT broker misconfigurations and authentication.
// @example
// ```javascript
// const mqtt = require('nuclei/mqtt');
// const client = new mqtt.Client();
// const res = client.Connect('acme.com', 1883);
// if (res.Success) {
// const msgs = client.Subscribe('#', 2);
// client.Publish('nuclei/probe', 'test');
// }
// client.Close();
// ```
Client struct {
conn net.Conn
packetID uint16
mu sync.Mutex
timeout time.Duration
}
)
// dial opens a network connection to host:port respecting protocolstate policy and context.
func dial(ctx context.Context, host string, port int, useTLS bool, insecureSkipVerify bool, timeout time.Duration) (net.Conn, error) {
if host == "" || port <= 0 {
return nil, fmt.Errorf("invalid host or port")
}
addr := net.JoinHostPort(host, strconv.Itoa(port))
conn, err := libnet.Dial(ctx, "tcp", addr)
if err != nil {
return nil, err
}
_ = conn.SetDeadline(time.Now().Add(timeout))
if useTLS {
tlsConn := tls.Client(conn, &tls.Config{
InsecureSkipVerify: insecureSkipVerify,
MinVersion: tls.VersionTLS12,
ServerName: host,
})
if err := tlsConn.HandshakeContext(ctx); err != nil {
_ = conn.Close()
return nil, fmt.Errorf("TLS handshake failed: %w", err)
}
conn = tlsConn
}
_ = conn.SetDeadline(time.Now().Add(timeout))
return conn, nil
}
// IsMQTT checks whether the target host:port is running an MQTT broker.
// @example
// ```javascript
// const mqtt = require('nuclei/mqtt');
// const isMqtt = mqtt.IsMQTT('acme.com', 1883);
// log(isMqtt);
// ```
func IsMQTT(ctx context.Context, host string, port int) (bool, error) {
conn, err := dial(ctx, host, port, false, true, defaultTimeout)
if err != nil {
return false, err
}
defer func() { _ = conn.Close() }()
pkt, err := buildConnectPacket("nuclei-probe", "", "", true, 60)
if err != nil {
return false, err
}
_ = conn.SetWriteDeadline(time.Now().Add(defaultTimeout))
if _, err := conn.Write(pkt); err != nil {
return false, err
}
header, payload, err := readPacket(conn)
if err != nil {
return false, nil
}
// Any valid CONNACK indicates an active MQTT broker
if (header >> 4) == PacketConnack {
_, _, err := parseConnack(payload)
return err == nil, nil
}
return false, nil
}
// IsAnonymousAllowed checks whether an MQTT broker permits unauthenticated / anonymous login.
// @example
// ```javascript
// const mqtt = require('nuclei/mqtt');
// const allowed = mqtt.IsAnonymousAllowed('acme.com', 1883);
// log(allowed);
// ```
func IsAnonymousAllowed(ctx context.Context, host string, port int) (bool, error) {
conn, err := dial(ctx, host, port, false, true, defaultTimeout)
if err != nil {
return false, err
}
defer func() { _ = conn.Close() }()
pkt, err := buildConnectPacket("", "", "", true, 60)
if err != nil {
return false, err
}
_ = conn.SetWriteDeadline(time.Now().Add(defaultTimeout))
if _, err := conn.Write(pkt); err != nil {
return false, err
}
header, payload, err := readPacket(conn)
if err != nil {
return false, err
}
if (header >> 4) != PacketConnack {
return false, fmt.Errorf("expected CONNACK, got packet type %d", header>>4)
}
_, returnCode, err := parseConnack(payload)
if err != nil {
return false, err
}
return returnCode == ConnAccepted, nil
}
// Connect connects to an MQTT broker using optional credentials and settings.
// @example
// ```javascript
// const mqtt = require('nuclei/mqtt');
// const client = new mqtt.Client();
// const res = client.Connect('acme.com', 1883);
// log(res.Success);
// ```
func (c *Client) Connect(ctx context.Context, host string, port int, options *ConnectOptions) (*ConnectResult, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.conn != nil {
_ = c.conn.Close()
c.conn = nil
}
timeout := defaultTimeout
cleanSession := true
keepAlive := uint16(60)
clientID := "nuclei"
var username, password string
useTLS := false
insecureSkipVerify := true
if options != nil {
if options.Timeout > 0 {
timeout = time.Duration(options.Timeout) * time.Second
}
if options.KeepAlive > 0 {
if options.KeepAlive > 65535 {
keepAlive = 65535
} else {
keepAlive = uint16(options.KeepAlive)
}
}
if options.ClientID != "" {
clientID = options.ClientID
}
if options.CleanSession != nil {
cleanSession = *options.CleanSession
}
if options.InsecureSkipVerify != nil {
insecureSkipVerify = *options.InsecureSkipVerify
}
username = options.Username
password = options.Password
useTLS = options.TLS
}
c.timeout = timeout
conn, err := dial(ctx, host, port, useTLS, insecureSkipVerify, timeout)
if err != nil {
return &ConnectResult{
Success: false,
ReturnCode: -1,
Reason: err.Error(),
}, err
}
pkt, err := buildConnectPacket(clientID, username, password, cleanSession, keepAlive)
if err != nil {
_ = conn.Close()
return &ConnectResult{
Success: false,
ReturnCode: -1,
Reason: err.Error(),
}, err
}
_ = conn.SetWriteDeadline(time.Now().Add(timeout))
if _, err := conn.Write(pkt); err != nil {
_ = conn.Close()
return &ConnectResult{
Success: false,
ReturnCode: -1,
Reason: fmt.Sprintf("failed to send CONNECT: %v", err),
}, err
}
header, payload, err := readPacket(conn)
if err != nil {
_ = conn.Close()
return &ConnectResult{
Success: false,
ReturnCode: -1,
Reason: fmt.Sprintf("failed to read CONNACK: %v", err),
}, err
}
if (header >> 4) != PacketConnack {
_ = conn.Close()
return &ConnectResult{
Success: false,
ReturnCode: -1,
Reason: fmt.Sprintf("unexpected packet %d (expected CONNACK)", header>>4),
}, fmt.Errorf("unexpected packet %d", header>>4)
}
sessPresent, returnCode, err := parseConnack(payload)
if err != nil {
_ = conn.Close()
return &ConnectResult{
Success: false,
ReturnCode: int(returnCode),
Reason: err.Error(),
}, err
}
res := &ConnectResult{
Success: returnCode == ConnAccepted,
ReturnCode: int(returnCode),
Reason: ConnackReason(returnCode),
SessionPresent: sessPresent,
}
if res.Success {
c.conn = conn
} else {
_ = conn.Close()
}
return res, nil
}
// ConnectAuth connects to an MQTT broker with a username and password.
// @example
// ```javascript
// const mqtt = require('nuclei/mqtt');
// const client = new mqtt.Client();
// const res = client.ConnectAuth('acme.com', 1883, 'admin', 'admin');
// log(res.Success);
// ```
func (c *Client) ConnectAuth(ctx context.Context, host string, port int, username, password string) (*ConnectResult, error) {
clean := true
return c.Connect(ctx, host, port, &ConnectOptions{
Username: username,
Password: password,
CleanSession: &clean,
})
}
// Subscribe subscribes to a topic filter and collects received messages up to timeoutSec.
// @example
// ```javascript
// const mqtt = require('nuclei/mqtt');
// const client = new mqtt.Client();
// client.Connect('acme.com', 1883);
// const msgs = client.Subscribe('#', 3);
//
// for (const m of msgs) {
// log(m.Topic, m.Payload);
// }
//
// ```
func (c *Client) Subscribe(ctx context.Context, topic string, timeoutSec int) ([]*Message, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.conn == nil {
return nil, fmt.Errorf("not connected to MQTT broker")
}
c.packetID++
if c.packetID == 0 {
c.packetID = 1
}
pktID := c.packetID
subPkt, err := buildSubscribePacket(pktID, topic, 0)
if err != nil {
return nil, fmt.Errorf("failed to build SUBSCRIBE: %w", err)
}
_ = c.conn.SetWriteDeadline(time.Now().Add(c.timeout))
if _, err := c.conn.Write(subPkt); err != nil {
_ = c.conn.Close()
c.conn = nil
return nil, fmt.Errorf("failed to send SUBSCRIBE: %w", err)
}
waitDuration := 2 * time.Second
if timeoutSec > 0 {
if timeoutSec > 60 {
timeoutSec = 60
}
waitDuration = time.Duration(timeoutSec) * time.Second
}
if deadline, ok := ctx.Deadline(); ok {
if remaining := time.Until(deadline); remaining < waitDuration {
waitDuration = remaining
}
}
readDeadline := time.Now().Add(waitDuration)
_ = c.conn.SetDeadline(readDeadline)
defer func() {
if c.conn != nil {
_ = c.conn.SetDeadline(time.Time{})
}
}()
var messages []*Message
var totalBytes int
subackReceived := false
for {
select {
case <-ctx.Done():
_ = c.conn.Close()
c.conn = nil
return messages, ctx.Err()
default:
}
header, payload, err := readPacket(c.conn)
if err != nil {
if errors.Is(err, errBrokenStream) {
// The window closed in the middle of a packet. The stream cannot be
// read in sync again, so the connection is dropped.
_ = c.conn.Close()
c.conn = nil
if !subackReceived {
return nil, fmt.Errorf("failed waiting for SUBACK: %w", err)
}
break
}
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
// Window finished
break
}
if !subackReceived {
_ = c.conn.Close()
c.conn = nil
return nil, fmt.Errorf("failed waiting for SUBACK: %w", err)
}
break
}
packetType := header >> 4
switch packetType {
case PacketSuback:
subackPID, grantedQoS, err := parseSuback(payload)
if err != nil {
_ = c.conn.Close()
c.conn = nil
return nil, err
}
if subackPID != pktID {
continue // Ignore stale/mismatched SUBACK from an earlier subscription
}
for _, qos := range grantedQoS {
if qos == 0x80 {
return nil, fmt.Errorf("subscription rejected by broker for topic %q", topic)
}
}
subackReceived = true
case PacketPublish:
msg, err := parsePublish(header, payload)
if err == nil {
messages = append(messages, msg)
totalBytes += len(msg.Topic) + len(msg.Payload)
if len(messages) >= maxSubscribedMessages || totalBytes >= maxSubscribedBytes {
return messages, nil
}
}
}
if time.Now().After(readDeadline) {
break
}
}
if !subackReceived {
_ = c.conn.Close()
c.conn = nil
return nil, fmt.Errorf("subscription timed out waiting for SUBACK")
}
return messages, nil
}
// Publish publishes a message payload to a topic.
// @example
// ```javascript
// const mqtt = require('nuclei/mqtt');
// const client = new mqtt.Client();
// client.Connect('acme.com', 1883);
// const ok = client.Publish('nuclei/test', 'probe payload');
// log(ok);
// ```
func (c *Client) Publish(ctx context.Context, topic string, payload string) (bool, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.conn == nil {
return false, fmt.Errorf("not connected to MQTT broker")
}
select {
case <-ctx.Done():
return false, ctx.Err()
default:
}
pkt, err := buildPublishPacket(topic, []byte(payload), 0, false, 0)
if err != nil {
return false, fmt.Errorf("failed to build PUBLISH: %w", err)
}
_ = c.conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
if _, err := c.conn.Write(pkt); err != nil {
_ = c.conn.Close()
c.conn = nil
return false, fmt.Errorf("failed to send PUBLISH: %w", err)
}
return true, nil
}
// Ping sends an MQTT PINGREQ and awaits PINGRESP.
// @example
// ```javascript
// const mqtt = require('nuclei/mqtt');
// const client = new mqtt.Client();
// client.Connect('acme.com', 1883);
// const alive = client.Ping();
// log(alive);
// ```
func (c *Client) Ping(ctx context.Context) (bool, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.conn == nil {
return false, fmt.Errorf("not connected to MQTT broker")
}
select {
case <-ctx.Done():
return false, ctx.Err()
default:
}
_ = c.conn.SetDeadline(time.Now().Add(5 * time.Second))
_ = c.conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
if _, err := c.conn.Write(buildPingreqPacket()); err != nil {
_ = c.conn.Close()
c.conn = nil
return false, fmt.Errorf("failed to send PINGREQ: %w", err)
}
for {
header, _, err := readPacket(c.conn)
if err != nil {
_ = c.conn.Close()
c.conn = nil
return false, fmt.Errorf("failed to read PINGRESP: %w", err)
}
packetType := header >> 4
if packetType == PacketPingresp {
return true, nil
}
if packetType == PacketPublish {
// Broker delivered an incoming publish message while ping was in flight; continue waiting for PINGRESP
continue
}
return false, fmt.Errorf("unexpected ping response packet: %d", packetType)
}
}
// Close cleanly disconnects and closes the connection.
// @example
// ```javascript
// const mqtt = require('nuclei/mqtt');
// const client = new mqtt.Client();
// client.Connect('acme.com', 1883);
// client.Close();
// ```
func (c *Client) Close() error {
c.mu.Lock()
defer c.mu.Unlock()
if c.conn != nil {
_ = c.conn.SetWriteDeadline(time.Now().Add(3 * time.Second))
_, _ = c.conn.Write(buildDisconnectPacket())
err := c.conn.Close()
c.conn = nil
return err
}
return nil
}
-395
View File
@@ -1,395 +0,0 @@
package mqtt
import (
"bytes"
"context"
"encoding/binary"
"net"
"strings"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestVarLengthEncoding(t *testing.T) {
testCases := []int{0, 1, 64, 127, 128, 16383, 16384, 2097151}
for _, tc := range testCases {
encoded, err := encodeVarLength(tc)
require.NoError(t, err)
decoded, err := decodeVarLength(bytes.NewReader(encoded))
require.NoError(t, err)
require.Equal(t, tc, decoded)
}
// Test negative or overflow
_, err := encodeVarLength(-1)
require.Error(t, err)
_, err = encodeVarLength(MaxRemainingLength + 1)
require.Error(t, err)
}
func TestOversizedStringsAndPackets(t *testing.T) {
hugeString := strings.Repeat("A", MaxStringLength+1)
_, err := encodeString(hugeString)
require.Error(t, err)
_, err = buildConnectPacket("id", hugeString, "pass", true, 60)
require.Error(t, err)
_, err = buildSubscribePacket(1, hugeString, 0)
require.Error(t, err)
_, err = buildPublishPacket(hugeString, []byte("ok"), 0, false, 0)
require.Error(t, err)
// Oversized packet reading
oversizedLen, err := encodeVarLength(MaxPacketSize + 100)
require.NoError(t, err)
buf := append([]byte{PacketPublish << 4}, oversizedLen...)
_, _, err = readPacket(bytes.NewReader(buf))
require.ErrorContains(t, err, "exceeds maximum allowed")
}
func TestPacketConnect(t *testing.T) {
pkt, err := buildConnectPacket("test-client", "user1", "pass1", true, 60)
require.NoError(t, err)
require.NotEmpty(t, pkt)
require.Equal(t, byte(PacketConnect<<4), pkt[0])
r := bytes.NewReader(pkt)
header, payload, err := readPacket(r)
require.NoError(t, err)
require.Equal(t, byte(PacketConnect<<4), header)
// Verify protocol name "MQTT"
protoLen := binary.BigEndian.Uint16(payload[0:2])
require.Equal(t, uint16(4), protoLen)
require.Equal(t, "MQTT", string(payload[2:6]))
require.Equal(t, byte(0x04), payload[6]) // Protocol level 4 (MQTT 3.1.1)
// CleanSession (0x02) | User (0x80) | Pass (0x40) = 0xC2
require.Equal(t, byte(0xC2), payload[7])
}
func TestPacketPublishAndParse(t *testing.T) {
pkt, err := buildPublishPacket("sensors/temperature", []byte("23.5C"), 0, true, 0)
require.NoError(t, err)
require.NotEmpty(t, pkt)
r := bytes.NewReader(pkt)
header, payload, err := readPacket(r)
require.NoError(t, err)
msg, err := parsePublish(header, payload)
require.NoError(t, err)
require.Equal(t, "sensors/temperature", msg.Topic)
require.Equal(t, "23.5C", msg.Payload)
require.Equal(t, 0, msg.QoS)
require.True(t, msg.Retain)
}
func TestPacketSubscribe(t *testing.T) {
pkt, err := buildSubscribePacket(42, "devices/#", 0)
require.NoError(t, err)
require.NotEmpty(t, pkt)
require.Equal(t, byte((PacketSubscribe<<4)|0x02), pkt[0])
r := bytes.NewReader(pkt)
header, payload, err := readPacket(r)
require.NoError(t, err)
require.Equal(t, byte((PacketSubscribe<<4)|0x02), header)
pid := binary.BigEndian.Uint16(payload[0:2])
require.Equal(t, uint16(42), pid)
subackPayload := []byte{0x00, 0x2A, 0x00} // PID 42, Success QoS 0
subackPid, granted, err := parseSuback(subackPayload)
require.NoError(t, err)
require.Equal(t, uint16(42), subackPid)
require.Equal(t, []byte{0x00}, granted)
}
func TestConnackReasons(t *testing.T) {
require.Equal(t, "connection accepted", ConnackReason(ConnAccepted))
require.Equal(t, "connection refused: bad user name or password", ConnackReason(ConnRefusedBadAuth))
require.Equal(t, "connection refused: not authorized", ConnackReason(ConnRefusedNotAuthorized))
require.Contains(t, ConnackReason(99), "connection refused: return code 99")
}
func TestRequestsHonourNetworkPolicy(t *testing.T) {
execID := "mqtt-deny"
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID, ExcludeTargets: []string{"203.0.113.50"}}))
t.Cleanup(func() { protocolstate.Close(execID) })
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
_, err := IsMQTT(ctx, "203.0.113.50", 1883)
require.Error(t, err)
client := &Client{}
_, err = client.Connect(ctx, "203.0.113.50", 1883, nil)
require.Error(t, err)
}
func startMockMQTTBroker(t *testing.T) (net.Listener, int) {
l, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
port := l.Addr().(*net.TCPAddr).Port
go func() {
for {
conn, err := l.Accept()
if err != nil {
return
}
go handleMockClient(conn)
}
}()
return l, port
}
func handleMockClient(conn net.Conn) {
defer func() { _ = conn.Close() }()
for {
header, payload, err := readPacket(conn)
if err != nil {
return
}
packetType := header >> 4
switch packetType {
case PacketConnect:
// Check if username is present (flags byte at offset 7)
if len(payload) >= 8 {
flags := payload[7]
if (flags & 0x80) != 0 {
// Check username
offset := 10 // after proto(6) + level(1) + flags(1) + keepalive(2)
if len(payload) >= offset+2 {
clientLen := int(binary.BigEndian.Uint16(payload[offset : offset+2]))
offset += 2 + clientLen
if len(payload) >= offset+2 {
userLen := int(binary.BigEndian.Uint16(payload[offset : offset+2]))
user := string(payload[offset+2 : offset+2+userLen])
if user == "baduser" {
// Send CONNACK Refused Bad Auth
_, _ = conn.Write([]byte{0x20, 0x02, 0x00, ConnRefusedBadAuth})
return
}
}
}
}
}
// Send CONNACK Accepted
_, _ = conn.Write([]byte{0x20, 0x02, 0x00, ConnAccepted})
case PacketSubscribe:
// Send SUBACK
pid := binary.BigEndian.Uint16(payload[0:2])
suback := []byte{0x90, 0x03, byte(pid >> 8), byte(pid & 0xFF), 0x00}
_, _ = conn.Write(suback)
// Publish test message
pubPkt, err := buildPublishPacket("nuclei/alert", []byte("vuln-detected"), 0, false, 0)
if err == nil {
_, _ = conn.Write(pubPkt)
}
case PacketPublish:
// QoS 0, no ack needed
case PacketPingreq:
// Send PINGRESP
_, _ = conn.Write([]byte{PacketPingresp << 4, 0x00})
case PacketDisconnect:
return
}
}
}
func TestMockBrokerInteractions(t *testing.T) {
execID := "mqtt-mock-test"
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
t.Cleanup(func() { protocolstate.Close(execID) })
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
l, port := startMockMQTTBroker(t)
defer func() { _ = l.Close() }()
// 1. Test IsMQTT
isMQTT, err := IsMQTT(ctx, "127.0.0.1", port)
require.NoError(t, err)
require.True(t, isMQTT)
// 2. Test IsAnonymousAllowed
anonAllowed, err := IsAnonymousAllowed(ctx, "127.0.0.1", port)
require.NoError(t, err)
require.True(t, anonAllowed)
// 3. Test Client Connect Auth Failure
badClient := &Client{}
badRes, _ := badClient.ConnectAuth(ctx, "127.0.0.1", port, "baduser", "wrongpass")
require.False(t, badRes.Success)
require.Equal(t, int(ConnRefusedBadAuth), badRes.ReturnCode)
// 4. Test Client Connect Success with default CleanSession preservation
client := &Client{}
res, err := client.Connect(ctx, "127.0.0.1", port, &ConnectOptions{KeepAlive: 70000})
require.NoError(t, err)
require.True(t, res.Success)
require.Equal(t, int(ConnAccepted), res.ReturnCode)
// 5. Test Ping
alive, err := client.Ping(ctx)
require.NoError(t, err)
require.True(t, alive)
// 6. Test Subscribe and Receive Message
msgs, err := client.Subscribe(ctx, "nuclei/alert", 1)
require.NoError(t, err)
require.NotEmpty(t, msgs)
require.Equal(t, "nuclei/alert", msgs[0].Topic)
require.Equal(t, "vuln-detected", msgs[0].Payload)
// 7. Test Publish
pubOk, err := client.Publish(ctx, "nuclei/probe", "test payload")
require.NoError(t, err)
require.True(t, pubOk)
// 8. Test Close and Double Close
require.NoError(t, client.Close())
require.NoError(t, client.Close())
}
func TestPasswordWithoutUsername(t *testing.T) {
// MQTT 3.1.1 (MQTT-3.1.2-22): If Password flag is set, User Name flag MUST be set.
pkt, err := buildConnectPacket("token-client", "", "my-secret-token", true, 60)
require.NoError(t, err)
r := bytes.NewReader(pkt)
_, payload, err := readPacket(r)
require.NoError(t, err)
// CleanSession (0x02) | User (0x80) | Pass (0x40) = 0xC2
require.Equal(t, byte(0xC2), payload[7])
// The client ID comes right after the keep alive at payload[8:10], and the
// username length follows it.
clientIDLen := int(binary.BigEndian.Uint16(payload[10:12]))
off := 12 + clientIDLen
// Username length should be 0 (empty string)
userLen := binary.BigEndian.Uint16(payload[off : off+2])
require.Equal(t, uint16(0), userLen)
off += 2
// Password follows the username
passLen := binary.BigEndian.Uint16(payload[off : off+2])
require.Equal(t, uint16(len("my-secret-token")), passLen)
off += 2
require.Equal(t, "my-secret-token", string(payload[off:off+int(passLen)]))
}
// A window that closes in the middle of a PUBLISH must drop the connection.
// Keeping it would let the next read start inside that packet.
func TestSubscribePartialPacketClosesConnection(t *testing.T) {
execID := "mqtt-partial-subscribe"
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
t.Cleanup(func() { protocolstate.Close(execID) })
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
l, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer func() { _ = l.Close() }()
port := l.Addr().(*net.TCPAddr).Port
go func() {
conn, err := l.Accept()
if err != nil {
return
}
defer func() { _ = conn.Close() }()
for {
header, payload, err := readPacket(conn)
if err != nil {
return
}
switch header >> 4 {
case PacketConnect:
_, _ = conn.Write([]byte{PacketConnack << 4, 0x02, 0x00, ConnAccepted})
case PacketSubscribe:
// SUBACK for the packet id, then a PUBLISH that promises 10 bytes
// and sends 5. The client's window closes in the middle of it.
_, _ = conn.Write([]byte{PacketSuback << 4, 0x03, payload[0], payload[1], 0x00})
_, _ = conn.Write([]byte{PacketPublish << 4, 0x0a, 0x00, 0x03, 'a', '/', 'b'})
}
}
}()
client := &Client{}
res, err := client.Connect(ctx, "127.0.0.1", port, nil)
require.NoError(t, err)
require.True(t, res.Success)
msgs, err := client.Subscribe(ctx, "a/b", 1)
require.NoError(t, err)
require.Empty(t, msgs)
require.Nil(t, client.conn, "a stream cut in the middle of a packet must not be reused")
}
func TestPingWithInFlightPublish(t *testing.T) {
execID := "mqtt-ping-inflight"
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
t.Cleanup(func() { protocolstate.Close(execID) })
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
l, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer func() { _ = l.Close() }()
port := l.Addr().(*net.TCPAddr).Port
go func() {
conn, err := l.Accept()
if err != nil {
return
}
defer func() { _ = conn.Close() }()
for {
header, _, err := readPacket(conn)
if err != nil {
return
}
packetType := header >> 4
switch packetType {
case PacketConnect:
_, _ = conn.Write([]byte{0x20, 0x02, 0x00, ConnAccepted})
case PacketPingreq:
// Send a PUBLISH packet first before PINGRESP
pubPkt, _ := buildPublishPacket("sensors/temp", []byte("25.0"), 0, false, 0)
_, _ = conn.Write(pubPkt)
// Then send PINGRESP
_, _ = conn.Write([]byte{PacketPingresp << 4, 0x00})
case PacketDisconnect:
return
}
}
}()
client := &Client{}
res, err := client.Connect(ctx, "127.0.0.1", port, nil)
require.NoError(t, err)
require.True(t, res.Success)
// Ping should skip the in-flight PUBLISH and successfully receive PINGRESP
alive, err := client.Ping(ctx)
require.NoError(t, err)
require.True(t, alive)
_ = client.Close()
}
-339
View File
@@ -1,339 +0,0 @@
package mqtt
import (
"encoding/binary"
"errors"
"fmt"
"io"
)
// Limits to prevent DoS and buffer overflows
const (
MaxStringLength = 65535
MaxRemainingLength = 268435455
MaxPacketSize = 10 * 1024 * 1024 // 10MB limit on incoming packets
)
// MQTT 3.1.1 Control Packet Types
const (
PacketConnect byte = 1
PacketConnack byte = 2
PacketPublish byte = 3
PacketPuback byte = 4
PacketPubrec byte = 5
PacketPubrel byte = 6
PacketPubcomp byte = 7
PacketSubscribe byte = 8
PacketSuback byte = 9
PacketUnsub byte = 10
PacketUnsuback byte = 11
PacketPingreq byte = 12
PacketPingresp byte = 13
PacketDisconnect byte = 14
)
// CONNACK Return Codes (MQTT 3.1.1 table 3.1)
const (
ConnAccepted byte = 0
ConnRefusedUnacceptableProtocol byte = 1
ConnRefusedIdentifierRejected byte = 2
ConnRefusedServerUnavailable byte = 3
ConnRefusedBadAuth byte = 4
ConnRefusedNotAuthorized byte = 5
)
// ConnackReason returns a human-readable description for CONNACK return codes.
func ConnackReason(code byte) string {
switch code {
case ConnAccepted:
return "connection accepted"
case ConnRefusedUnacceptableProtocol:
return "connection refused: unacceptable protocol version"
case ConnRefusedIdentifierRejected:
return "connection refused: identifier rejected"
case ConnRefusedServerUnavailable:
return "connection refused: server unavailable"
case ConnRefusedBadAuth:
return "connection refused: bad user name or password"
case ConnRefusedNotAuthorized:
return "connection refused: not authorized"
default:
return fmt.Sprintf("connection refused: return code %d", code)
}
}
// encodeVarLength encodes a length integer as an MQTT variable byte integer.
func encodeVarLength(length int) ([]byte, error) {
if length < 0 || length > MaxRemainingLength {
return nil, fmt.Errorf("remaining length %d exceeds MQTT max %d", length, MaxRemainingLength)
}
var out []byte
for {
encodedByte := byte(length % 128)
length /= 128
if length > 0 {
encodedByte |= 128
}
out = append(out, encodedByte)
if length == 0 {
break
}
}
return out, nil
}
// decodeVarLength decodes an MQTT variable byte integer from reader.
func decodeVarLength(r io.Reader) (int, error) {
multiplier := 1
value := 0
buf := make([]byte, 1)
for i := 0; i < 4; i++ {
if _, err := io.ReadFull(r, buf); err != nil {
return 0, err
}
encodedByte := buf[0]
value += int(encodedByte&127) * multiplier
if multiplier > 128*128*128 {
return 0, errors.New("malformed remaining length")
}
multiplier *= 128
if encodedByte&128 == 0 {
return value, nil
}
}
return 0, errors.New("malformed remaining length")
}
// encodeString encodes a UTF-8 string with a 2-byte big-endian length prefix.
func encodeString(s string) ([]byte, error) {
b := []byte(s)
if len(b) > MaxStringLength {
return nil, fmt.Errorf("string length %d exceeds MQTT max %d", len(b), MaxStringLength)
}
out := make([]byte, 2+len(b))
binary.BigEndian.PutUint16(out[0:2], uint16(len(b)))
copy(out[2:], b)
return out, nil
}
// buildConnectPacket constructs an MQTT 3.1.1 CONNECT packet.
func buildConnectPacket(clientID, username, password string, cleanSession bool, keepAlive uint16) ([]byte, error) {
var varHeader []byte
protoName, err := encodeString("MQTT")
if err != nil {
return nil, err
}
varHeader = append(varHeader, protoName...)
varHeader = append(varHeader, 0x04) // Protocol Level: 4 (MQTT 3.1.1)
// Connect Flags
var flags byte
if cleanSession {
flags |= 0x02
}
// Per MQTT 3.1.1 spec (MQTT-3.1.2-22): If the User Name Flag is 0, the Password Flag MUST be 0.
// If a password is provided, the User Name Flag MUST be set (encoding an empty username if omitted).
hasUsername := username != "" || password != ""
if hasUsername {
flags |= 0x80
}
if password != "" {
flags |= 0x40
}
varHeader = append(varHeader, flags)
// Keep Alive (2 bytes big endian)
ka := make([]byte, 2)
binary.BigEndian.PutUint16(ka, keepAlive)
varHeader = append(varHeader, ka...)
// Payload
var payload []byte
cidBytes, err := encodeString(clientID)
if err != nil {
return nil, fmt.Errorf("invalid clientID: %w", err)
}
payload = append(payload, cidBytes...)
if hasUsername {
userBytes, err := encodeString(username)
if err != nil {
return nil, fmt.Errorf("invalid username: %w", err)
}
payload = append(payload, userBytes...)
}
if password != "" {
passBytes, err := encodeString(password)
if err != nil {
return nil, fmt.Errorf("invalid password: %w", err)
}
payload = append(payload, passBytes...)
}
remainingLength := len(varHeader) + len(payload)
lenBytes, err := encodeVarLength(remainingLength)
if err != nil {
return nil, err
}
var pkt []byte
pkt = append(pkt, PacketConnect<<4)
pkt = append(pkt, lenBytes...)
pkt = append(pkt, varHeader...)
pkt = append(pkt, payload...)
return pkt, nil
}
// buildSubscribePacket constructs an MQTT 3.1.1 SUBSCRIBE packet for a single topic.
func buildSubscribePacket(packetID uint16, topic string, qos byte) ([]byte, error) {
var varHeader [2]byte
binary.BigEndian.PutUint16(varHeader[:], packetID)
topicBytes, err := encodeString(topic)
if err != nil {
return nil, fmt.Errorf("invalid topic: %w", err)
}
var payload []byte
payload = append(payload, topicBytes...)
payload = append(payload, qos&0x03)
remainingLength := len(varHeader) + len(payload)
lenBytes, err := encodeVarLength(remainingLength)
if err != nil {
return nil, err
}
var pkt []byte
// SUBSCRIBE fixed header flags must be 0010 (0x02)
pkt = append(pkt, (PacketSubscribe<<4)|0x02)
pkt = append(pkt, lenBytes...)
pkt = append(pkt, varHeader[:]...)
pkt = append(pkt, payload...)
return pkt, nil
}
// buildPublishPacket constructs an MQTT 3.1.1 PUBLISH packet.
func buildPublishPacket(topic string, payload []byte, qos byte, retain bool, packetID uint16) ([]byte, error) {
headerByte := (PacketPublish << 4)
if retain {
headerByte |= 0x01
}
headerByte |= (qos & 0x03) << 1
topicBytes, err := encodeString(topic)
if err != nil {
return nil, fmt.Errorf("invalid topic: %w", err)
}
var varHeader []byte
varHeader = append(varHeader, topicBytes...)
if qos > 0 {
pid := make([]byte, 2)
binary.BigEndian.PutUint16(pid, packetID)
varHeader = append(varHeader, pid...)
}
remainingLength := len(varHeader) + len(payload)
lenBytes, err := encodeVarLength(remainingLength)
if err != nil {
return nil, err
}
var pkt []byte
pkt = append(pkt, headerByte)
pkt = append(pkt, lenBytes...)
pkt = append(pkt, varHeader...)
pkt = append(pkt, payload...)
return pkt, nil
}
// buildPingreqPacket constructs an MQTT PINGREQ packet.
func buildPingreqPacket() []byte {
return []byte{PacketPingreq << 4, 0x00}
}
// buildDisconnectPacket constructs an MQTT DISCONNECT packet.
func buildDisconnectPacket() []byte {
return []byte{PacketDisconnect << 4, 0x00}
}
// errBrokenStream means a read failed after part of a packet was consumed, or
// a packet was refused, so the next read would start in the middle of a packet.
// The connection must not be used again.
var errBrokenStream = errors.New("MQTT stream is out of sync")
// readPacket reads a full MQTT packet from reader, rejecting packets over MaxPacketSize.
// Failures before the first byte is read are returned as they are. Failures after
// it wrap errBrokenStream.
func readPacket(r io.Reader) (headerByte byte, payload []byte, err error) {
var header [1]byte
if _, err := io.ReadFull(r, header[:]); err != nil {
return 0, nil, err
}
length, err := decodeVarLength(r)
if err != nil {
return 0, nil, fmt.Errorf("%w: %w", errBrokenStream, err)
}
if length > MaxPacketSize {
return 0, nil, fmt.Errorf("%w: MQTT packet length %d exceeds maximum allowed %d", errBrokenStream, length, MaxPacketSize)
}
buf := make([]byte, length)
if length > 0 {
if _, err := io.ReadFull(r, buf); err != nil {
return 0, nil, fmt.Errorf("%w: %w", errBrokenStream, err)
}
}
return header[0], buf, nil
}
// parseConnack decodes a CONNACK variable header.
func parseConnack(payload []byte) (sessionPresent bool, returnCode byte, err error) {
if len(payload) < 2 {
return false, 0, errors.New("short CONNACK response")
}
sessionPresent = (payload[0] & 0x01) != 0
returnCode = payload[1]
return sessionPresent, returnCode, nil
}
// parseSuback decodes a SUBACK packet payload.
func parseSuback(payload []byte) (packetID uint16, grantedQoS []byte, err error) {
if len(payload) < 3 {
return 0, nil, errors.New("short SUBACK response")
}
packetID = binary.BigEndian.Uint16(payload[0:2])
grantedQoS = payload[2:]
return packetID, grantedQoS, nil
}
// parsePublish decodes a PUBLISH packet.
func parsePublish(headerByte byte, body []byte) (*Message, error) {
qos := (headerByte >> 1) & 0x03
retain := (headerByte & 0x01) != 0
if len(body) < 2 {
return nil, errors.New("short PUBLISH packet")
}
topicLen := int(binary.BigEndian.Uint16(body[0:2]))
if len(body) < 2+topicLen {
return nil, errors.New("short PUBLISH topic")
}
topic := string(body[2 : 2+topicLen])
offset := 2 + topicLen
if qos > 0 {
if len(body) < offset+2 {
return nil, errors.New("short PUBLISH packet ID")
}
offset += 2
}
payload := string(body[offset:])
return &Message{
Topic: topic,
Payload: payload,
QoS: int(qos),
Retain: retain,
}, nil
}
-31
View File
@@ -6,7 +6,6 @@ import (
"crypto/tls"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"net"
"net/http"
@@ -21,8 +20,6 @@ import (
var (
defaultTimeout = time.Duration(5) * time.Second
errNoConnection = errors.New("no connection: open one with net.Open or net.OpenTLS")
)
// getTimeoutFromContext returns the configured TCP timeout or defaultTimeout.
@@ -274,14 +271,12 @@ type (
// const conn = net.Open('tcp', 'acme.com:80');
// ```
NetConn struct {
// conn is nil when no connection is open.
conn net.Conn
timeout time.Duration
}
)
// Close closes the connection.
// Close does nothing if no connection is open.
// @example
// ```javascript
// const net = require('nuclei/net');
@@ -289,13 +284,7 @@ type (
// conn.Close();
// ```
func (c *NetConn) Close() error {
if c.conn == nil {
return nil
}
err := c.conn.Close()
c.conn = nil
return err
}
@@ -332,10 +321,6 @@ func (c *NetConn) unsetDeadLine() {
// conn.SendArray(['hello', 'world']);
// ```
func (c *NetConn) SendArray(data []interface{}) error {
if c.conn == nil {
return errNoConnection
}
c.setDeadLine()
defer c.unsetDeadLine()
input := types.ToByteSlice(data)
@@ -357,10 +342,6 @@ func (c *NetConn) SendArray(data []interface{}) error {
// conn.SendHex('68656c6c6f');
// ```
func (c *NetConn) SendHex(data string) error {
if c.conn == nil {
return errNoConnection
}
c.setDeadLine()
defer c.unsetDeadLine()
bin, err := hex.DecodeString(data)
@@ -385,10 +366,6 @@ func (c *NetConn) SendHex(data string) error {
// conn.Send('hello');
// ```
func (c *NetConn) Send(data string) error {
if c.conn == nil {
return errNoConnection
}
c.setDeadLine()
defer c.unsetDeadLine()
bin := []byte(data)
@@ -412,10 +389,6 @@ func (c *NetConn) Send(data string) error {
// const data = conn.RecvFull(1024);
// ```
func (c *NetConn) RecvFull(N int) ([]byte, error) {
if c.conn == nil {
return []byte{}, errNoConnection
}
c.setDeadLine()
defer c.unsetDeadLine()
if N == 0 {
@@ -441,10 +414,6 @@ func (c *NetConn) RecvFull(N int) ([]byte, error) {
// log(`Received ${data.length} bytes from the server`)
// ```
func (c *NetConn) Recv(N int) ([]byte, error) {
if c.conn == nil {
return []byte{}, errNoConnection
}
c.setDeadLine()
defer c.unsetDeadLine()
if N == 0 {
-38
View File
@@ -406,41 +406,3 @@ func (f *fakeConn) RemoteAddr() net.Addr { return nil }
func (f *fakeConn) SetDeadline(time.Time) error { return nil }
func (f *fakeConn) SetReadDeadline(time.Time) error { return nil }
func (f *fakeConn) SetWriteDeadline(time.Time) error { return nil }
func TestNetConnWithoutConnection(t *testing.T) {
t.Parallel()
tests := map[string]func(t *testing.T) *NetConn{
"never opened": func(*testing.T) *NetConn { return new(NetConn) },
"already closed": func(t *testing.T) *NetConn {
t.Helper()
local, remote := net.Pipe()
t.Cleanup(func() { _ = remote.Close() })
conn := new(NetConn)
conn.conn = local
require.NoError(t, conn.Close())
return conn
},
}
for name, newConn := range tests {
t.Run(name, func(t *testing.T) {
t.Parallel()
conn := newConn(t)
require.NoError(t, conn.Close())
require.ErrorContains(t, conn.Send("x"), "no connection")
require.ErrorContains(t, conn.SendHex("78"), "no connection")
require.ErrorContains(t, conn.SendArray([]any{120}), "no connection")
_, err := conn.Recv(1)
require.ErrorContains(t, err, "no connection")
_, err = conn.RecvFull(1)
require.ErrorContains(t, err, "no connection")
})
}
}
+37 -36
View File
@@ -3,12 +3,12 @@ package postgres
import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
"net"
"net/url"
"strings"
"time"
"github.com/go-pg/pg/v10"
"github.com/praetorian-inc/fingerprintx/pkg/plugins"
postgres "github.com/praetorian-inc/fingerprintx/pkg/plugins/services/postgresql"
utils "github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
@@ -18,7 +18,7 @@ import (
type (
// PGClient is a client for Postgres database.
// Internally client uses lib/pq driver.
// Internally client uses go-pg/pg driver.
// @example
// ```javascript
// const postgres = require('nuclei/postgres');
@@ -124,7 +124,9 @@ func executeQuery(ctx context.Context, executionId string, host string, port int
return nil, protocolstate.ErrHostDenied.Msgf(host)
}
connStr := buildPostgresDSN(host, port, username, password, dbName)
target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
connStr := buildPostgresConnURL(username, password, target, dbName, executionId)
db, err := pgwrap.OpenDB(ctx, executionId, connStr)
if err != nil {
return nil, err
@@ -144,30 +146,17 @@ func executeQuery(ctx context.Context, executionId string, host string, port int
return resp, nil
}
// buildPostgresDSN returns a lib/pq key=value connection string. user, password, dbname, service and passfile
// are always set, also when empty: lib/pq fills unset values from the scanner's environment (PGUSER, PGPASSWORD,
// PGDATABASE), a service file (PGSERVICE) and ~/.pgpass (PGPASSFILE), which sends the operator's credentials to the target.
func buildPostgresDSN(host string, port int, username, password, dbName string) string {
quote := strings.NewReplacer(`\`, `\\`, `'`, `\'`)
params := []struct{ key, value string }{
{"host", host},
{"port", strconv.Itoa(port)},
{"user", username},
{"password", password},
{"dbname", dbName},
{"sslmode", "disable"},
{"connect_timeout", "10"},
{"service", ""},
{"options", ""},
{"application_name", ""},
// a path below a file never exists, so no password file is read
{"passfile", filepath.Join(os.DevNull, "pgpass")},
}
kvs := make([]string, 0, len(params))
for _, param := range params {
kvs = append(kvs, param.key+"='"+quote.Replace(param.value)+"'")
}
return strings.Join(kvs, " ")
func buildPostgresConnURL(username, password, target, dbName, executionId string) string {
values := url.Values{}
values.Set("sslmode", "disable")
values.Set("executionId", executionId)
return fmt.Sprintf("postgres://%s@%s/%s?%s",
url.UserPassword(username, password).String(),
target,
url.PathEscape(dbName),
values.Encode(),
)
}
// ConnectWithDB connects to Postgres database using given credentials and database name.
@@ -205,20 +194,32 @@ func connect(ctx context.Context, executionId string, host string, port int, use
return false, protocolstate.ErrHostDenied.Msgf(host)
}
execCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
execCtx, cancel := context.WithCancel(ctx)
defer cancel()
// lib/pq with every credential set, as go-pg replaces an empty user, password or database
// with PGUSER, PGPASSWORD or PGDATABASE from the scanner's environment
db, err := pgwrap.OpenDB(execCtx, executionId, buildPostgresDSN(host, port, username, password, dbName))
if err != nil {
return false, err
dialer := protocolstate.GetDialersWithId(executionId)
if dialer == nil {
return false, fmt.Errorf("dialers not initialized for %s", executionId)
}
db := pg.Connect(&pg.Options{
Addr: target,
User: username,
Password: password,
Database: dbName,
Dialer: func(dialCtx context.Context, network, addr string) (net.Conn, error) {
return dialer.Fastdialer.Dial(dialCtx, network, addr)
},
IdleCheckFrequency: -1,
}).WithTimeout(10 * time.Second)
defer func() {
_ = db.Close()
}()
_, err = db.ExecContext(execCtx, "select 1")
_, err := db.ExecContext(execCtx, "select 1")
if err != nil {
switch true {
case strings.Contains(err.Error(), "connect: connection refused"):
+56 -147
View File
@@ -1,165 +1,74 @@
package postgres
import (
"context"
"encoding/binary"
"io"
"net"
"os"
"path/filepath"
"net/url"
"strings"
"testing"
"github.com/lib/pq"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestBuildPostgresDSNEscapesDBName(t *testing.T) {
dbName := "testdb' sslrootcert='/etc/passwd' sslmode='verify-ca"
cfg, err := pq.NewConfig(buildPostgresDSN("127.0.0.1", 5432, "postgres", "x", dbName))
require.NoError(t, err)
func TestBuildPostgresConnectionURLDoesNotAllowDBNameQueryInjection(t *testing.T) {
dbName := "testdb?sslrootcert=/etc/passwd&sslmode=verify-ca&junk="
connStr := buildPostgresConnURL("postgres", "x", "127.0.0.1:5432", dbName, "exec-1")
require.Equal(t, dbName, cfg.Database)
require.Equal(t, pq.SSLModeDisable, cfg.SSLMode)
require.Empty(t, cfg.SSLRootCert)
require.Empty(t, cfg.Runtime)
}
u, err := url.Parse(connStr)
if err != nil {
t.Fatalf("parse connection URL: %v", err)
}
func TestBuildPostgresDSNEscapesCredentials(t *testing.T) {
username := `user:[email protected]' host='other`
password := `pa:ss@word?x\' sslmode='require`
cfg, err := pq.NewConfig(buildPostgresDSN("127.0.0.1", 5432, username, password, "postgres"))
require.NoError(t, err)
if got := strings.TrimPrefix(u.Path, "/"); got != dbName {
t.Fatalf("database name = %q, want %q", got, dbName)
}
require.Equal(t, username, cfg.User)
require.Equal(t, password, cfg.Password)
require.Equal(t, "127.0.0.1", cfg.Host)
require.Equal(t, uint16(5432), cfg.Port)
require.Equal(t, "postgres", cfg.Database)
require.Equal(t, pq.SSLModeDisable, cfg.SSLMode)
}
// setOperatorPostgresEnv sets every source lib/pq and go-pg read credentials from
func setOperatorPostgresEnv(t *testing.T) {
home := t.TempDir()
pgpass := filepath.Join(home, ".pgpass")
require.NoError(t, os.WriteFile(pgpass, []byte("*:*:*:*:OPERATOR-PGPASS-SECRET\n"), 0600))
serviceFile := filepath.Join(home, ".pg_service.conf")
require.NoError(t, os.WriteFile(serviceFile, []byte("[operator]\npassword=OPERATOR-SERVICE-SECRET\n"), 0600))
t.Setenv("HOME", home)
t.Setenv("PGPASSFILE", pgpass)
t.Setenv("PGSERVICEFILE", serviceFile)
t.Setenv("PGSERVICE", "operator")
t.Setenv("PGPASSWORD", "OPERATOR-ENV-PGPASSWORD")
t.Setenv("PGUSER", "operator-user")
t.Setenv("PGDATABASE", "operator-db")
t.Setenv("PGOPTIONS", "-c search_path=operator")
t.Setenv("PGAPPNAME", "operator-app")
}
func TestBuildPostgresDSNIgnoresOperatorEnvironment(t *testing.T) {
setOperatorPostgresEnv(t)
cfg, err := pq.NewConfig(buildPostgresDSN("127.0.0.1", 5432, "", "", ""))
require.NoError(t, err)
require.Empty(t, cfg.User)
require.Empty(t, cfg.Password)
require.Empty(t, cfg.Database)
require.Empty(t, cfg.Service)
require.Empty(t, cfg.Options)
require.Empty(t, cfg.ApplicationName)
}
type startup struct {
user, database, password string
}
// fakePostgres asks every client for a cleartext password, records it and rejects the login
func fakePostgres(t *testing.T) (int, <-chan startup) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = listener.Close() })
logins := make(chan startup, 16)
go func() {
for {
conn, err := listener.Accept()
if err != nil {
return
}
go func(conn net.Conn) {
defer func() { _ = conn.Close() }()
readMessage := func(typed bool) []byte {
header := make([]byte, 4)
if typed {
header = make([]byte, 5)
}
if _, err := io.ReadFull(conn, header); err != nil {
return nil
}
body := make([]byte, binary.BigEndian.Uint32(header[len(header)-4:])-4)
if _, err := io.ReadFull(conn, body); err != nil {
return nil
}
return body
}
params := readMessage(false)
if len(params) < 4 {
return
}
fields := strings.Split(string(params[4:]), "\x00")
var login startup
for i := 0; i+1 < len(fields); i += 2 {
switch fields[i] {
case "user":
login.user = fields[i+1]
case "database":
login.database = fields[i+1]
}
}
// AuthenticationCleartextPassword
if _, err := conn.Write([]byte{'R', 0, 0, 0, 8, 0, 0, 0, 3}); err != nil {
return
}
password := readMessage(true)
login.password = strings.TrimSuffix(string(password), "\x00")
logins <- login
errorBody := "SFATAL\x00C28P01\x00Mpassword authentication failed\x00\x00"
header := []byte{'E', 0, 0, 0, 0}
binary.BigEndian.PutUint32(header[1:], uint32(len(errorBody)+4))
_, _ = conn.Write(append(header, errorBody...))
}(conn)
values := u.Query()
if got := values.Get("sslmode"); got != "disable" {
t.Fatalf("sslmode = %q, want disable", got)
}
if got := values.Get("executionId"); got != "exec-1" {
t.Fatalf("executionId = %q, want exec-1", got)
}
deniedParams := []string{"sslrootcert", "sslcert", "sslkey", "service", "junk"}
for _, denied := range deniedParams {
if got := values.Get(denied); got != "" {
t.Fatalf("%s was injected with value %q", denied, got)
}
}()
return listener.Addr().(*net.TCPAddr).Port, logins
}
pqDSN, err := pq.ParseURL(connStr) //nolint:staticcheck // validates lib/pq URL parsing of the generated DSN.
if err != nil {
t.Fatalf("parse connection URL as lib/pq DSN: %v", err)
}
if !strings.Contains(pqDSN, "dbname='"+dbName+"'") {
t.Fatalf("lib/pq DSN = %q, want dbName preserved as dbname", pqDSN)
}
for _, denied := range deniedParams {
if strings.Contains(pqDSN, " "+denied+"=") {
t.Fatalf("%s was injected into lib/pq DSN %q", denied, pqDSN)
}
}
}
func TestPostgresEmptyCredentialsAreNotFilledFromEnvironment(t *testing.T) {
setOperatorPostgresEnv(t)
func TestBuildPostgresConnectionURLEscapesCredentials(t *testing.T) {
username := "user:[email protected]"
password := "pa:ss@word?x"
connStr := buildPostgresConnURL(username, password, "127.0.0.1:5432", "postgres", "exec-1")
executionID := t.Name()
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: executionID}))
t.Cleanup(func() { protocolstate.Close(executionID) })
port, logins := fakePostgres(t)
ctx := context.Background()
_, err := executeQuery(ctx, executionID, "127.0.0.1", port, "postgres", "", "postgres", "select 1")
require.Error(t, err)
require.Equal(t, startup{user: "postgres", database: "postgres"}, <-logins)
connected, err := connect(ctx, executionID, "127.0.0.1", port, "postgres", "", "postgres")
require.NoError(t, err)
require.False(t, connected)
require.Equal(t, startup{user: "postgres", database: "postgres"}, <-logins)
_, _ = connect(ctx, executionID, "127.0.0.1", port, "", "", "")
require.Equal(t, startup{}, <-logins)
_, _ = executeQuery(ctx, executionID, "127.0.0.1", port, "admin", "secret", "db", "select 1")
require.Equal(t, startup{user: "admin", database: "db", password: "secret"}, <-logins)
u, err := url.Parse(connStr)
if err != nil {
t.Fatalf("parse connection URL: %v", err)
}
if got := u.User.Username(); got != username {
t.Fatalf("username = %q, want %q", got, username)
}
if got, _ := u.User.Password(); got != password {
t.Fatalf("password = %q, want %q", got, password)
}
if got := u.Host; got != "127.0.0.1:5432" {
t.Fatalf("host = %q, want 127.0.0.1:5432", got)
}
if got := strings.TrimPrefix(u.Path, "/"); got != "postgres" {
t.Fatalf("database name = %q, want postgres", got)
}
}
+2 -12
View File
@@ -23,7 +23,6 @@ type (
// const client = new ssh.SSHClient();
// ```
SSHClient struct {
// connection is nil when no connection is open.
connection *ssh.Client
timeout time.Duration
}
@@ -159,8 +158,7 @@ func (c *SSHClient) Run(cmd string) (string, error) {
// Close closes the SSH connection and destroys the client
// Returns the success state and error. If error is not nil,
// state will be false. Close does nothing and returns true
// if no connection is open.
// state will be false
// @example
// ```javascript
// const ssh = require('nuclei/ssh');
@@ -169,17 +167,9 @@ func (c *SSHClient) Run(cmd string) (string, error) {
// const closed = client.Close();
// ```
func (c *SSHClient) Close() (bool, error) {
if c.connection == nil {
return true, nil
}
err := c.connection.Close()
c.connection = nil
if err != nil {
if err := c.connection.Close(); err != nil {
return false, err
}
return true, nil
}
-162
View File
@@ -1,162 +0,0 @@
package ssh_test
//nolint:depguard // No depguard rules are configured, so its default list allows only the standard library.
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"errors"
"net"
"strings"
"testing"
libssh "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/ssh"
"github.com/zmap/zgrab2/lib/ssh"
)
const (
testUser = "user"
testPassword = "secret"
)
var errAccessDenied = errors.New("access denied")
func TestCloseWithoutConnection(t *testing.T) {
t.Parallel()
host, port := startServer(t)
tests := map[string]func(t *testing.T, client *libssh.SSHClient){
"never connected": func(*testing.T, *libssh.SSHClient) {},
"failed connect": func(t *testing.T, client *libssh.SSHClient) {
t.Helper()
_, err := client.Connect(executionContext(t), host, port, testUser, "wrong")
if err == nil {
t.Fatal("Connect() with a wrong password succeeded")
}
},
"already closed": func(t *testing.T, client *libssh.SSHClient) {
t.Helper()
connect(t, client, host, port)
closeClient(t, client)
},
}
for name, setup := range tests {
t.Run(name, func(t *testing.T) {
t.Parallel()
client := new(libssh.SSHClient)
setup(t, client)
closeClient(t, client)
_, err := client.Run("id")
if err == nil || !strings.Contains(err.Error(), "no connection") {
t.Errorf("Run() error = %v, want no connection", err)
}
})
}
}
func TestCloseConnected(t *testing.T) {
t.Parallel()
host, port := startServer(t)
client := new(libssh.SSHClient)
connect(t, client, host, port)
closeClient(t, client)
}
func connect(t *testing.T, client *libssh.SSHClient, host string, port int) {
t.Helper()
_, err := client.Connect(executionContext(t), host, port, testUser, testPassword)
if err != nil {
t.Fatalf("Connect() error = %v", err)
}
}
func closeClient(t *testing.T, client *libssh.SSHClient) {
t.Helper()
closed, err := client.Close()
if err != nil || !closed {
t.Fatalf("Close() = %v, %v; want true, nil", closed, err)
}
}
func executionContext(t *testing.T) context.Context {
t.Helper()
return context.WithValue(t.Context(), "executionId", t.Name()) //nolint:staticcheck // Connect reads this string key.
}
// startServer serves SSH on a loopback port and accepts only
// testUser with testPassword. It rejects every channel.
func startServer(t *testing.T) (string, int) {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
signer, err := ssh.NewSignerFromKey(key)
if err != nil {
t.Fatal(err)
}
config := new(ssh.ServerConfig)
config.PasswordCallback = checkPassword
config.AddHostKey(signer)
listener, err := new(net.ListenConfig).Listen(t.Context(), "tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = listener.Close() })
go func() {
for {
conn, err := listener.Accept()
if err != nil {
return
}
go serve(conn, config)
}
}()
addr, ok := listener.Addr().(*net.TCPAddr)
if !ok {
t.Fatalf("listener address %v is not TCP", listener.Addr())
}
return addr.IP.String(), addr.Port
}
func checkPassword(conn ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) {
if conn.User() != testUser || string(password) != testPassword {
return nil, errAccessDenied
}
return new(ssh.Permissions), nil
}
func serve(conn net.Conn, config *ssh.ServerConfig) {
defer func() { _ = conn.Close() }()
_, chans, reqs, err := ssh.NewServerConn(conn, config)
if err != nil {
return
}
go ssh.DiscardRequests(reqs)
for ch := range chans {
_ = ch.Reject(ssh.Prohibited, "test server")
}
}
+1 -50
View File
@@ -1,10 +1,6 @@
package output
import (
"fmt"
json "github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
)
import json "github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
// formatJSON formats the output for json based formatting
func (w *StandardWriter) formatJSON(output *ResultEvent) ([]byte, error) {
@@ -12,50 +8,5 @@ func (w *StandardWriter) formatJSON(output *ResultEvent) ([]byte, error) {
output.Request = ""
output.Response = ""
}
if len(output.Info.Metadata) > 0 {
output.Info.Metadata = sanitizeMetadataForJSON(output.Info.Metadata)
}
if len(output.Metadata) > 0 {
output.Metadata = sanitizeMetadataForJSON(output.Metadata)
}
return json.Marshal(output)
}
// sanitizeMetadataForJSON normalizes any map[interface{}]interface{} or nested
// collections in metadata to map[string]interface{}, ensuring JSON encoders
// (such as sonic or standard encoding/json) can serialize them without error (#7971).
func sanitizeMetadataForJSON(m map[string]interface{}) map[string]interface{} {
if len(m) == 0 {
return m
}
sanitized := sanitizeJSONValue(m)
if res, ok := sanitized.(map[string]interface{}); ok {
return res
}
return m
}
func sanitizeJSONValue(v interface{}) interface{} {
switch typed := v.(type) {
case map[interface{}]interface{}:
converted := make(map[string]interface{}, len(typed))
for k, val := range typed {
converted[fmt.Sprint(k)] = sanitizeJSONValue(val)
}
return converted
case map[string]interface{}:
converted := make(map[string]interface{}, len(typed))
for k, val := range typed {
converted[k] = sanitizeJSONValue(val)
}
return converted
case []interface{}:
converted := make([]interface{}, len(typed))
for i, val := range typed {
converted[i] = sanitizeJSONValue(val)
}
return converted
default:
return v
}
}
-30
View File
@@ -209,33 +209,3 @@ type testWriteCloser struct {
func (w testWriteCloser) Close() error {
return nil
}
func TestFormatJSONSanitizesNestedMetadataMaps(t *testing.T) {
w, err := NewStandardWriter(&types.Options{})
require.NoError(t, err)
event := &ResultEvent{
TemplateID: "nested-metadata",
Type: "file",
Host: "target",
Matched: "target/input.txt",
Info: model.Info{
Name: "Nested metadata",
Metadata: map[string]interface{}{
"labels": map[interface{}]interface{}{
"en": "Hello",
"nested_list": []interface{}{
map[interface{}]interface{}{
"inner_key": "inner_val",
},
},
},
},
},
}
data, err := w.formatJSON(event)
require.NoError(t, err, "formatJSON should not fail on nested map[interface{}]interface{} (#7971)")
require.Contains(t, string(data), `"labels":{"en":"Hello","nested_list":[{"inner_key":"inner_val"}]}`)
}
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"strings"
"time"
"github.com/alecthomas/chroma/v2/quick"
"github.com/alecthomas/chroma/quick"
"github.com/ditashi/jsbeautifier-go/jsbeautifier"
"github.com/projectdiscovery/goja"
+82 -107
View File
@@ -2,7 +2,6 @@ package expressions
import (
"fmt"
"slices"
"strings"
"github.com/projectdiscovery/govaluate"
@@ -49,75 +48,55 @@ func EvaluateWithOptions(data string, base map[string]interface{}, options *type
return evaluate(data, base, options)
}
// evaluate renders data, the template text, with base, the data values.
//
// Expressions are located in data before anything is substituted, and each
// result is written at its expression's position. The text between
// expressions only has its placeholders replaced. Nothing that evaluate
// writes is scanned again, so a value or a helper result that looks like an
// expression stays literal and never takes another expression's result. On
// error, data is returned unchanged.
//
// Expressions can be simple (base keys) or complex (helper functions and base
// keys). Literals such as {{2+2}} are not expressions.
func evaluate(data string, base map[string]interface{}, options *types.Options) (string, error) {
spans := findExpressionSpans(data, marker.ParenthesisOpen, marker.ParenthesisClose, base)
if len(spans) == 0 {
return replacer.Replace(data, base), nil
}
expressions := FindExpressions(data, marker.ParenthesisOpen, marker.ParenthesisClose, base)
var (
builder strings.Builder
last int
)
// replace simple placeholders (key => value) MarkerOpen + key + MarkerClose and General + key + General to value
data = replacer.Replace(data, base)
builder.Grow(len(data))
// expressions can be:
// - simple: containing base values keys (variables)
// - complex: containing helper functions [ + variables]
// literals like {{2+2}} are not considered expressions
for _, expression := range expressions {
originalExpression := expression
// data has already had simple placeholders replaced; keep the same
// marker shape for output replacement, but never compile this string.
replacedExpression := replacer.Replace(expression, base)
expression = replaceStringPlaceholders(expression, base)
for _, span := range spans {
result, err := evaluateExpression(span.expression, base, options)
// turns expressions (either helper functions+base values or base values)
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions)
if err != nil {
return data, err
return data, fmt.Errorf("failed to compile expression %q: %w", originalExpression, err)
}
builder.WriteString(replacer.Replace(data[last:span.start], base))
builder.WriteString(result)
last = span.end
}
builder.WriteString(replacer.Replace(data[last:], base))
return builder.String(), nil
}
// evaluateExpression compiles and evaluates one template-authored expression,
// the text between its markers, and returns the result as text.
func evaluateExpression(expression string, base map[string]any, options *types.Options) (string, error) {
source := replaceStringPlaceholders(expression, base)
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(source, dsl.HelperFunctions)
if err != nil {
return "", fmt.Errorf("failed to compile expression %q: %w", expression, err)
}
result, err := dsl.EvalWithOptions(compiled, base, options)
if err != nil {
return "", fmt.Errorf("failed to evaluate expression %q: %w", expression, err)
}
// Preserve unresolved markers only when a helper call would otherwise
// hide them from downstream validation. Plain expressions such as
// comparisons should evaluate normally.
if markers := unresolvedVarMarkers(compiled.Vars(), base); markers != "" {
usesFunctions := slices.ContainsFunc(compiled.Tokens(), func(token govaluate.ExpressionToken) bool {
return token.Kind == govaluate.FUNCTION
})
if usesFunctions && ContainsUnresolvedVariables(fmt.Sprint(result)) == nil {
return markers, nil
result, err := dsl.EvalWithOptions(compiled, base, options)
if err != nil {
return data, fmt.Errorf("failed to evaluate expression %q: %w", originalExpression, err)
}
}
return types.ToString(result), nil
replacement := result
// Preserve unresolved markers only when a helper call would otherwise
// hide them from downstream validation. Plain expressions such as
// comparisons should evaluate normally.
if markers := unresolvedVarMarkers(compiled.Vars(), base); markers != "" {
usesFunctions := false
for _, token := range compiled.Tokens() {
if token.Kind == govaluate.FUNCTION {
usesFunctions = true
break
}
}
if usesFunctions && ContainsUnresolvedVariables(fmt.Sprint(result)) == nil {
replacement = markers
}
}
// replace incrementally
data = replacer.ReplaceOne(data, replacedExpression, replacement)
}
return data, nil
}
func replaceStringPlaceholders(expression string, base map[string]interface{}) string {
@@ -227,63 +206,59 @@ func EscapeStringValue(value string, quote byte) string {
// maxIterations to avoid infinite loop
const maxIterations = 250
// FindExpressions returns the expressions in data, the text between each
// matching OpenMarker and CloseMarker pair, in order.
func FindExpressions(data, OpenMarker, CloseMarker string, base map[string]interface{}) []string {
spans := findExpressionSpans(data, OpenMarker, CloseMarker, base)
if len(spans) == 0 {
return nil
}
var (
iterations int
exps []string
)
for iterations <= maxIterations {
// check if we reached the maximum number of iterations
exps := make([]string, 0, len(spans))
for _, span := range spans {
exps = append(exps, span.expression)
}
return exps
}
// expressionSpan is one expression in template text: data[start:end] is the
// whole marker and expression is the text between its markers.
type expressionSpan struct {
start, end int
expression string
}
// findExpressionSpans finds the expressions in data and where each one sits.
// For each open marker, it takes the nearest close marker that makes the text
// between them an expression. When none does, the search resumes right after
// the open marker.
func findExpressionSpans(data, openMarker, closeMarker string, base map[string]any) []expressionSpan {
var spans []expressionSpan
for offset, iterations := 0, 0; iterations <= maxIterations; iterations++ {
openIndex := strings.Index(data[offset:], openMarker)
if openIndex < 0 {
iterations++
// attempt to find open markers
indexOpenMarker := strings.Index(data, OpenMarker)
// exits if not found
if indexOpenMarker < 0 {
break
}
start := offset + openIndex
inner := start + len(openMarker)
offset = inner
indexOpenMarkerOffset := indexOpenMarker + len(OpenMarker)
for search := inner; ; {
closeIndex := stringsutil.IndexAt(data, closeMarker, search)
if closeIndex < 0 {
break
shouldSearchCloseMarker := true
closeMarkerFound := false
innerData := data
var potentialMatch string
var indexCloseMarker, indexCloseMarkerOffset int
skip := indexOpenMarkerOffset
for shouldSearchCloseMarker {
// attempt to find close marker
indexCloseMarker = stringsutil.IndexAt(innerData, CloseMarker, skip)
// if no close markers are found exit
if indexCloseMarker < 0 {
shouldSearchCloseMarker = false
continue
}
indexCloseMarkerOffset = indexCloseMarker + len(CloseMarker)
search = closeIndex + len(closeMarker)
if expression := data[inner:closeIndex]; isExpression(expression, base) {
spans = append(spans, expressionSpan{start: start, end: search, expression: expression})
offset = search
break
potentialMatch = innerData[indexOpenMarkerOffset:indexCloseMarker]
if isExpression(potentialMatch, base) {
closeMarkerFound = true
shouldSearchCloseMarker = false
exps = append(exps, potentialMatch)
} else {
skip = indexCloseMarkerOffset
}
}
}
return spans
if closeMarkerFound {
// move after the close marker
data = data[indexCloseMarkerOffset:]
} else {
// move after the open marker
data = data[indexOpenMarkerOffset:]
}
}
return exps
}
func isExpression(data string, base map[string]interface{}) bool {
@@ -109,71 +109,6 @@ func TestEvaluateDoesNotReinterpretResolvedValues(t *testing.T) {
}
}
func TestEvaluateReplacesExpressionsInTheirOwnPosition(t *testing.T) {
t.Parallel()
// tpl and echo stand for response values that repeat an expression's text.
values := map[string]any{
"seed": "hello",
"tpl": "{{to_upper(seed)}}",
"echo": "{{base64('127.0.0.1')}}",
"Host": "127.0.0.1",
}
items := []struct {
name string
input string
expected string
}{
{
name: "resolved value echoes a later expression",
input: "[{{tpl}}] {{to_upper(seed)}}",
expected: "[{{to_upper(seed)}}] HELLO",
},
{
name: "general marker value echoes a later expression",
input: "[§tpl§] {{to_upper(seed)}}",
expected: "[{{to_upper(seed)}}] HELLO",
},
{
name: "resolved value echoes an expression with substituted arguments",
input: "{{echo}} {{base64('{{Host}}')}}",
expected: "{{base64('127.0.0.1')}} MTI3LjAuMC4x",
},
{
name: "helper result echoes a later expression",
input: "{{concat('{{', 'to_upper(seed)', '}}')}} {{to_upper(seed)}}",
expected: "{{to_upper(seed)}} HELLO",
},
{
name: "repeated expression is evaluated in each position",
input: "{{to_upper(seed)}}-{{to_upper(seed)}}",
expected: "HELLO-HELLO",
},
}
for _, item := range items {
t.Run(item.name, func(t *testing.T) {
t.Parallel()
value, err := Evaluate(item.input, values)
require.NoError(t, err)
require.Equal(t, item.expected, value)
})
}
}
func TestEvaluateDoesNotResolveGeneralMarkersFromResolvedValues(t *testing.T) {
t.Parallel()
value, err := Evaluate("Authorization: {{body}}", map[string]any{
"body": "§secret_token§",
"secret_token": "top-secret-cia-mi6-kgb-mossad-classified",
})
require.NoError(t, err)
require.Equal(t, "Authorization: §secret_token§", value)
}
func TestEvaluateDoesNotExecuteHelpersFromResolvedValues(t *testing.T) {
var calls int
@@ -271,18 +206,6 @@ func TestEvaluateReturnsErrorForInvalidTemplateExpression(t *testing.T) {
require.ErrorContains(t, err, `failed to evaluate expression "base64()"`)
}
func TestEvaluateReturnsInputOnError(t *testing.T) {
t.Parallel()
input := "{{secret_token}} {{base64()}}"
value, err := Evaluate(input, map[string]any{
"secret_token": "top-secret-cia-mi6-kgb-mossad-classified",
})
require.Error(t, err)
require.Equal(t, input, value)
}
func TestEvaluateErrorDoesNotLeakResolvedValues(t *testing.T) {
_, err := Evaluate("{{base64('{{secret_token}}', 'extra')}}", map[string]interface{}{
"secret_token": "top-secret-cia-mi6-kgb-mossad-classified",
@@ -115,26 +115,6 @@ func ContainsVariablesWithIgnoreList(skipNames map[string]interface{}, items ...
return nil
}
// ReferencesVariables reports whether data contains a {{...}} marker that
// refers to a variable in names, either alone or inside an expression. It
// parses the marker but does not evaluate it.
func ReferencesVariables(data string, names map[string]interface{}) bool {
for _, match := range unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) {
expr, err := govaluate.NewEvaluableExpressionWithFunctions(match[1], dsl.HelperFunctions)
if err != nil {
continue
}
for _, name := range expr.Vars() {
if _, ok := names[name]; ok {
return true
}
}
}
return false
}
func hasLiteralsOnly(data string) bool {
expr, err := govaluate.NewEvaluableExpressionWithFunctions(data, dsl.HelperFunctions)
if err != nil {
@@ -44,26 +44,6 @@ func TestUnresolvedVariablesCheck(t *testing.T) {
}
}
func TestReferencesVariables(t *testing.T) {
names := map[string]interface{}{"Hostname": "", "Port": ""}
tests := []struct {
data string
want bool
}{
{data: "user@{{Hostname}}", want: true},
{data: "{{to_lower(Hostname)}}:{{Port}}", want: true},
{data: "{{Hostname_suffix}}", want: false},
{data: "{{secret}}-{{to_upper(seed)}}", want: false},
{data: "{{7*7}}", want: false},
{data: "Hostname", want: false},
}
for _, test := range tests {
require.Equal(t, test.want, ReferencesVariables(test.data, names), test.data)
}
}
func TestUnresolvedVariablesCheckDoesNotExecuteHelpers(t *testing.T) {
var calls int
withTestHelperFunction(t, "test_side_effect", func(args ...interface{}) (interface{}, error) {
@@ -35,7 +35,5 @@ func Init(options *types.Options) error {
}
func Close(executionId string) {
dnsclientpool.Close(executionId)
rdapclientpool.Close(executionId)
protocolstate.Close(executionId)
}
+13 -172
View File
@@ -3,186 +3,27 @@ package replacer
import (
"strings"
"github.com/projectdiscovery/fasttemplate"
convUtil "github.com/projectdiscovery/utils/conversion"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/marker"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
)
// Replace substitutes each {{key}} and §key§ placeholder in template whose key
// is in values, scanning template once from left to right.
//
// Values are data: Replace never scans an inserted value for placeholders, so
// a value cannot pull in another value, alone or together with the template
// text around it. Text that does not form a known placeholder is kept as is.
// Replace replaces placeholders in template with values on the fly.
func Replace(template string, values map[string]interface{}) string {
if len(values) == 0 {
return template
valuesMap := make(map[string]interface{}, len(values))
for k, v := range values {
// fasttemplate copies string values to []byte on every write, so
// pass a read-only []byte view instead
valuesMap[k] = convUtil.Bytes(types.ToString(v))
}
scan := newScanner(template, values)
start := scan.nextPlaceholder(0)
if start < 0 {
return template
}
var builder strings.Builder
builder.Grow(len(template))
written := 0 // template[:written] is already in builder
for pos := start; pos >= 0; {
value, end, ok := scan.resolve(pos)
if !ok {
// Not a known placeholder: keep it as text and look again one
// byte later, so a placeholder nested in it still resolves.
pos = scan.nextPlaceholder(pos + 1)
continue
}
builder.WriteString(template[written:pos])
builder.WriteString(value)
written = end
pos = scan.nextPlaceholder(end)
}
builder.WriteString(template[written:])
return builder.String()
replaced := fasttemplate.ExecuteStringStd(template, marker.ParenthesisOpen, marker.ParenthesisClose, valuesMap)
final := fasttemplate.ExecuteStringStd(replaced, marker.General, marker.General, valuesMap)
return final
}
// directLookupLimit is the longest candidate key that resolve looks up in
// values without first comparing it to the longest key there.
const directLookupLimit = 32
// scanner finds the placeholders of one template. Replace asks for positions
// that never decrease, so each finder searches every byte at most once. A
// candidate key longer than every key in values is rejected without hashing
// it. Together, these keep Replace linear in the template length.
type scanner struct {
template string
values map[string]any
longestKey int // length of the longest key in values; -1 until needed
braceOpen, braceClose finder
generalOpen, generalClose finder
}
func newScanner(template string, values map[string]any) scanner {
return scanner{
template: template,
values: values,
longestKey: -1,
braceOpen: newFinder(marker.ParenthesisOpen),
braceClose: newFinder(marker.ParenthesisClose),
generalOpen: newFinder(marker.General),
generalClose: newFinder(marker.General),
}
}
// nextPlaceholder returns the index of the next {{ or § at or after from that
// can still be closed, or -1 when there is none. Once no closing marker of a
// kind is left, no later opening marker of that kind can close.
func (s *scanner) nextPlaceholder(from int) int {
brace, general := -1, -1
if !s.braceClose.exhausted() {
brace = s.braceOpen.index(s.template, from)
}
if !s.generalClose.exhausted() {
general = s.generalOpen.index(s.template, from)
}
if brace < 0 || (general >= 0 && general < brace) {
return general
}
return brace
}
// resolve returns the value of the placeholder that starts at index start,
// and the index just past it, when its key is in values.
func (s *scanner) resolve(start int) (string, int, bool) {
open, closing := marker.ParenthesisOpen, &s.braceClose
if strings.HasPrefix(s.template[start:], marker.General) {
open, closing = marker.General, &s.generalClose
}
keyStart := start + len(open)
keyEnd := closing.index(s.template, keyStart)
if keyEnd < 0 {
return "", 0, false
}
key := s.template[keyStart:keyEnd]
if len(key) > directLookupLimit && len(key) > s.longestKeyLen() {
return "", 0, false
}
value, ok := s.values[key]
if !ok {
return "", 0, false
}
return types.ToString(value), keyEnd + len(closing.needle), true
}
// longestKeyLen returns the length of the longest key in values.
func (s *scanner) longestKeyLen() int {
if s.longestKey < 0 {
s.longestKey = 0
for key := range s.values {
s.longestKey = max(s.longestKey, len(key))
}
}
return s.longestKey
}
// finder finds one marker in a template. Its callers pass positions that
// never decrease, so it keeps the last match until the scan passes it, and a
// failed search answers every later call.
type finder struct {
needle string
next int // last match, or -1 when none is left
searched bool
}
func newFinder(needle string) finder {
return finder{needle: needle, next: -1, searched: false}
}
// index returns the index of the first needle at or after from, or -1.
func (f *finder) index(template string, from int) int {
if f.searched && (f.next < 0 || f.next >= from) {
return f.next
}
f.searched = true
f.next = strings.Index(template[from:], f.needle)
if f.next >= 0 {
f.next += from
}
return f.next
}
// exhausted reports whether a search found no match left in the template.
func (f *finder) exhausted() bool {
return f.searched && f.next < 0
}
// ReplaceOne replaces the first {{key}} and the first §key§ in template with
// value.
//
// Deprecated: ReplaceOne finds the placeholder by searching template, so when
// template already holds substituted values it can replace text that a value
// introduced instead of the placeholder the caller meant. Render by position
// with Replace instead.
// Replace replaces one placeholder in template with one value on the fly.
func ReplaceOne(template string, key string, value interface{}) string {
data := replaceOneWithMarkers(template, key, value, marker.ParenthesisOpen, marker.ParenthesisClose)
return replaceOneWithMarkers(data, key, value, marker.General, marker.General)
@@ -1,39 +0,0 @@
package replacer_test
import (
"strconv"
"strings"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
)
func BenchmarkReplace(b *testing.B) {
// An HTTP request renders with about 60 values, enough that map lookups
// hash their keys.
values := map[string]any{"Hostname": "example.com", "token": "abc"}
for i := range 60 {
values["var"+strconv.Itoa(i)] = i
}
header := "POST /?t={{token}} HTTP/1.1\r\nHost: {{Hostname}}\r\n\r\n"
for _, item := range []struct {
name string
template string
}{
{name: "raw request", template: header},
{name: "64KB JSON body", template: header + strings.Repeat(`{"k": "v", "n": [1, 2]} `, 2730)},
{name: "64KB non-ASCII body", template: header + strings.Repeat("Привет, мир! ", 2730)},
{name: "64KB unclosed markers", template: strings.Repeat(`{{`, 32768)},
{name: "64KB markers closed at the end", template: strings.Repeat(`{`, 65536) + `}}`},
} {
b.Run(item.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
replacer.Replace(item.template, values)
}
})
}
}
@@ -73,24 +73,6 @@ func TestReplacerReplace(t *testing.T) {
values: map[string]interface{}{"var1": "first variable", "var2": "second variable"},
expected: "first variable and first variable and another second variable",
},
{
name: "Extra braces around a placeholder",
template: "${{{num}}*{{{num}}",
values: map[string]any{"num": "7"},
expected: "${7*{7",
},
{
name: "Unknown general marker before a known one",
template: "§unknown§var1§",
values: map[string]any{"var1": "variable 1"},
expected: "§unknownvariable 1",
},
{
name: "Unclosed markers",
template: "{{{{ §§ {{var1}",
values: map[string]any{"var1": "variable 1"},
expected: "{{{{ §§ {{var1}",
},
}
for _, test := range tests {
@@ -100,48 +82,6 @@ func TestReplacerReplace(t *testing.T) {
}
}
func TestReplacerReplaceDoesNotRescanValues(t *testing.T) {
t.Parallel()
values := map[string]any{
"secret": "leaked",
"token": "leaked",
"general": "§secret§",
"parenthesis": "{{secret}}",
"prefix": "§",
}
tests := []struct {
name string
template string
expected string
}{
{
name: "general marker inside a parenthesis value",
template: "{{general}}",
expected: "§secret§",
},
{
name: "parenthesis marker inside a general value",
template: "§parenthesis§",
expected: "{{secret}}",
},
{
name: "value completes a marker with the template text after it",
template: "{{prefix}}token§",
expected: "§token§",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
require.Equal(t, test.expected, Replace(test.template, values))
})
}
}
func TestReplacerReplaceOne(t *testing.T) {
tests := []struct {
name string
+3 -2
View File
@@ -15,6 +15,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool"
"github.com/projectdiscovery/retryabledns"
fileutil "github.com/projectdiscovery/utils/file"
)
@@ -86,7 +87,7 @@ type Request struct {
generator *generators.PayloadGenerator
CompiledOperators *operators.Operators `yaml:"-" json:"-"`
dnsClient *dnsclientpool.Client
dnsClient *retryabledns.Client
options *protocols.ExecutorOptions
// cache any variables that may be needed for operation.
@@ -204,7 +205,7 @@ func resolverHost(resolver string) string {
return r
}
func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadata map[string]interface{}) (*dnsclientpool.Client, error) {
func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadata map[string]interface{}) (*retryabledns.Client, error) {
dnsClientOptions := &dnsclientpool.Configuration{
Retries: request.Retries,
Proxy: options.Options.AliveSocksProxy,
+7 -12
View File
@@ -12,9 +12,9 @@ import (
)
var (
clientPool *mapsutil.SyncLockMap[string, *Client]
clientPool *mapsutil.SyncLockMap[string, *retryabledns.Client]
normalClient *Client
normalClient *retryabledns.Client
m sync.Mutex
)
@@ -46,21 +46,20 @@ func resolversFor(options *types.Options) []string {
return defaultResolvers
}
func cachedClient(resolvers []string) (*Client, error) {
func cachedClient(resolvers []string) (*retryabledns.Client, error) {
m.Lock()
defer m.Unlock()
if clientPool == nil {
clientPool = mapsutil.NewSyncLockMap[string, *Client]()
clientPool = mapsutil.NewSyncLockMap[string, *retryabledns.Client]()
}
key := strings.Join(resolvers, ",")
if client, ok := clientPool.Get(key); ok {
return client, nil
}
dnsClient, err := retryabledns.New(resolvers, 1)
client, err := retryabledns.New(resolvers, 1)
if err != nil {
return nil, err
}
client := &Client{Client: dnsClient}
_ = clientPool.Set(key, client)
return client, nil
}
@@ -89,13 +88,10 @@ func (c *Configuration) Hash() string {
}
// Get creates or gets a client for the protocol based on custom configuration
func Get(options *types.Options, configuration *Configuration) (*Client, error) {
func Get(options *types.Options, configuration *Configuration) (*retryabledns.Client, error) {
if (configuration.Retries <= 1) && len(configuration.Resolvers) == 0 {
return cachedClient(resolversFor(options))
}
if len(options.InternalResolversList) == 0 && hasDohResolver(configuration.Resolvers) {
return getPolicyClient(options, configuration)
}
hash := configuration.Hash()
if client, ok := clientPool.Get(hash); ok {
return client, nil
@@ -107,7 +103,7 @@ func Get(options *types.Options, configuration *Configuration) (*Client, error)
} else if len(configuration.Resolvers) > 0 {
resolvers = configuration.Resolvers
}
dnsClient, err := retryabledns.NewWithOptions(retryabledns.Options{
client, err := retryabledns.NewWithOptions(retryabledns.Options{
BaseResolvers: resolvers,
MaxRetries: configuration.Retries,
Proxy: options.AliveSocksProxy,
@@ -115,7 +111,6 @@ func Get(options *types.Options, configuration *Configuration) (*Client, error)
if err != nil {
return nil, errors.Wrap(err, "could not create dns client")
}
client := &Client{Client: dnsClient}
_ = clientPool.Set(hash, client)
return client, nil
-171
View File
@@ -1,171 +0,0 @@
package dnsclientpool
import (
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
"strings"
"sync"
"sync/atomic"
"github.com/miekg/dns"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/retryabledns"
"github.com/projectdiscovery/retryabledns/doh"
)
// Client sends the DNS queries of a template. It sends queries for template
// DoH resolvers itself, because the retryabledns DoH client dials and follows
// redirects without the network policy of the scan.
type Client struct {
*retryabledns.Client
resolvers []resolver
retries int
index uint32
doh *doh.Client
transport *http.Transport
}
// resolver is one template resolver. url is empty for a resolver that
// retryabledns queries over UDP, TCP or DoT.
type resolver struct {
url string
method doh.Method
}
var (
// policyClients holds the clients with DoH resolvers of each execution,
// because each one dials with the dialer of its execution.
policyClients = map[string]map[string]*Client{}
policyMu sync.Mutex
)
func hasDohResolver(resolvers []string) bool {
for _, r := range resolvers {
if strings.HasPrefix(r, "doh:") {
return true
}
}
return false
}
// parseDohResolver returns the URL and method of a resolver such as
// doh:https://dns.example/dns-query:get, in the same way as retryabledns.
func parseDohResolver(r string) (resolver, bool) {
if !strings.HasPrefix(r, "doh:") {
return resolver{}, false
}
u := strings.TrimPrefix(r, "doh:")
method := doh.MethodPost
for _, suffix := range []string{":get", ":post", ":jsonapi"} {
if strings.HasSuffix(u, suffix) {
u = strings.TrimSuffix(u, suffix)
if suffix == ":get" {
method = doh.MethodGet
}
break
}
}
return resolver{url: u, method: method}, true
}
func getPolicyClient(options *types.Options, configuration *Configuration) (*Client, error) {
executionID := options.ExecutionId
hash := configuration.Hash()
policyMu.Lock()
defer policyMu.Unlock()
if client, ok := policyClients[executionID][hash]; ok {
return client, nil
}
client := &Client{retries: configuration.Retries}
var networkResolvers []string
for _, r := range configuration.Resolvers {
if parsed, ok := parseDohResolver(r); ok {
client.resolvers = append(client.resolvers, parsed)
continue
}
client.resolvers = append(client.resolvers, resolver{})
networkResolvers = append(networkResolvers, r)
}
// Trace and the non DoH resolvers still go through retryabledns.
if len(networkResolvers) == 0 {
networkResolvers = defaultResolvers
}
dnsClient, err := retryabledns.NewWithOptions(retryabledns.Options{
BaseResolvers: networkResolvers,
MaxRetries: 1,
Proxy: options.AliveSocksProxy,
})
if err != nil {
return nil, errors.Wrap(err, "could not create dns client")
}
client.Client = dnsClient
// The scan dialer checks every address it connects to, so redirects
// and names that resolve to a denied address are refused too. It also
// goes through the scan proxy when one is set.
client.transport = &http.Transport{
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
dialers := protocolstate.GetDialersWithId(executionID)
if dialers == nil || dialers.Fastdialer == nil {
return nil, fmt.Errorf("dialers not initialized for %s", executionID)
}
return dialers.Fastdialer.Dial(ctx, network, address)
},
// same as the retryabledns DoH client
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec G402
}
client.doh = doh.NewWithOptions(doh.Options{
HttpClient: &http.Client{
Transport: client.transport,
Timeout: options.GetTimeouts().HttpTimeout,
},
})
if policyClients[executionID] == nil {
policyClients[executionID] = map[string]*Client{}
}
policyClients[executionID][hash] = client
return client, nil
}
// Do sends a DNS message to the resolvers in turn until one answers with
// NOERROR, like retryabledns.Client.Do.
func (c *Client) Do(msg *dns.Msg) (*dns.Msg, error) {
if len(c.resolvers) == 0 {
return c.Client.Do(msg)
}
var resp *dns.Msg
var err error
for i := 0; i < c.retries; i++ {
index := atomic.AddUint32(&c.index, 1)
r := c.resolvers[index%uint32(len(c.resolvers))]
if r.url != "" {
resp, err = c.doh.QueryWithDOHMsg(r.method, doh.Resolver{URL: r.url}, msg)
} else {
resp, err = c.Client.Do(msg)
}
if err == nil && resp != nil && resp.Rcode == dns.RcodeSuccess {
return resp, nil
}
}
return resp, retryabledns.ErrRetriesExceeded
}
// Close drops the clients with DoH resolvers of an execution.
func Close(executionID string) {
policyMu.Lock()
clients := policyClients[executionID]
delete(policyClients, executionID)
policyMu.Unlock()
for _, client := range clients {
client.transport.CloseIdleConnections()
}
}
@@ -1,97 +0,0 @@
package dnsclientpool
import (
"net"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"github.com/miekg/dns"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
// newDohServer answers every DoH query with a TXT record and counts the hits.
func newDohServer(t *testing.T, hits *atomic.Int32) *httptest.Server {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
m := new(dns.Msg)
m.SetQuestion("example.com.", dns.TypeTXT)
m.Response = true
rr, err := dns.NewRR(`example.com. 60 IN TXT "doh-answer"`)
require.NoError(t, err)
m.Answer = append(m.Answer, rr)
packed, err := m.Pack()
require.NoError(t, err)
w.Header().Set("Content-Type", "application/dns-message")
_, _ = w.Write(packed)
}))
t.Cleanup(server.Close)
return server
}
func dohOptions(t *testing.T, excludeTargets []string) *types.Options {
t.Helper()
options := types.DefaultOptions()
options.ExecutionId = t.Name()
options.ExcludeTargets = excludeTargets
require.NoError(t, protocolstate.Init(options))
t.Cleanup(func() { protocolstate.Close(options.ExecutionId) })
require.NoError(t, Init(options))
return options
}
func txtQuery() *dns.Msg {
msg := new(dns.Msg)
msg.SetQuestion("example.com.", dns.TypeTXT)
return msg
}
func TestDohResolverObeysNetworkPolicy(t *testing.T) {
t.Run("denied address", func(t *testing.T) {
var hits atomic.Int32
internal := newDohServer(t, &hits)
options := dohOptions(t, []string{"127.0.0.1"})
client, err := Get(options, &Configuration{Retries: 1, Resolvers: []string{"doh:" + internal.URL + "/dns-query:get"}})
require.NoError(t, err)
_, _ = client.Do(txtQuery())
require.Zero(t, hits.Load(), "DoH query reached a denied address")
})
t.Run("redirect to denied address", func(t *testing.T) {
var hits atomic.Int32
internal := newDohServer(t, &hits)
redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, internal.URL+"/internal", http.StatusFound)
}))
t.Cleanup(redirector.Close)
// localhost passes the resolver check through ::1, while 127.0.0.1 is denied
_, port, err := net.SplitHostPort(strings.TrimPrefix(redirector.URL, "http://"))
require.NoError(t, err)
options := dohOptions(t, []string{"127.0.0.1"})
require.True(t, protocolstate.IsHostAllowed(options.ExecutionId, "localhost"))
client, err := Get(options, &Configuration{Retries: 1, Resolvers: []string{"doh:http://localhost:" + port + "/dns-query:get"}})
require.NoError(t, err)
_, _ = client.Do(txtQuery())
require.Zero(t, hits.Load(), "DoH redirect reached a denied address")
})
t.Run("allowed address", func(t *testing.T) {
var hits atomic.Int32
server := newDohServer(t, &hits)
options := dohOptions(t, nil)
client, err := Get(options, &Configuration{Retries: 2, Resolvers: []string{"doh:" + server.URL + "/dns-query:get"}})
require.NoError(t, err)
resp, err := client.Do(txtQuery())
require.NoError(t, err)
require.Len(t, resp.Answer, 1)
require.Equal(t, int32(1), hits.Load())
})
}
-15
View File
@@ -31,11 +31,6 @@ type Browser struct {
// use getHTTPClient to get the http client
httpClient *http.Client
httpClientOnce *sync.Once
// use getScreenshotDir to get the screenshot directory
screenshotDir string
screenshotDirErr error
screenshotDirOnce sync.Once
}
// New creates a new nuclei headless browser module
@@ -241,16 +236,6 @@ func (b *Browser) getHTTPClient() (*http.Client, error) {
return b.httpClient, err
}
// getScreenshotDir returns the directory that holds the screenshots of
// templates when local file access is off. It is created once per browser
// and kept after Close, so the screenshots stay available.
func (b *Browser) getScreenshotDir() (string, error) {
b.screenshotDirOnce.Do(func() {
b.screenshotDir, b.screenshotDirErr = os.MkdirTemp("", "nuclei-screenshots-*")
})
return b.screenshotDir, b.screenshotDirErr
}
// Close closes the browser engine
//
// When connected over CDP, it does NOT close the browsers.
-1
View File
@@ -43,7 +43,6 @@ type Page struct {
variables map[string]interface{}
lastActionNavigate *Action
cancel context.CancelFunc
screenshots int
}
// HistoryData contains the page request/response pairs
+26 -61
View File
@@ -526,13 +526,6 @@ func (p *Page) RightClickElement(act *Action, out ActionData) error {
return nil
}
// maxScreenshotsPerPage and maxScreenshotSize bound the disk space that the
// screenshots of one page can use when local file access is off.
const (
maxScreenshotsPerPage = 10
maxScreenshotSize = 10 * 1024 * 1024
)
// Screenshot executes screenshot action on a page
func (p *Page) Screenshot(act *Action, out ActionData) error {
to, err := p.getActionArg(act, "to")
@@ -540,52 +533,13 @@ func (p *Page) Screenshot(act *Action, out ActionData) error {
return err
}
generatedName := to == ""
if generatedName {
if to == "" {
to = ksuid.New().String()
if act.Name != "" {
out[act.Name] = to
}
}
lfaAllowed := protocolstate.IsLfaAllowed(p.options.Options)
if !lfaAllowed {
if p.screenshots >= maxScreenshotsPerPage {
return errkit.Newf("screenshot limit of %d per page reached, use -allow-local-file-access flag to remove it", maxScreenshotsPerPage)
}
p.screenshots++
}
var filePath string
if lfaAllowed {
filePath, err = fileutil.CleanPath(to)
if err != nil {
return errkit.Newf("could not clean output screenshot path %s", to)
}
if !strings.HasSuffix(filePath, ".png") {
filePath += ".png"
}
} else {
if err := p.isScreenshotPathAllowed(to); err != nil {
return err
}
// Without local file access the template only names a file inside a
// directory that nuclei creates, so it cannot choose where files land.
baseDir, err := p.instance.browser.getScreenshotDir()
if err != nil {
return errkit.Wrap(err, "could not create screenshot directory")
}
filePath = filepath.Join(baseDir, filepath.Clean(to))
// Add .png before the containment check, so that a name such as "."
// cannot turn into a sibling of the directory.
if !strings.HasSuffix(filePath, ".png") {
filePath += ".png"
}
if !isScreenshotPathWithinDirectory(filePath, baseDir) {
return ErrLFAccessDenied
}
}
var data []byte
fullpage, err := p.getActionArg(act, "fullpage")
@@ -601,8 +555,23 @@ func (p *Page) Screenshot(act *Action, out ActionData) error {
if err != nil {
return errors.Wrap(err, "could not take screenshot")
}
if !lfaAllowed && len(data) > maxScreenshotSize {
return errkit.Newf("screenshot of %d bytes is over the limit of %d bytes, use -allow-local-file-access flag to remove it", len(data), maxScreenshotSize)
to, err = fileutil.CleanPath(to)
if err != nil {
return errkit.Newf("could not clean output screenshot path %s", to)
}
// Build the final write path (with .png) BEFORE running any containment
// gate. Otherwise inputs like "." or a bare directory name pass the gate
// against `to` and then the .png suffix moves the actual write outside
// cwd (e.g. <cwd>.png is a sibling of cwd and not contained by it).
filePath := to
if !strings.HasSuffix(filePath, ".png") {
filePath += ".png"
}
if err := p.isScreenshotPathAllowed(filePath); err != nil {
return err
}
mkdir, err := p.getActionArg(act, "mkdir")
@@ -627,29 +596,25 @@ func (p *Page) Screenshot(act *Action, out ActionData) error {
if err != nil {
return errors.Wrap(err, "could not write screenshot")
}
// The generated name alone does not locate a file inside the screenshot directory.
if generatedName && act.Name != "" && !lfaAllowed {
out[act.Name] = filePath
}
gologger.Info().Msgf("Screenshot successfully saved at %v\n", filePath)
return nil
}
// isScreenshotPathAllowed checks the path a template gives for a screenshot.
// Without local file access it must be relative and must not contain "..".
func (p *Page) isScreenshotPathAllowed(to string) error {
if protocolstate.IsLfaAllowed(p.options.Options) {
return nil
}
if filepath.IsAbs(to) || filepath.VolumeName(to) != "" || strings.HasPrefix(to, "/") || strings.HasPrefix(to, `\`) {
cwd, err := os.Getwd()
if err != nil {
return errkit.Wrap(err, "could not get current working directory")
}
if !isScreenshotPathWithinDirectory(to, cwd) {
// writing outside of cwd requires -lfa flag
return ErrLFAccessDenied
}
for _, part := range strings.FieldsFunc(to, func(r rune) bool { return r == '/' || r == '\\' }) {
if part == ".." {
return ErrLFAccessDenied
}
}
return nil
}
@@ -449,7 +449,7 @@ func TestActionScreenshotDeniesSiblingPrefixPathWithoutLFA(t *testing.T) {
err = page.isScreenshotPathAllowed(filePath)
require.ErrorIs(t, err, ErrLFAccessDenied)
err = page.isScreenshotPathAllowed("test.png")
err = page.isScreenshotPathAllowed(filepath.Join(cwd, "test.png"))
require.NoError(t, err)
}
@@ -541,8 +541,8 @@ func TestActionScreenshotDeniesPostExtensionEscape(t *testing.T) {
"a sibling .png in cwd's parent must be rejected")
// Sanity: a path inside cwd remains allowed.
require.NoError(t, page.isScreenshotPathAllowed("ok.png"),
"a relative path must still be allowed")
require.NoError(t, page.isScreenshotPathAllowed(filepath.Join(cwd, "ok.png")),
"a path inside cwd must still be allowed")
}
func TestActionScreenshotDeniesSymlinkedParentOutsideCWDWithoutLFA(t *testing.T) {
@@ -1051,10 +1051,6 @@ func TestMain(m *testing.M) {
code := m.Run()
browser.Close()
// screenshots of tests without local file access land here
if browser.screenshotDir != "" {
_ = os.RemoveAll(browser.screenshotDir)
}
os.Exit(code)
}
@@ -1,155 +0,0 @@
package engine
import (
"context"
"io/fs"
"net/http"
"net/http/cookiejar"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
)
// chdirTemp moves the test into a fresh working directory and returns it.
func chdirTemp(t *testing.T) string {
t.Helper()
cwd := t.TempDir()
originalWd, err := os.Getwd()
require.NoError(t, err)
require.NoError(t, os.Chdir(cwd))
t.Cleanup(func() { require.NoError(t, os.Chdir(originalWd)) })
return cwd
}
// runWithoutLFA opens a test page with local file access off and runs the
// actions on it.
func runWithoutLFA(t *testing.T, actions []*Action) error {
t.Helper()
_, err := runWithoutLFAData(t, actions)
return err
}
// runWithoutLFAData is runWithoutLFA that also returns the action data.
func runWithoutLFAData(t *testing.T, actions []*Action) (ActionData, error) {
t.Helper()
// the proxy of the shared browser dials with the empty execution id
require.NoError(t, protocolstate.Init(&types.Options{}))
opts := &types.Options{ExecutionId: t.Name(), AllowLocalFileAccess: false}
require.NoError(t, protocolstate.Init(opts))
t.Cleanup(func() { protocolstate.Close(opts.ExecutionId) })
instance, err := sharedBrowser.NewInstance()
require.NoError(t, err)
t.Cleanup(func() { _ = instance.Close() })
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`<html><body>Nuclei Test Page</body></html>`))
}))
t.Cleanup(ts.Close)
input := contextargs.NewWithInput(context.Background(), ts.URL)
input.CookieJar, err = cookiejar.New(nil)
require.NoError(t, err)
actions = append([]*Action{
{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
}, actions...)
data, page, err := instance.Run(input, actions, nil, &Options{Timeout: 20 * time.Second, Options: opts})
if page != nil {
page.Close()
}
return data, err
}
func screenshotAction(args map[string]string) *Action {
return &Action{ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: args}
}
func requireNoFiles(t *testing.T, dir string) {
t.Helper()
var files []string
require.NoError(t, filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
if err == nil && !d.IsDir() {
files = append(files, path)
}
return err
}))
require.Empty(t, files, "template wrote into the working directory")
}
func TestScreenshotWithoutLFAWritesOnlyToScreenshotDir(t *testing.T) {
cwd := chdirTemp(t)
name := "shot-" + strconv.FormatInt(time.Now().UnixNano(), 10)
err := runWithoutLFA(t, []*Action{
screenshotAction(map[string]string{"to": name + "/deep/dir/marker", "mkdir": "true"}),
})
require.NoError(t, err)
requireNoFiles(t, cwd)
matches, err := filepath.Glob(filepath.Join(os.TempDir(), "nuclei-screenshots-*", name, "deep", "dir", "marker.png"))
require.NoError(t, err)
require.Len(t, matches, 1, "screenshot is not in the nuclei screenshot directory")
}
func TestScreenshotWithoutLFANamedResultIsFullPath(t *testing.T) {
chdirTemp(t)
action := screenshotAction(map[string]string{})
action.Name = "shot"
data, err := runWithoutLFAData(t, []*Action{action})
require.NoError(t, err)
path, ok := data["shot"].(string)
require.True(t, ok)
require.True(t, filepath.IsAbs(path), "named result %q does not locate the screenshot", path)
require.FileExists(t, path)
}
func TestScreenshotWithoutLFARejectsPaths(t *testing.T) {
cases := map[string]func(cwd string) string{
"absolute inside working directory": func(cwd string) string { return filepath.Join(cwd, "abs") },
"parent": func(string) string { return "../outside" },
"parent after folder": func(string) string { return "a/../../outside" },
}
for name, to := range cases {
t.Run(name, func(t *testing.T) {
cwd := filepath.Join(chdirTemp(t), "work")
require.NoError(t, os.Mkdir(cwd, 0700))
require.NoError(t, os.Chdir(cwd))
err := runWithoutLFA(t, []*Action{screenshotAction(map[string]string{"to": to(cwd), "mkdir": "true"})})
require.ErrorIs(t, err, ErrLFAccessDenied)
requireNoFiles(t, filepath.Dir(cwd))
})
}
}
func TestScreenshotWithoutLFAIsLimitedPerPage(t *testing.T) {
chdirTemp(t)
// one more than the limit of 10 screenshots per page
actions := make([]*Action, 0, 11)
for range 11 {
actions = append(actions, screenshotAction(nil))
}
require.ErrorContains(t, runWithoutLFA(t, actions), "screenshot limit", "more screenshots than the limit were written for one page")
}
func TestScreenshotWithoutLFAIsLimitedInSize(t *testing.T) {
chdirTemp(t)
noise := `() => { document.body.style.margin='0'; const c=document.createElement('canvas'); c.width=2000; c.height=2000; document.body.appendChild(c); const x=c.getContext('2d'); const d=x.createImageData(2000,2000); for(let i=0;i<d.data.length;i++){d.data[i]=(Math.random()*256)|0; if(i%4==3)d.data[i]=255;} x.putImageData(d,0,0); return 'ok'; }`
err := runWithoutLFA(t, []*Action{
{ActionType: ActionTypeHolder{ActionType: ActionScript}, Data: map[string]string{"code": noise}},
screenshotAction(map[string]string{"fullpage": "true"}),
})
require.ErrorContains(t, err, "over the limit", "a screenshot over the size limit was written")
}
-103
View File
@@ -1,103 +0,0 @@
package http
import (
"context"
"sync"
"sync/atomic"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
)
// bodySizeEstimate is the largest request body a template has built so far.
type bodySizeEstimate struct {
known atomic.Bool
max atomic.Int64
learning sync.Mutex
}
type bodyReservation struct {
budget *httpbody.Budget
size int64
release func()
unlock func()
}
func noop() {}
// reserveBody reserves the template's largest body size before the body is
// built. Until a size is known, builds run one at a time.
func (request *Request) reserveBody(ctx context.Context) (*bodyReservation, error) {
r := &bodyReservation{budget: request.options.RequestBodyBudget, release: noop, unlock: noop}
estimate := request.bodySize
if r.budget == nil || estimate == nil {
return r, nil
}
if !estimate.known.Load() {
estimate.learning.Lock()
if !estimate.known.Load() {
r.unlock = estimate.learning.Unlock
return r, nil
}
estimate.learning.Unlock()
}
r.size = estimate.max.Load()
release, err := r.budget.Acquire(ctx, r.size)
if err != nil {
return nil, err
}
r.release = release
return r, nil
}
// makeReserved builds the next request inside a body reservation, waiting
// for budget until waitCtx ends. On success the caller owns the reservation
// and calls done once the send ends.
func (request *Request) makeReserved(waitCtx context.Context, generator *requestGenerator, input *contextargs.Context, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, *bodyReservation, error) {
reservation, err := request.reserveBody(waitCtx)
if err != nil {
return nil, nil, err
}
generated, err := generator.Make(input.Context(), input, data, payloads, dynamicValues)
if err == nil {
err = reservation.built(waitCtx, request, generated.bodyLen())
}
if err != nil {
reservation.done()
return nil, nil, err
}
generated.bodyReserved = true
return generated, reservation, nil
}
// built grows the reservation to the body's real size. It releases before
// acquiring again: waiting while holding part of the budget can deadlock.
func (r *bodyReservation) built(ctx context.Context, request *Request, size int64) error {
if estimate := request.bodySize; estimate != nil {
for {
seen := estimate.max.Load()
if size <= seen || estimate.max.CompareAndSwap(seen, size) {
break
}
}
estimate.known.Store(true)
}
r.unlock()
r.unlock = noop
if size <= r.size {
return nil
}
r.release()
r.release = noop
release, err := r.budget.Acquire(ctx, size)
if err != nil {
return err
}
r.release, r.size = release, size
return nil
}
func (r *bodyReservation) done() {
r.unlock()
r.release()
}
@@ -1,125 +0,0 @@
//go:build !race
package http
import (
"bufio"
"context"
"fmt"
"io"
"net"
"net/http"
"runtime"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
)
// Threads and pipeline build bodies ahead of the send. Built bodies must stay
// within the budget while the server holds the first request.
func TestRequestBodyBudgetBoundsBodiesBuiltAhead(t *testing.T) {
const bodySize = 8 << 20
const payloads = 12
for _, mode := range []string{"threads", "pipeline"} {
t.Run(mode, func(t *testing.T) {
if mode == "pipeline" {
t.Skip("needs projectdiscovery/rawhttp#551: the pipeline client hangs after 10s idle")
}
arrived := make(chan struct{}, payloads)
unblock := make(chan struct{})
var received atomic.Int32
// net/http drops pipelined connections with large bodies, so serve
// Content-Length framed requests in order by hand.
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer func() { _ = listener.Close() }()
go func() {
for {
conn, err := listener.Accept()
if err != nil {
return
}
go func() {
defer func() { _ = conn.Close() }()
reader := bufio.NewReader(conn)
for {
req, err := http.ReadRequest(reader)
if err != nil {
return
}
_, _ = io.Copy(io.Discard, req.Body)
received.Add(1)
arrived <- struct{}{}
<-unblock
if _, err := io.WriteString(conn, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"); err != nil {
return
}
}
}()
}
}()
target := "http://" + listener.Addr().String()
options := testutils.DefaultOptions.Copy()
testutils.Init(options)
values := make([]string, payloads)
for i := range values {
values[i] = fmt.Sprint(i)
}
request := &Request{
ID: "body-budget-" + mode,
Payloads: map[string]interface{}{"v": values},
}
body := `{{repeat("A", 8388608)}}`
if mode == "threads" {
request.Threads = 8
request.Path = []string{"{{BaseURL}}/?v={{v}}"}
request.Method = HTTPMethodTypeHolder{MethodType: HTTPPost}
request.Body = body
} else {
request.Pipeline = true
request.PipelineConcurrentConnections = 1
request.PipelineRequestsPerConnection = payloads
request.Raw = []string{"POST /?v={{v}} HTTP/1.1\nHost: {{Hostname}}\nContent-Length: 8388608\n\n" + body}
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOpts.RequestBodyBudget = httpbody.NewBudget(bodySize)
require.NoError(t, request.Compile(executerOpts))
var before runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&before)
done := make(chan error, 1)
go func() {
done <- request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), target), make(output.InternalEvent), make(output.InternalEvent), func(*output.InternalWrappedEvent) {})
}()
<-arrived
time.Sleep(300 * time.Millisecond)
var held runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&held)
close(unblock)
require.NoError(t, <-done)
grown := int64(held.HeapAlloc) - int64(before.HeapAlloc)
t.Logf("heap held: %d MB", grown>>20)
require.Less(t, grown, int64(4*bodySize), "bodies built ahead of the send must stay within the budget")
require.EqualValues(t, payloads, received.Load())
})
}
}
-200
View File
@@ -1,200 +0,0 @@
package http
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
)
// The third request queues ~2.4s, so it only beats the 3s timeout if the
// timeout starts after admission.
func TestRequestBodyBudgetQueuesHeavyRequests(t *testing.T) {
const bodySize = 1 << 20
const requests = 3
var inFlight, maxInFlight atomic.Int32
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := inFlight.Add(1)
defer inFlight.Add(-1)
for {
seen := maxInFlight.Load()
if n <= seen || maxInFlight.CompareAndSwap(seen, n) {
break
}
}
body, _ := io.ReadAll(r.Body)
time.Sleep(1200 * time.Millisecond)
if len(body) == bodySize {
_, _ = io.WriteString(w, "full body")
}
}))
defer ts.Close()
options := testutils.DefaultOptions.Copy()
options.SetTimeouts(&types.Timeouts{DialTimeout: time.Second, HttpTimeout: 3 * time.Second})
testutils.Init(options)
request := &Request{
ID: "body-budget",
Path: []string{"{{BaseURL}}"},
Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
Body: `{{repeat("A", 1048576)}}`,
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Part: "body",
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"full body"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOpts.RequestBodyBudget = httpbody.NewBudget(bodySize)
require.NoError(t, request.Compile(executerOpts))
var matched atomic.Int32
var wg sync.WaitGroup
for range requests {
wg.Go(func() {
err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), ts.URL), make(output.InternalEvent), make(output.InternalEvent), func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matched.Add(1)
}
})
require.NoError(t, err)
})
}
wg.Wait()
require.EqualValues(t, 1, maxInFlight.Load(), "budget must hold heavy requests to one in flight")
require.EqualValues(t, requests, matched.Load(), "queued requests must not time out while waiting")
}
func TestReserveBodyQueuesBuildsOnLearnedSize(t *testing.T) {
const bodySize = 4 << 20
request := &Request{
options: &protocols.ExecutorOptions{RequestBodyBudget: httpbody.NewBudget(6 << 20)},
bodySize: &bodySizeEstimate{},
}
first, err := request.reserveBody(t.Context())
require.NoError(t, err)
blocked := make(chan struct{})
go func() {
second, err := request.reserveBody(t.Context())
require.NoError(t, err)
second.done()
close(blocked)
}()
select {
case <-blocked:
t.Fatal("second build started before the first recorded its size")
case <-time.After(20 * time.Millisecond):
}
require.NoError(t, first.built(t.Context(), request, bodySize))
first.done()
<-blocked
held, err := request.reserveBody(t.Context())
require.NoError(t, err)
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
defer cancel()
_, err = request.reserveBody(ctx)
require.ErrorIs(t, err, context.DeadlineExceeded)
held.done()
next, err := request.reserveBody(t.Context())
require.NoError(t, err)
next.done()
}
func TestReserveBodyGrowsToLargerBody(t *testing.T) {
request := &Request{
options: &protocols.ExecutorOptions{RequestBodyBudget: httpbody.NewBudget(6 << 20)},
bodySize: &bodySizeEstimate{},
}
learn, err := request.reserveBody(t.Context())
require.NoError(t, err)
require.NoError(t, learn.built(t.Context(), request, 1<<20))
learn.done()
grown, err := request.reserveBody(t.Context())
require.NoError(t, err)
require.NoError(t, grown.built(t.Context(), request, 5<<20))
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
defer cancel()
_, err = request.reserveBody(ctx)
require.ErrorIs(t, err, context.DeadlineExceeded)
grown.done()
}
// Stop-at-first-match ends the workers with tasks still queued. Their
// reservations must go back to the shared budget.
func TestRequestBodyBudgetStopAtFirstMatchReleasesQueued(t *testing.T) {
const budget = 16 << 20
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
time.Sleep(100 * time.Millisecond)
_, _ = io.WriteString(w, "match")
}))
defer ts.Close()
options := testutils.DefaultOptions.Copy()
testutils.Init(options)
values := make([]string, 64)
for i := range values {
values[i] = fmt.Sprint(i)
}
request := &Request{
ID: "body-budget-stop-first",
Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
Path: []string{"{{BaseURL}}/?v={{v}}"},
Body: `{{repeat("A", 1048576)}}`,
Threads: 4,
Payloads: map[string]interface{}{"v": values},
StopAtFirstMatch: true,
Operators: operators.Operators{Matchers: []*matchers.Matcher{{
Part: "body",
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"match"},
}}},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOpts.RequestBodyBudget = httpbody.NewBudget(budget)
require.NoError(t, request.Compile(executerOpts))
err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), ts.URL), make(output.InternalEvent), make(output.InternalEvent), func(*output.InternalWrappedEvent) {})
require.NoError(t, err)
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
defer cancel()
release, err := executerOpts.RequestBodyBudget.Acquire(ctx, budget)
require.NoError(t, err, "every reservation must be released")
release()
}
+86 -230
View File
@@ -1,6 +1,7 @@
package http
import (
"bufio"
"context"
"fmt"
"net/http"
@@ -82,9 +83,6 @@ type generatedRequest struct {
dynamicValues map[string]interface{}
interactshURLs []string
customCancelFunction context.CancelFunc
// sendTimeout starts after the body budget admits the request.
sendTimeout time.Duration
bodyReserved bool
// requestURLPattern tracks unmodified request url pattern without values ( it is used for constant vuln_hash)
// ex: {{BaseURL}}/api/exp?param={{randstr}}
requestURLPattern string
@@ -274,19 +272,98 @@ func (r *requestGenerator) validateSelfContainedHost(host string) error {
// selfContained templates do not need/use target data and all values i.e {{Hostname}} , {{BaseURL}} etc are already available
// in template . makeSelfContainedRequest parses and creates variables map and then creates corresponding http request or raw request
func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, input *contextargs.Context, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
if r.request.isRaw() {
return r.makeSelfContainedRawRequest(ctx, input, data, payloads, dynamicValues)
isRawRequest := r.request.isRaw()
optionValues := generators.BuildPayloadFromOptions(r.request.options.Options)
values := generators.MergeMaps(optionValues)
if r.request.options.HasTemplateCtx(input.MetaInput) {
values = generators.MergeMaps(values, r.request.options.GetTemplateCtx(input.MetaInput).GetAll())
}
values = generators.MergeMaps(values, dynamicValues, payloads)
// adds all variables from `variables` section in template
scope := r.request.options.NewVariablesScope(optionValues)
r.request.options.AddTemplateCtxToVariablesScope(input.MetaInput, scope)
scope.AddData(dynamicValues, payloads, r.options.Constants)
evaluation := r.request.options.Variables.EvaluateWithInteractshScope(scope, r.request.options.Interactsh)
variablesMap, interactURLs := evaluation.Values, evaluation.InteractURLs
if len(interactURLs) > 0 {
r.interactshURLs = append(r.interactshURLs, interactURLs...)
}
values, _, err := r.selfContainedValues(input, payloads, dynamicValues, nil)
if err != nil {
return nil, err
values = generators.MergeMaps(variablesMap, values)
signerVars := GetDefaultSignerVars(r.request.Signature.Value)
// this will ensure that default signer variables are overwritten by other variables
values = generators.MergeMaps(signerVars, values, r.options.Constants)
for payloadName, payloadValue := range payloads {
renderedPayload, err := r.renderText(types.ToString(payloadValue), values)
if err != nil {
return nil, errkit.Wrap(err, "could not evaluate payload helper expressions")
}
payloads[payloadName] = renderedPayload
}
data, err = r.renderText(data, values)
values = generators.MergeMaps(values, payloads, r.options.Constants)
// priority of variables is as follows (from low to high) for self contained templates
// default signer vars < variables < cli vars < dynamic values < payload < constants
// evaluate request
data, err := r.renderText(data, values)
if err != nil {
return nil, errkit.Wrap(err, "could not evaluate helper expressions")
}
// If the request is a raw request, get the URL from the request
// header and use it to make the request.
if isRawRequest {
// Get the hostname from the URL section to build the request.
reader := bufio.NewReader(strings.NewReader(data))
read_line:
s, err := reader.ReadString('\n')
if err != nil {
return nil, fmt.Errorf("could not read request: %w", err)
}
// ignore all annotations
if stringsutil.HasPrefixAny(s, "@") {
goto read_line
}
parts := strings.Split(s, " ")
if len(parts) < 3 {
return nil, fmt.Errorf("malformed request supplied")
}
if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil && !r.request.SkipVariablesCheck {
return nil, errkit.Newf("unresolved variables `%v` found in request", parts[1])
}
parsed, err := urlutil.ParseURL(parts[1], true)
if err != nil {
return nil, fmt.Errorf("could not parse request URL: %w", err)
}
// self-contained requests choose their own destination URL, so validate
// that host against the network policy.
if err := r.validateSelfContainedHost(parsed.Hostname()); err != nil {
return nil, err
}
values = generators.MergeMaps(
generators.MergeMaps(dynamicValues, protocolutils.GenerateVariables(parsed, false, nil)),
values,
)
// Evaluate (replace) variable with final values
data, err = r.renderText(data, values)
if err != nil {
return nil, errkit.Wrap(err, "could not evaluate helper expressions")
}
return r.generateRawRequest(ctx, data, parsed, values, payloads)
}
if err := expressions.ContainsUnresolvedVariables(data); err != nil && !r.request.SkipVariablesCheck {
// early exit: if there are any unresolved variables in `path` after evaluation
// then return early since this will definitely fail
@@ -304,227 +381,6 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, input *
return r.generateHttpRequest(ctx, urlx, values, payloads)
}
// makeSelfContainedRawRequest makes a self-contained raw request. Its request
// line names the URL, and that URL provides variables such as {{Hostname}} to
// the rest of the request. Each part of the template text is rendered once,
// so inserted values such as response data are sent unchanged:
//
// 1. Render the request line and check its host against the network policy.
// 2. Evaluate again the template variables and payloads whose template text
// names URL variables. The others keep their values, so a random value
// is the same in the request line and in the rest of the request.
// 3. Render the annotations and the rest of the request.
func (r *requestGenerator) makeSelfContainedRawRequest(ctx context.Context, input *contextargs.Context, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
annotations, requestLine, rest, err := splitRequestLine(data)
if err != nil {
return nil, err
}
// Payloads are rendered in place, so keep their template text for step 2.
payloadText := generators.CopyMap(payloads)
values, authored, err := r.selfContainedValues(input, payloads, dynamicValues, nil)
if err != nil {
return nil, err
}
requestLine, err = r.renderText(requestLine, values)
if err != nil {
return nil, errkit.Wrap(err, "could not evaluate helper expressions")
}
// Parse the line that raw.ParseRawRequest will read, even if a value
// added a line break.
firstLine, _, _ := strings.Cut(requestLine, "\n")
parsed, err := r.requestLineURL(firstLine)
if err != nil {
return nil, err
}
// URL variables only fill in names that have no other value.
fixed := make(map[string]interface{})
for name, value := range protocolutils.GenerateVariables(parsed, false, nil) {
if _, ok := values[name]; !ok {
fixed[name] = value
}
}
stale := r.staleValues(authored, payloadText, fixed)
if len(stale) == 0 {
values = generators.MergeMaps(fixed, values)
} else {
// Keep every other value, and evaluate the stale template variables
// and payloads again from their template text.
for name, value := range values {
if _, ok := stale[name]; !ok {
fixed[name] = value
}
}
for name := range stale {
if text, ok := payloadText[name]; ok {
payloads[name] = text
}
}
values, _, err = r.selfContainedValues(input, payloads, dynamicValues, fixed)
if err != nil {
return nil, err
}
}
annotations, err = r.renderText(annotations, values)
if err != nil {
return nil, errkit.Wrap(err, "could not evaluate helper expressions")
}
rest, err = r.renderText(rest, values)
if err != nil {
return nil, errkit.Wrap(err, "could not evaluate helper expressions")
}
data = annotations + requestLine + rest
// A value with a line break must not change which line is the request
// line, or the request would go to a host that was not checked.
if _, line, _, err := splitRequestLine(data); err != nil || line != firstLine {
return nil, errkit.New("malformed request supplied: a value changed the request line")
}
return r.generateRawRequest(ctx, data, parsed, values, payloads)
}
// staleValues returns the names of the authored values whose template text
// names one of urlValues, directly or through another stale value. It reads
// template text only, so data inserted into a value cannot mark it stale.
// Variables are checked in evaluation order, then payloads, which are
// rendered after the variables.
func (r *requestGenerator) staleValues(authored, payloadText, urlValues map[string]interface{}) map[string]struct{} {
stale := make(map[string]struct{})
names := generators.CopyMap(urlValues)
mark := func(name string, text interface{}) {
if _, ok := authored[name]; !ok {
return
}
if expressions.ReferencesVariables(types.ToString(text), names) {
stale[name] = struct{}{}
names[name] = nil
}
}
r.request.options.Variables.ForEach(mark)
for name, text := range payloadText {
mark(name, text)
}
return stale
}
// requestLineURL returns the URL named by a rendered raw request line after
// checking its host against the network policy.
func (r *requestGenerator) requestLineURL(line string) (*urlutil.URL, error) {
parts := strings.Split(line, " ")
if len(parts) < 3 {
return nil, fmt.Errorf("malformed request supplied")
}
if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil && !r.request.SkipVariablesCheck {
return nil, errkit.Newf("unresolved variables `%v` found in request", parts[1])
}
parsed, err := urlutil.ParseURL(parts[1], true)
if err != nil {
return nil, fmt.Errorf("could not parse request URL: %w", err)
}
// self-contained requests choose their own destination URL, so validate
// that host against the network policy.
if err := r.validateSelfContainedHost(parsed.Hostname()); err != nil {
return nil, err
}
return parsed, nil
}
// selfContainedValues evaluates the template variables and renders the
// payloads of a self-contained request, and returns the values to render the
// request with. Payloads are rendered in place. Values in fixed are data:
// they fill in names that have no other value, and a template variable or
// payload with the same name keeps that value instead of being evaluated.
// It also returns the values it produced from template text, that is the
// evaluated template variables and the rendered payloads.
func (r *requestGenerator) selfContainedValues(input *contextargs.Context, payloads, dynamicValues, fixed map[string]interface{}) (map[string]interface{}, map[string]interface{}, error) {
optionValues := generators.BuildPayloadFromOptions(r.request.options.Options)
values := generators.MergeMaps(fixed, optionValues)
if r.request.options.HasTemplateCtx(input.MetaInput) {
values = generators.MergeMaps(values, r.request.options.GetTemplateCtx(input.MetaInput).GetAll())
}
values = generators.MergeMaps(values, dynamicValues, payloads)
// adds all variables from `variables` section in template
scope := r.request.options.NewVariablesScope(optionValues)
r.request.options.AddTemplateCtxToVariablesScope(input.MetaInput, scope)
scope.AddData(dynamicValues, payloads, r.options.Constants, fixed)
evaluation := r.request.options.Variables.EvaluateWithInteractshScope(scope, r.request.options.Interactsh)
variablesMap, interactURLs := evaluation.Values, evaluation.InteractURLs
if len(interactURLs) > 0 {
r.interactshURLs = append(r.interactshURLs, interactURLs...)
}
values = generators.MergeMaps(variablesMap, values)
signerVars := GetDefaultSignerVars(r.request.Signature.Value)
// this will ensure that default signer variables are overwritten by other variables
values = generators.MergeMaps(signerVars, values, r.options.Constants)
// The evaluation owns TemplateValues, so add the rendered payloads to it.
authored := evaluation.TemplateValues
for payloadName, payloadValue := range payloads {
if _, ok := fixed[payloadName]; ok {
continue
}
renderedPayload, err := r.renderText(types.ToString(payloadValue), values)
if err != nil {
return nil, nil, errkit.Wrap(err, "could not evaluate payload helper expressions")
}
payloads[payloadName] = renderedPayload
authored[payloadName] = renderedPayload
}
// priority of variables is as follows (from low to high) for self contained templates
// default signer vars < variables < cli vars < dynamic values < payload < constants
return generators.MergeMaps(values, payloads, r.options.Constants), authored, nil
}
// splitRequestLine splits raw request template text at its request line, the
// first line that is not an annotation. It returns the annotation lines, the
// request line without its line break, and the remaining text starting with
// that line break. The three parts concatenate back to raw.
func splitRequestLine(raw string) (annotations, requestLine, rest string, err error) {
start := 0
for {
line, _, found := strings.Cut(raw[start:], "\n")
if !found {
return "", "", "", errkit.New("could not read request: no request line found")
}
if !strings.HasPrefix(line, "@") {
return raw[:start], line, raw[start+len(line):], nil
}
start += len(line) + 1
}
}
// generateHttpRequest generates http request from request data from template and variables
// finalVars = contains all variables including generator and protocol specific variables
// generatorValues = contains variables used in fuzzing or other generator specific values
-209
View File
@@ -14,14 +14,10 @@ import (
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables"
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -455,211 +451,6 @@ func TestMakeSelfContainedRequestDoesNotAllocateRuntimeInteractshMarkerFromPaylo
require.Equal(t, "{{interactsh-url}}", string(body))
}
func TestSelfContainedRawRequestSendsResponseDataLiterally(t *testing.T) {
const envName = "NUCLEI_TEST_SELF_CONTAINED_ENV"
t.Setenv(envName, "ENV-VALUE")
// A fresh Options pointer, because option payloads are cached per pointer.
options := testutils.DefaultOptions.Copy()
options.EnvironmentVariables = true
testutils.Init(options)
tests := []struct {
name string
body string
}{
{name: "template variable", body: "{{secret}}"},
{name: "helper expression", body: "{{to_upper(seed)}}"},
{name: "environment variable", body: "{{" + envName + "}}"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
probes := make(chan http.Header, 1)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/source":
_, _ = io.WriteString(w, tt.body)
case "/probe":
header := r.Header.Clone()
header.Set("Host", r.Host)
probes <- header
}
}))
t.Cleanup(server.Close)
request := &Request{
ID: "testing-self-contained-raw-response-data",
SelfContained: true,
// Send the probe even though it holds a literal {{...}}.
SkipVariablesCheck: true,
Raw: []string{
"GET " + server.URL + "/source HTTP/1.1\nHost: {{Hostname}}\n\n",
"GET " + server.URL + "/probe HTTP/1.1\nHost: {{Hostname}}\n" +
"Probe: [{{tpl}}]\nAuthored: [{{base64(seed)}}-{{" + envName + "}}]\n\n",
},
Operators: operators.Operators{
Extractors: []*extractors.Extractor{{
Name: "tpl",
Part: "body",
Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{".+"},
Internal: true,
}},
},
}
templateVars := variables.Variable{
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(2),
}
templateVars.Set("seed", "hello")
templateVars.Set("secret", "TOP-SECRET")
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOpts.Variables = templateVars
require.NoError(t, request.Compile(executerOpts))
err := request.ExecuteWithResults(
contextargs.NewWithInput(context.Background(), ""),
make(output.InternalEvent),
make(output.InternalEvent),
func(*output.InternalWrappedEvent) {},
)
require.NoError(t, err)
var probe http.Header
select {
case probe = <-probes:
default:
t.Fatal("probe request was not sent")
}
require.Equal(t, "["+tt.body+"]", probe.Get("Probe"), "response data must be sent unchanged")
// Template-authored text still renders, including variables derived
// from the request URL.
require.Equal(t, "[aGVsbG8=-ENV-VALUE]", probe.Get("Authored"))
require.Equal(t, strings.TrimPrefix(server.URL, "http://"), probe.Get("Host"))
})
}
}
// makeSelfContainedRawRequest compiles request as a self-contained raw request
// with the given template variables and makes its first request.
func makeSelfContainedRawRequest(t *testing.T, request *Request, vars [][2]string, dynamicValues map[string]interface{}) (*generatedRequest, error) {
t.Helper()
options := testutils.DefaultOptions
testutils.Init(options)
request.ID = "testing-self-contained-raw"
request.SelfContained = true
templateVars := variables.Variable{
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(len(vars)),
}
for _, v := range vars {
templateVars.Set(v[0], v[1])
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOpts.Variables = templateVars
require.NoError(t, request.Compile(executerOpts))
generator := request.newGenerator(false)
inputData, payloads, ok := generator.nextValue()
require.True(t, ok)
return generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), ""), inputData, payloads, dynamicValues)
}
func TestMakeSelfContainedRawRequestResolvesURLVariables(t *testing.T) {
request := &Request{
Raw: []string{"GET http://198.51.100.7/?t={{token}}&d={{data_token}} HTTP/1.1\nHost: {{Hostname}}\n" +
"X-Var: {{host_var}}\nX-Upper: {{upper_host}}\nX-Payload: {{user}}\n" +
"X-Token: {{token}}\nX-Data-Token: {{data_token}}\nX-Data: {{data}}\n\n"},
Payloads: map[string]interface{}{"user": []string{"admin@{{Hostname}}"}},
// Send values that hold a literal {{...}} from data.
SkipVariablesCheck: true,
}
got, err := makeSelfContainedRawRequest(t, request, [][2]string{
{"token", "{{rand_base(8)}}"},
{"data_token", "{{rand_base(8)}}-{{tpl}}"},
{"data", "{{tpl}}"},
{"host_var", "{{Hostname}}"},
{"upper_host", "{{to_upper(host_var)}}"},
}, map[string]interface{}{"tpl": "{{Hostname}}"})
require.NoError(t, err)
require.Equal(t, "198.51.100.7", got.request.Header.Get("X-Var"))
require.Equal(t, "198.51.100.7", got.request.Header.Get("X-Upper"))
require.Equal(t, "[email protected]", got.request.Header.Get("X-Payload"))
// Data that names a URL variable is sent unchanged.
require.Equal(t, "{{Hostname}}", got.request.Header.Get("X-Data"))
// Values whose template text names no URL variable are not evaluated
// again, even when data inserted into them does.
require.Equal(t, got.request.Params.Get("t"), got.request.Header.Get("X-Token"))
require.Equal(t, got.request.Params.Get("d"), got.request.Header.Get("X-Data-Token"))
require.True(t, strings.HasSuffix(got.request.Header.Get("X-Data-Token"), "-{{Hostname}}"))
}
func TestMakeSelfContainedRawRequestRejectsMovedRequestLine(t *testing.T) {
request := &Request{
Raw: []string{"@timeout: {{wait}}\nGET http://198.51.100.7/ HTTP/1.1\nHost: {{Hostname}}\n\n"},
}
_, err := makeSelfContainedRawRequest(t, request, nil, map[string]interface{}{
"wait": "5s\nGET http://192.0.2.1/secret HTTP/1.1",
})
require.ErrorContains(t, err, "changed the request line")
}
func TestSplitRequestLine(t *testing.T) {
t.Parallel()
tests := []struct {
name string
raw string
annotations, requestLine, rest string
wantErr bool
}{
{
name: "no annotations",
raw: "GET http://a/ HTTP/1.1\r\nHost: a\r\n\r\n",
requestLine: "GET http://a/ HTTP/1.1\r",
rest: "\nHost: a\r\n\r\n",
},
{
name: "annotations",
raw: "@tls-sni: {{Hostname}}\n@timeout: 5s\nGET http://a/ HTTP/1.1\nHost: a\n",
annotations: "@tls-sni: {{Hostname}}\n@timeout: 5s\n",
requestLine: "GET http://a/ HTTP/1.1",
rest: "\nHost: a\n",
},
{name: "no line break", raw: "GET http://a/ HTTP/1.1", wantErr: true},
{name: "annotations only", raw: "@timeout: 5s\n", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
annotations, requestLine, rest, err := splitRequestLine(tt.raw)
if tt.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
require.Equal(t, tt.annotations, annotations)
require.Equal(t, tt.requestLine, requestLine)
require.Equal(t, tt.rest, rest)
})
}
}
func TestExecuteRequestDoesNotReevaluateGeneratedPayloadMetadata(t *testing.T) {
options := testutils.DefaultOptions
+1 -3
View File
@@ -147,7 +147,6 @@ type Request struct {
rawhttpClient *rawhttp.Client
replayClient *http.Client // set when -replay-proxy is given
dialer *fastdialer.Dialer
bodySize *bodySizeEstimate
// description: |
// SelfContained specifies if the request is self-contained.
@@ -403,7 +402,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
// If we have request level timeout, ignore http client timeouts
for _, req := range request.Raw {
if hasTimeoutAnnotation(req) {
if reTimeoutAnnotation.MatchString(req) {
connectionConfiguration.NoTimeout = true
}
}
@@ -417,7 +416,6 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
return errors.Wrap(err, "could not get dialer")
}
request.dialer = dialer
request.bodySize = &bodySizeEstimate{}
request.options = options
for _, option := range request.options.Options.CustomHeaders {
+1 -2
View File
@@ -15,7 +15,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
convUtil "github.com/projectdiscovery/utils/conversion"
)
// Match matches a generic data response again a given matcher
@@ -226,7 +225,7 @@ func (request *Request) savedRequest(dump string) string {
if request.options == nil || request.options.Options == nil {
return dump
}
return string(capDumpBody(convUtil.Bytes(dump), request.options.Options.RequestSaveSize))
return string(capDumpBody([]byte(dump), request.options.Options.RequestSaveSize))
}
// eventResponse returns the full response of event. Events matched after the
+33 -13
View File
@@ -14,6 +14,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/authprovider/authx"
"github.com/projectdiscovery/rawhttp/client"
"github.com/projectdiscovery/utils/errkit"
stringsutil "github.com/projectdiscovery/utils/strings"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -192,20 +193,23 @@ func readRawRequest(request string, unsafe bool) (*Request, error) {
Headers: make(map[string]string),
}
_, request = SplitAnnotations(request)
// store body if it is unsafe request
if unsafe {
rawRequest.UnsafeRawBytes = []byte(request)
rawRequest.UnsafeRawBytes = stripLeadingAnnotations(request)
}
// parse raw request
source := strings.NewReader(request)
reader := bufio.NewReader(source)
read_line:
s, err := reader.ReadString('\n')
if err != nil {
return nil, fmt.Errorf("could not read request: %w", err)
}
// ignore all annotations
if stringsutil.HasPrefixAny(s, "@") {
goto read_line
}
parts := strings.Fields(s)
if len(parts) > 0 {
@@ -288,18 +292,34 @@ func readRawRequest(request string, unsafe bool) (*Request, error) {
}
// SplitAnnotations splits a raw request into its annotations and the request
// that follows them. Annotations are the lines starting with "@" before the
// request line. A line starting with "@" after the request line is part of the
// request, so text rendered into the headers or the body never becomes an
// annotation.
func SplitAnnotations(request string) (annotations, rest string) {
rest = request
for strings.HasPrefix(rest, "@") {
_, rest, _ = strings.Cut(rest, "\n")
func stripLeadingAnnotations(request string) []byte {
reader := bufio.NewReader(strings.NewReader(request))
buffer := bufferPool.Get().(*bytes.Buffer)
buffer.Reset()
defer func() {
buffer.Reset()
bufferPool.Put(buffer)
}()
requestLineFound := false
for {
line, err := reader.ReadString('\n')
if len(line) > 0 {
if requestLineFound || !stringsutil.HasPrefixAny(line, "@") {
requestLineFound = true
_, _ = buffer.WriteString(line)
}
}
if err != nil {
if err == io.EOF {
break
}
return []byte(request)
}
}
return request[:len(request)-len(rest)], rest
return append([]byte(nil), buffer.Bytes()...)
}
// TryFillCustomHeaders after the Host header
-45
View File
@@ -310,51 +310,6 @@ Connection: close`, parseURL(t, "http://scanme.sh"), true, false)
require.NotContains(t, string(request.UnsafeRawBytes), "@Host:", "annotation lines must not be present in unsafe raw bytes")
}
func TestSplitAnnotations(t *testing.T) {
tests := []struct {
name string
request string
annotations string
}{
{name: "empty"},
{name: "no annotations", request: "GET / HTTP/1.1\r\nHost: a\r\n\r\n"},
{
name: "leading annotations",
request: "@Host: b\r\n@timeout: 5s\r\nGET / HTTP/1.1\r\nHost: a\r\n\r\n",
annotations: "@Host: b\r\n@timeout: 5s\r\n",
},
{
name: "annotation lines after the request line",
request: "@once\nPOST / HTTP/1.1\n@Host: c\n\nhello\n@Host: d\n",
annotations: "@once\n",
},
{
name: "annotations without a request line",
request: "@Host: b\n@once",
annotations: "@Host: b\n@once",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
annotations, rest := SplitAnnotations(tt.request)
require.Equal(t, tt.annotations, annotations)
require.Equal(t, tt.request, annotations+rest)
})
}
}
func TestParseUnsafeRequestKeepsAnnotationLinesInBody(t *testing.T) {
const (
head = "POST /foo HTTP/1.1\nHost: {{Hostname}}\n\n"
body = "hello\n@Host: decoy.example\n"
)
request, err := Parse("@Host: honey.scanme.sh\n"+head+body, parseURL(t, "http://scanme.sh"), true, false)
require.NoError(t, err)
require.Equal(t, head+body, string(request.UnsafeRawBytes))
require.Equal(t, body, request.Data)
}
func TestTryFillCustomHeaders(t *testing.T) {
testValue := "GET /manager/html HTTP/1.1\r\nHost: Test\r\n"
expected := "GET /test/manager/html HTTP/1.1\r\nHost: Test\r\ntest: test\r\n"
+83 -121
View File
@@ -331,7 +331,6 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
// bounded worker-pool to avoid spawning one goroutine per payload
type task struct {
req *generatedRequest
body *bodyReservation
updatedInput *contextargs.Context
hasInteractMarkers bool
}
@@ -339,87 +338,72 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
var workersWg sync.WaitGroup
currentWorkers := maxWorkers
tasks := make(chan task, maxWorkers)
// Workers exit on cancel without draining, and the budget is shared, so
// release what they left queued.
stopWorkers := func() {
close(tasks)
workersWg.Wait()
for t := range tasks {
t.body.done()
}
}
spawnWorker := func(ctx context.Context) {
workersWg.Add(1)
go func() {
defer workersWg.Done()
for t := range tasks {
if func() (exit bool) {
defer t.body.done()
select {
case <-ctx.Done():
return true
default:
}
if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
return false
}
if err := spmHandler.Acquire(); err != nil {
return false
}
if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
spmHandler.Release()
return false
}
// Full URL, so https and an explicit port stay on the same key the observation uses.
// The handler context ends the wait when stop-at-first-match fires.
waitCtx, cancelWait := stopWithHandler(ctx, spmHandler)
err := request.rateLimitTake(waitCtx, hostBackoffKey(t.req, t.updatedInput))
cancelWait()
if err != nil {
select {
case <-spmHandler.Done():
spmHandler.Release()
return false
case <-ctx.Done():
spmHandler.Release()
return false
case spmHandler.ResultChan <- err:
spmHandler.Release()
return false
}
}
if ctx.Err() != nil || spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
spmHandler.Release()
return false
}
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
needsRequestEvent := hasInteractMatchers && request.NeedsRequestCondition()
err = request.executeRequest(t.updatedInput, t.req, make(map[string]interface{}), hasInteractMatchers, func(event *output.InternalWrappedEvent) {
if (t.hasInteractMarkers || needsRequestEvent) && request.options.Interactsh != nil {
requestData := &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
Event: event,
Operators: request.CompiledOperators,
MatchFunc: request.Match,
ExtractFunc: request.Extract,
}
allOASTUrls := httputils.GetInteractshURLSFromEvent(event.InternalEvent)
allOASTUrls = append(allOASTUrls, t.req.interactshURLs...)
request.options.RegisterInteractshRequest(sliceutil.Dedupe(allOASTUrls), requestData)
}
wrappedCallback(event)
}, 0)
select {
case <-ctx.Done():
return
default:
}
if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
continue
}
if err := spmHandler.Acquire(); err != nil {
continue
}
if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
spmHandler.Release()
continue
}
// Full URL, so https and an explicit port stay on the same key the observation uses.
// The handler context ends the wait when stop-at-first-match fires.
waitCtx, cancelWait := stopWithHandler(ctx, spmHandler)
err := request.rateLimitTake(waitCtx, hostBackoffKey(t.req, t.updatedInput))
cancelWait()
if err != nil {
select {
case <-spmHandler.Done():
spmHandler.Release()
return false
case spmHandler.ResultChan <- err:
request.recordHostResultAndCancelIfUnresponsive(t.updatedInput, err, spmHandler.Cancel)
continue
case <-ctx.Done():
spmHandler.Release()
continue
case spmHandler.ResultChan <- err:
spmHandler.Release()
continue
}
return false
}() {
return
}
if ctx.Err() != nil || spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
spmHandler.Release()
continue
}
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
needsRequestEvent := hasInteractMatchers && request.NeedsRequestCondition()
err = request.executeRequest(t.updatedInput, t.req, make(map[string]interface{}), hasInteractMatchers, func(event *output.InternalWrappedEvent) {
if (t.hasInteractMarkers || needsRequestEvent) && request.options.Interactsh != nil {
requestData := &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
Event: event,
Operators: request.CompiledOperators,
MatchFunc: request.Match,
ExtractFunc: request.Extract,
}
allOASTUrls := httputils.GetInteractshURLSFromEvent(event.InternalEvent)
allOASTUrls = append(allOASTUrls, t.req.interactshURLs...)
request.options.RegisterInteractshRequest(sliceutil.Dedupe(allOASTUrls), requestData)
}
wrappedCallback(event)
}, 0)
select {
case <-spmHandler.Done():
spmHandler.Release()
continue
case spmHandler.ResultChan <- err:
request.recordHostResultAndCancelIfUnresponsive(t.updatedInput, err, spmHandler.Cancel)
spmHandler.Release()
}
}
}()
@@ -438,7 +422,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
select {
case <-input.Context().Done():
stopWorkers()
close(tasks)
workersWg.Wait()
return input.Context().Err()
default:
}
@@ -446,7 +431,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
// resize check point - nop if there are no changes
if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency {
if err := spmHandler.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil {
stopWorkers()
close(tasks)
workersWg.Wait()
return err
}
// if payload concurrency increased, add more workers
@@ -463,22 +449,15 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
break
}
// Queued tasks hold budget and are only released once the workers
// stop, so the wait has to end with them.
generatedHttpRequest, reservation, err := request.makeReserved(workerCtx, generator, input, inputData, payloads, dynamicValues)
ctx := input.Context()
generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
if err != nil {
if err == types.ErrNoMoreRequests {
break
}
if workerCtx.Err() != nil {
stopWorkers()
if spmHandler.FoundFirstMatch() {
return nil
}
return context.Cause(workerCtx)
}
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
stopWorkers()
close(tasks)
workersWg.Wait()
return err
}
hasInteractMarkers := interactsh.HasMarkers(inputData) || len(generatedHttpRequest.interactshURLs) > 0
@@ -488,32 +467,33 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
if request.isUnresponsiveAddress(updatedInput) {
// skip on unresponsive host no need to continue
reservation.done()
spmHandler.Cancel()
stopWorkers()
close(tasks)
workersWg.Wait()
return nil
}
select {
case <-spmHandler.Done():
reservation.done()
stopWorkers()
close(tasks)
workersWg.Wait()
spmHandler.Wait()
if spmHandler.FoundFirstMatch() {
return nil
}
return multierr.Combine(spmHandler.CombinedResults()...)
case <-workerCtx.Done():
reservation.done()
stopWorkers()
close(tasks)
workersWg.Wait()
if spmHandler.FoundFirstMatch() {
return nil
}
return context.Cause(workerCtx)
case tasks <- task{req: generatedHttpRequest, body: reservation, updatedInput: updatedInput, hasInteractMarkers: hasInteractMarkers}:
case tasks <- task{req: generatedHttpRequest, updatedInput: updatedInput, hasInteractMarkers: hasInteractMarkers}:
}
request.options.Progress.IncrementRequests()
}
stopWorkers()
close(tasks)
workersWg.Wait()
spmHandler.Wait()
if spmHandler.FoundFirstMatch() {
// ignore any context cancellation and in-transit execution errors
@@ -602,7 +582,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
}
ctx := input.Context()
generatedHttpRequest, reservation, err := request.makeReserved(ctx, generator, input, inputData, payloads, dynamicValues)
generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
if err != nil {
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
return err
@@ -613,18 +593,15 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
if request.isUnresponsiveAddress(updatedInput) {
// skip on unresponsive host no need to continue
reservation.done()
spmHandler.Cancel()
return nil
}
generatedHttpRequest.pipelinedClient = pipeClient
if err := spmHandler.Acquire(); err != nil {
reservation.done()
break
}
go func(httpRequest *generatedRequest, body *bodyReservation, requestInput *contextargs.Context) {
go func(httpRequest *generatedRequest, requestInput *contextargs.Context) {
defer spmHandler.Release()
defer body.done()
if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(requestInput) {
// skip if first match is found
return
@@ -649,7 +626,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
request.recordHostResultAndCancelIfUnresponsive(requestInput, err, spmHandler.Cancel)
return
}
}(generatedHttpRequest, reservation, updatedInput)
}(generatedHttpRequest, updatedInput)
request.options.Progress.IncrementRequests()
}
spmHandler.Wait()
@@ -699,17 +676,16 @@ executionLoop:
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
ctx := input.Context()
generatedHttpRequest, reservation, err := request.makeReserved(ctx, generator, input, data, payloads, dynamicValue)
ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, request.options.Options.GetTimeouts().HttpTimeout, ErrHttpEngineRequestDeadline)
defer cancel()
generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
if err != nil {
if err == types.ErrNoMoreRequests {
return false, true, nil, nil
}
return false, true, nil, err
}
defer reservation.done()
if generatedHttpRequest.customCancelFunction == nil {
generatedHttpRequest.sendTimeout = request.options.Options.GetTimeouts().HttpTimeout
}
// ideally if http template used a custom port or hostname
// we would want to update it in input but currently templateCtx logic
// is closely tied to contextargs.Context so we are temporarily creating
@@ -1084,20 +1060,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
return fmt.Errorf("hostErrorsCache : host %s is unresponsive", input.MetaInput.Input)
}
// Race requests must all be in flight together to reach the open gate.
if !generatedRequest.original.Race && !generatedRequest.bodyReserved {
releaseBody, err := request.options.RequestBodyBudget.Acquire(input.Context(), generatedRequest.bodyLen())
if err != nil {
return err
}
defer releaseBody()
}
if generatedRequest.sendTimeout > 0 && generatedRequest.request != nil {
ctx, cancel := context.WithTimeoutCause(generatedRequest.request.Context(), generatedRequest.sendTimeout, ErrHttpEngineRequestDeadline)
defer cancel()
generatedRequest.request = generatedRequest.request.WithContext(ctx)
}
// wrap one more callback for validation and fixing event
callback := func(event *output.InternalWrappedEvent) {
// validateNFixEvent performs necessary validation on generated event
@@ -1713,7 +1675,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
}
responseContentType := respChain.Response().Header.Get("Content-Type")
isResponseTruncated := respChain.Body().Len() >= maxBodylimit
isResponseTruncated := (request.MaxSize > 0 && respChain.Body().Len() >= request.MaxSize) ||
(request.options.Options.ResponseReadSize > 0 && respChain.Body().Len() >= request.options.Options.ResponseReadSize)
dumpResponse(event, request, fullResponseStr, formedURL, responseContentType, isResponseTruncated, input.MetaInput.Input)
callback(event)
@@ -2026,8 +1989,7 @@ func (request *Request) needsFullResponse(generatedRequest *generatedRequest) bo
}
if request.options != nil && request.options.Options != nil {
cliOptions := request.options.Options
// -ms writes the response of failed requests too, and a failure never reaches the lazy fill
if cliOptions.Debug || cliOptions.DebugResponse || cliOptions.StoreResponse || cliOptions.HTTPStats || cliOptions.MatcherStatus {
if cliOptions.Debug || cliOptions.DebugResponse || cliOptions.StoreResponse || cliOptions.HTTPStats {
return true
}
}
+5 -15
View File
@@ -11,7 +11,6 @@ import (
"github.com/projectdiscovery/fastdialer/fastdialer"
commoninteractsh "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/raw"
"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/utils/errkit"
@@ -55,9 +54,8 @@ func parseFlowAnnotations(rawRequest string) (flowMark, bool) {
var fm flowMark
// parse request for known override annotations
var hasFlowOverride bool
annotations, _ := raw.SplitAnnotations(rawRequest)
// @once
if reOnceAnnotation.MatchString(annotations) {
if reOnceAnnotation.MatchString(rawRequest) {
fm = Once
hasFlowOverride = true
}
@@ -65,12 +63,6 @@ func parseFlowAnnotations(rawRequest string) (flowMark, bool) {
return fm, hasFlowOverride
}
// hasTimeoutAnnotation reports whether rawRequest has a @timeout annotation.
func hasTimeoutAnnotation(rawRequest string) bool {
annotations, _ := raw.SplitAnnotations(rawRequest)
return reTimeoutAnnotation.MatchString(annotations)
}
type annotationOverrides struct {
request *retryablehttp.Request
cancelFunc context.CancelFunc
@@ -80,12 +72,10 @@ type annotationOverrides struct {
// parseAnnotations and override requests settings
func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Request) (overrides annotationOverrides, modified bool) {
// rawRequest is rendered, so its headers and body can hold values taken
// from responses. Only the annotation lines may override the request.
annotations, _ := raw.SplitAnnotations(rawRequest)
// parse request for known override annotations
// @Host:target
if hosts := reHostAnnotation.FindStringSubmatch(annotations); len(hosts) > 0 {
if hosts := reHostAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 {
value := strings.TrimSpace(hosts[1])
// handle scheme
switch {
@@ -111,7 +101,7 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req
}
// @tls-sni:target
if hosts := reSniAnnotation.FindStringSubmatch(annotations); len(hosts) > 0 {
if hosts := reSniAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 {
value := strings.TrimSpace(hosts[1])
value = stringsutil.TrimPrefixAny(value, "http://", "https://")
@@ -150,7 +140,7 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req
modified = true
var ctx context.Context
if duration := reTimeoutAnnotation.FindStringSubmatch(annotations); len(duration) > 0 {
if duration := reTimeoutAnnotation.FindStringSubmatch(rawRequest); len(duration) > 0 {
value := strings.TrimSpace(duration[1])
if parsedTimeout, err := time.ParseDuration(value); err == nil {
// Cap at maximum allowed timeout to prevent DoS via extremely large timeout values
@@ -2,25 +2,13 @@ package http
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
commoninteractsh "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/retryablehttp-go"
@@ -203,123 +191,3 @@ func TestRequestParseAnnotationsTimeout(t *testing.T) {
require.False(t, deadlined, "could not get set request deadline")
})
}
func TestRequestParseAnnotationsIgnoresLinesAfterRequestLine(t *testing.T) {
req := &Request{connConfiguration: &httpclientpool.Configuration{}}
rawRequest := "POST /probe HTTP/1.1\r\n" +
"Host: example.com\r\n" +
"\r\n" +
"hello\n@Host: http://127.0.0.1:8797\n@tls-sni: decoy.example\n"
httpReq, err := retryablehttp.NewRequest(http.MethodPost, "https://example.com/probe", nil)
require.NoError(t, err)
overrides, modified := req.parseAnnotations(rawRequest, httpReq)
require.False(t, modified)
require.Equal(t, "https://example.com/probe", overrides.request.String())
require.Nil(t, overrides.request.TLS)
}
// A response value reused in a raw request must stay request content even when
// one of its lines looks like an annotation. Annotations are only the lines
// before the request line.
func TestRawRequestReusingResponseWithAnnotationLine(t *testing.T) {
tests := []struct {
name string
unsafe bool
selfContained bool
}{
{name: "raw"},
{name: "unsafe", unsafe: true},
{name: "self-contained", selfContained: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var decoyRequests atomic.Int32
decoy := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
decoyRequests.Add(1)
}))
defer decoy.Close()
responseBody := fmt.Sprintf("hello\n@Host: %s\n", decoy.URL)
var mu sync.Mutex
var probes []string
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/source" {
_, _ = io.WriteString(w, responseBody)
return
}
body, _ := io.ReadAll(r.Body)
mu.Lock()
probes = append(probes, r.Method+" "+r.URL.Path+" "+string(body))
mu.Unlock()
}))
defer target.Close()
baseURL, host, input := "", "{{Hostname}}", target.URL
if tt.selfContained {
// A self-contained request names its own target.
baseURL, host, input = target.URL, strings.TrimPrefix(target.URL, "http://"), ""
}
probeHeaders := "Host: " + host + "\nContent-Type: text/plain\n"
if tt.unsafe {
// An unsafe request is sent as written, so it states its own length.
probeHeaders += "Content-Length: {{len(tpl)}}\n"
}
request := &Request{
ID: "testing-http-response-annotation-line",
Unsafe: tt.unsafe,
SelfContained: tt.selfContained,
Raw: []string{
"GET " + baseURL + "/source HTTP/1.1\nHost: " + host + "\n\n",
"POST " + baseURL + "/probe HTTP/1.1\n" + probeHeaders + "\n{{tpl}}",
},
Operators: operators.Operators{
Extractors: []*extractors.Extractor{{
Name: "tpl",
Part: "body",
Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{`(?s).+`},
Internal: true,
}},
},
}
options := testutils.DefaultOptions
testutils.Init(options)
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}, Name: "test"},
})
require.NoError(t, request.Compile(executerOpts))
err := request.ExecuteWithResults(
contextargs.NewWithInput(context.Background(), input),
make(output.InternalEvent),
make(output.InternalEvent),
func(*output.InternalWrappedEvent) {},
)
require.NoError(t, err)
require.Zero(t, decoyRequests.Load(), "a response line must not redirect the request")
mu.Lock()
defer mu.Unlock()
require.Equal(t, []string{"POST /probe " + responseBody}, probes)
})
}
}
func TestTemplateAnnotationsAfterRequestLine(t *testing.T) {
const request = "GET / HTTP/1.1\r\nHost: {{Hostname}}\r\n\r\n@timeout: 5s\r\n@once\r\n"
require.False(t, hasTimeoutAnnotation(request))
require.True(t, hasTimeoutAnnotation("@timeout: 5s\r\n"+request))
_, hasFlowOverride := parseFlowAnnotations(request)
require.False(t, hasFlowOverride)
mark, hasFlowOverride := parseFlowAnnotations("@once\r\n" + request)
require.True(t, hasFlowOverride)
require.Equal(t, Once, mark)
}
-79
View File
@@ -1,7 +1,6 @@
package http
import (
"cmp"
"context"
"crypto/tls"
"fmt"
@@ -1866,84 +1865,6 @@ func TestHTTPResponseReadSizeHonoursTemplateMaxSize(t *testing.T) {
}
}
// storedResponseWriter keeps the stored response dumps.
type storedResponseWriter struct {
*testutils.MockOutputWriter
responses []string
}
func (w *storedResponseWriter) WriteStoreDebugData(_, _, _, data string) {
if strings.Contains(data, "Dumped HTTP response") {
w.responses = append(w.responses, data)
}
}
// TestHTTPResponseTruncatedUsesEffectiveReadLimit checks that a stored response
// is marked truncated exactly when its body reached the limit it was read with.
func TestHTTPResponseTruncatedUsesEffectiveReadLimit(t *testing.T) {
previous := MaxBodyRead
t.Cleanup(func() { MaxBodyRead = previous })
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/plain")
_, _ = w.Write([]byte(strings.Repeat("A", 2000)))
}))
defer server.Close()
tests := []struct {
name string
defaultLimit int // replaces MaxBodyRead when set
templateMaxSize int
responseReadSize int
wantTruncated bool
}{
{name: "default limit", defaultLimit: 1000, wantTruncated: true},
{name: "default limit, negative response-size-read", defaultLimit: 1000, responseReadSize: -1, wantTruncated: true},
{name: "body under default limit", wantTruncated: false},
{name: "template max-size", templateMaxSize: 500, wantTruncated: true},
{name: "response-size-read", responseReadSize: 800, wantTruncated: true},
{name: "max-size below response-size-read", templateMaxSize: 500, responseReadSize: 3000, wantTruncated: true},
{name: "limits above body length", templateMaxSize: 4000, responseReadSize: 3000, wantTruncated: false},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
MaxBodyRead = cmp.Or(tc.defaultLimit, previous)
opts := testutils.DefaultOptions.Copy()
opts.StoreResponse = true
opts.ResponseReadSize = tc.responseReadSize
testutils.Init(opts)
t.Cleanup(func() { testutils.Cleanup(opts) })
request := &Request{
ID: "test-truncated",
MaxSize: tc.templateMaxSize,
Raw: []string{"GET / HTTP/1.1\nHost: {{Hostname}}\n\n"},
}
executerOptions := testutils.NewMockExecuterOptions(opts, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
writer := &storedResponseWriter{MockOutputWriter: testutils.NewMockOutputWriter(opts.OmitTemplate)}
executerOptions.Output = writer
require.NoError(t, request.Compile(executerOptions))
err := request.ExecuteWithResults(
contextargs.NewWithInput(context.Background(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(*output.InternalWrappedEvent) {},
)
require.NoError(t, err)
require.Len(t, writer.responses, 1)
require.Equal(t, tc.wantTruncated, strings.Contains(writer.responses[0], "(Truncated)"))
})
}
}
const httpToHTTPSPage = "<html><title>400 The plain HTTP request was sent to HTTPS port</title><center>The plain HTTP request was sent to HTTPS port</center></html>"
// servePlainOrTLS accepts HTTP and TLS on one port. A plaintext request is
-63
View File
@@ -248,26 +248,6 @@ func TestNeedsFullResponse(t *testing.T) {
require.NoError(t, req.Compile(execOpts))
require.True(t, req.needsFullResponse(nil))
})
t.Run("matcher status option requires full response", func(t *testing.T) {
msOpts := testutils.DefaultOptions.Copy()
msOpts.MatcherStatus = true
req := &Request{
ID: "matcher-status-flag",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Part: "body",
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"test"},
}},
},
}
execOpts := testutils.NewMockExecuterOptions(msOpts, &testutils.TemplateInfo{ID: "matcher-status-flag"})
require.NoError(t, req.Compile(execOpts))
require.True(t, req.needsFullResponse(nil))
})
}
func TestExecuteRequestDeferredResponse_NoMatch(t *testing.T) {
@@ -315,49 +295,6 @@ func TestExecuteRequestDeferredResponse_NoMatch(t *testing.T) {
require.Contains(t, types.ToString(capturedEvent.InternalEvent["all_headers"]), "X-Custom-Header")
}
// With -ms a failed match is written out too, so its event must carry the
// response even though no result is created.
func TestExecuteRequestDeferredResponse_NoMatchWithMatcherStatus(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprintf(w, "response body content")
}))
defer ts.Close()
options := testutils.DefaultOptions.Copy()
options.MatcherStatus = true
testutils.Init(options)
req := &Request{
ID: "defer-no-match-ms",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}"},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Part: "body",
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"nonexistent-needle"},
}},
},
}
execOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: "defer-no-match-ms",
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}, Name: "test"},
})
require.NoError(t, req.Compile(execOpts))
var capturedEvent *output.InternalWrappedEvent
ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
err := req.ExecuteWithResults(ctxArgs, make(output.InternalEvent), make(output.InternalEvent), func(event *output.InternalWrappedEvent) {
capturedEvent = event
})
require.NoError(t, err)
require.NotNil(t, capturedEvent)
require.False(t, capturedEvent.HasResults())
require.Contains(t, types.ToString(capturedEvent.InternalEvent["response"]), "HTTP/1.1 200")
require.Contains(t, types.ToString(capturedEvent.InternalEvent["response"]), "response body content")
}
func TestExecuteRequestDeferredResponse_WithMatch(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Server", "nuclei-test")
+1 -22
View File
@@ -7,7 +7,6 @@ import (
"fmt"
"io"
"net/http"
"slices"
"strings"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
@@ -83,9 +82,6 @@ func dumpRequest(req *retryablehttp.Request, maxBody int) (requestDump, error) {
}
capture := &cappedWriter{headerEnd: -1, maxBody: maxBody}
if maxBody <= 0 {
capture.buf = make([]byte, 0, len(body)+dumpHeaderRoom)
}
if err := clone.Write(capture); err != nil {
return requestDump{}, err
}
@@ -93,7 +89,7 @@ func dumpRequest(req *retryablehttp.Request, maxBody int) (requestDump, error) {
// http.Transport asks for gzip unless the caller set an encoding.
if clone.Header.Get("Accept-Encoding") == "" && clone.Header.Get("Range") == "" && clone.Method != http.MethodHead && capture.headerEnd >= 2 {
end := capture.headerEnd - 2
data = slices.Insert(data, end, acceptEncodingGzip...)
data = append(data[:end:end], append([]byte("Accept-Encoding: gzip\r\n"), data[end:]...)...)
}
return requestDump{data: appendTruncationMarker(data, capture.dropped), truncated: capture.dropped}, nil
}
@@ -127,10 +123,6 @@ func capDumpBody(dump []byte, maxBody int) []byte {
var headerTerminator = []byte("\r\n\r\n")
var acceptEncodingGzip = []byte("Accept-Encoding: gzip\r\n")
const dumpHeaderRoom = 8 * 1024
// cappedWriter keeps the request line and headers in full and at most
// maxBody bytes of the body that follows, counting the rest.
type cappedWriter struct {
@@ -185,19 +177,6 @@ func getHTTPProjectCacheScope(requestDump requestDump, body []byte, scheme, host
return scoped.Bytes()
}
func (g *generatedRequest) bodyLen() int64 {
if g.rawRequest != nil {
return int64(len(g.rawRequest.Data))
}
if g.request == nil || g.request.Body == nil {
return 0
}
if body, ok := g.request.Body.(*httpbody.Body); ok {
return int64(body.Len())
}
return g.request.ContentLength
}
// body returns the request body without copying it.
func (g *generatedRequest) body() []byte {
if g.request != nil {
+1 -1
View File
@@ -11,7 +11,7 @@ import (
"sync/atomic"
"time"
"github.com/alecthomas/chroma/v2/quick"
"github.com/alecthomas/chroma/quick"
"github.com/ditashi/jsbeautifier-go/jsbeautifier"
"github.com/pkg/errors"
"github.com/projectdiscovery/goja"
-5
View File
@@ -44,7 +44,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
unitutils "github.com/projectdiscovery/utils/unit"
)
@@ -107,8 +106,6 @@ type ExecutorOptions struct {
Progress progress.Progress
// RateLimiter is a rate-limiter for limiting sent number of requests.
RateLimiter *ratelimit.Limiter
// RequestBodyBudget bounds request-body bytes in flight; nil is no limit.
RequestBodyBudget *httpbody.Budget
// Catalog is a template catalog implementation for nuclei
Catalog catalog.Catalog
// ProjectFile is the project file for nuclei
@@ -398,7 +395,6 @@ func (e *ExecutorOptions) Copy() *ExecutorOptions {
IssuesClient: e.IssuesClient,
Progress: e.Progress,
RateLimiter: e.RateLimiter,
RequestBodyBudget: e.RequestBodyBudget,
Catalog: e.Catalog,
ProjectFile: e.ProjectFile,
Browser: e.Browser,
@@ -710,7 +706,6 @@ func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) {
e.IssuesClient = n.IssuesClient
e.Progress = n.Progress
e.RateLimiter = n.RateLimiter
e.RequestBodyBudget = n.RequestBodyBudget
e.Catalog = n.Catalog
e.ProjectFile = n.ProjectFile
e.Browser = n.Browser
@@ -1,57 +1,39 @@
package rdapclientpool
import (
"context"
"fmt"
"net"
"net/http"
"sync"
"github.com/openrdap/rdap/bootstrap"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/rdap"
)
// clients holds one client per execution, because each one dials with the
// network policy of its execution.
var clients = map[string]*rdap.Client{}
var normalClient *rdap.Client
var m sync.Mutex
// Init initializes the client pool implementation
func Init(options *types.Options) error {
_, err := Get(options, nil)
return err
}
m.Lock()
defer m.Unlock()
// newClient returns an rdap client whose requests, bootstrap lookups and
// redirects all dial through the fastdialer of the execution, so every
// address it connects to is checked against the network policy.
func newClient(options *types.Options) *rdap.Client {
executionID := options.ExecutionId
httpClient := &http.Client{
Transport: &http.Transport{
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
dialers := protocolstate.GetDialersWithId(executionID)
if dialers == nil || dialers.Fastdialer == nil {
return nil, fmt.Errorf("dialers not initialized for %s", executionID)
}
return dialers.Fastdialer.Dial(ctx, network, address)
},
},
Timeout: options.GetTimeouts().HttpTimeout,
}
client := &rdap.Client{
HTTP: httpClient,
Bootstrap: &bootstrap.Client{HTTP: httpClient},
// Don't create clients if already created in the past.
if normalClient != nil {
return nil
}
normalClient = &rdap.Client{}
if options.Verbose || options.Debug || options.DebugRequests || options.DebugResponse {
client.Verbose = func(text string) {
normalClient.Verbose = func(text string) {
gologger.Debug().Msgf("rdap: %s", text)
}
}
return client
return nil
}
func getNormalClient() *rdap.Client {
m.Lock()
defer m.Unlock()
return normalClient
}
// Configuration contains the custom configuration options for a client - placeholder
@@ -62,25 +44,7 @@ func (c *Configuration) Hash() string {
return ""
}
// Get creates or gets the client of the execution
// Get creates or gets a client for the protocol based on custom configuration
func Get(options *types.Options, configuration *Configuration) (*rdap.Client, error) {
m.Lock()
defer m.Unlock()
if client, ok := clients[options.ExecutionId]; ok {
return client, nil
}
client := newClient(options)
clients[options.ExecutionId] = client
return client, nil
}
// Close drops the client of an execution.
func Close(executionID string) {
m.Lock()
client := clients[executionID]
delete(clients, executionID)
m.Unlock()
if client != nil {
client.HTTP.CloseIdleConnections()
}
return getNormalClient(), nil
}
@@ -1,66 +0,0 @@
package rdapclientpool
import (
"net/http"
"net/http/httptest"
"net/url"
"sync/atomic"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/rdap"
"github.com/stretchr/testify/require"
)
func newRdapServer(t *testing.T, hits *atomic.Int32) *httptest.Server {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.Header().Set("Content-Type", "application/rdap+json")
_, _ = w.Write([]byte(`{"objectClassName":"domain","ldhName":"example.com"}`))
}))
t.Cleanup(server.Close)
return server
}
func rdapOptions(t *testing.T, executionID string, excludeTargets []string) *types.Options {
t.Helper()
options := types.DefaultOptions()
options.ExecutionId = executionID
options.ExcludeTargets = excludeTargets
require.NoError(t, protocolstate.Init(options))
t.Cleanup(func() { protocolstate.Close(options.ExecutionId) })
require.NoError(t, Init(options))
return options
}
func domainRequest(t *testing.T, server string) *rdap.Request {
t.Helper()
serverURL, err := url.Parse(server)
require.NoError(t, err)
request := rdap.NewAutoRequest("example.com")
request.Server = serverURL
return request
}
// Each execution gets a client that dials with its own network policy.
func TestClientObeysExecutionNetworkPolicy(t *testing.T) {
var hits atomic.Int32
server := newRdapServer(t, &hits)
open := rdapOptions(t, t.Name()+"-open", nil)
restricted := rdapOptions(t, t.Name()+"-restricted", []string{"127.0.0.1"})
openClient, err := Get(open, nil)
require.NoError(t, err)
_, err = openClient.Do(domainRequest(t, server.URL))
require.NoError(t, err)
require.Equal(t, int32(1), hits.Load())
restrictedClient, err := Get(restricted, nil)
require.NoError(t, err)
_, err = restrictedClient.Do(domainRequest(t, server.URL))
require.Error(t, err)
require.Equal(t, int32(1), hits.Load(), "rdap request reached a denied address")
}
-56
View File
@@ -1,56 +0,0 @@
package whois
import (
"context"
"net"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/stretchr/testify/require"
)
func TestWhoisServerRedirectObeysNetworkPolicy(t *testing.T) {
var hits atomic.Int32
internal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
hits.Add(1)
w.Header().Set("Content-Type", "application/rdap+json")
_, _ = w.Write([]byte(`{"objectClassName":"domain","ldhName":"example.com"}`))
}))
t.Cleanup(internal.Close)
// the redirector listens on ::1 so that it is reached while 127.0.0.1 is denied
var redirects atomic.Int32
redirector := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
redirects.Add(1)
http.Redirect(w, r, internal.URL+r.URL.Path, http.StatusFound)
}))
listener, err := net.Listen("tcp", "[::1]:0")
require.NoError(t, err)
redirector.Listener = listener
redirector.Start()
t.Cleanup(redirector.Close)
_, port, err := net.SplitHostPort(listener.Addr().String())
require.NoError(t, err)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ExcludeTargets = []string{"127.0.0.1"}
testutils.Init(options)
t.Cleanup(func() { protocolinit.Close(options.ExecutionId) })
require.True(t, protocolstate.IsHostAllowed(options.ExecutionId, "::1"))
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "whois-redirect"})
request := &Request{ID: "whois-redirect", Query: "example.com", Server: "http://[::1]:" + port + "/rdap/"}
require.NoError(t, request.Compile(executerOpts))
input := contextargs.NewWithInput(context.Background(), "example.com")
_ = request.ExecuteWithResults(input, nil, nil, func(*output.InternalWrappedEvent) {})
require.NotZero(t, redirects.Load(), "the redirector was never reached")
require.Zero(t, hits.Load(), "whois redirect reached a denied address")
}
+3 -4
View File
@@ -71,10 +71,9 @@ func New(option *Options) (*Exporter, error) {
Transport: &http.Transport{
MaxIdleConns: 10,
MaxIdleConnsPerHost: 10,
// No DialTLSContext: fastdialer DialTLS ignores TLSClientConfig and
// skips verification; net/http does the handshake with TLSClientConfig.
DialContext: dialers.Fastdialer.Dial,
TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification},
DialContext: dialers.Fastdialer.Dial,
DialTLSContext: dialers.Fastdialer.DialTLS,
TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification},
},
}
}
-46
View File
@@ -1,46 +0,0 @@
package es
import (
"net"
"net/http"
"net/http/httptest"
"strconv"
"sync/atomic"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestExporterVerifiesTLS(t *testing.T) {
var got atomic.Int32
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got.Add(1)
w.WriteHeader(http.StatusCreated)
}))
defer srv.Close()
host, port, err := net.SplitHostPort(srv.Listener.Addr().String())
require.NoError(t, err)
p, err := strconv.Atoi(port)
require.NoError(t, err)
opts := types.DefaultOptions()
opts.SetExecutionID("es-tls-verify")
require.NoError(t, protocolstate.Init(opts))
defer protocolstate.Close(opts.ExecutionId)
newExporter := func(skipVerify bool) *Exporter {
e, err := New(&Options{IP: host, Port: p, SSL: true, SSLVerification: skipVerify, Username: "elastic", Password: "secret", IndexName: "nuclei", ExecutionId: opts.ExecutionId})
require.NoError(t, err)
return e
}
err = newExporter(false).Export(&output.ResultEvent{})
require.ErrorContains(t, err, "certificate")
require.Zero(t, got.Load(), "the password went to an untrusted server")
require.NoError(t, newExporter(true).Export(&output.ResultEvent{}))
require.EqualValues(t, 1, got.Load())
}
+3 -4
View File
@@ -63,10 +63,9 @@ func New(option *Options) (*Exporter, error) {
Transport: &http.Transport{
MaxIdleConns: 10,
MaxIdleConnsPerHost: 10,
// No DialTLSContext: fastdialer DialTLS ignores TLSClientConfig and
// skips verification; net/http does the handshake with TLSClientConfig.
DialContext: dialers.Fastdialer.Dial,
TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification},
DialContext: dialers.Fastdialer.Dial,
DialTLSContext: dialers.Fastdialer.DialTLS,
TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification},
},
}
}
@@ -1,46 +0,0 @@
package splunk
import (
"net"
"net/http"
"net/http/httptest"
"strconv"
"sync/atomic"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestExporterVerifiesTLS(t *testing.T) {
var got atomic.Int32
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got.Add(1)
w.WriteHeader(http.StatusCreated)
}))
defer srv.Close()
host, port, err := net.SplitHostPort(srv.Listener.Addr().String())
require.NoError(t, err)
p, err := strconv.Atoi(port)
require.NoError(t, err)
opts := types.DefaultOptions()
opts.SetExecutionID("splunk-tls-verify")
require.NoError(t, protocolstate.Init(opts))
defer protocolstate.Close(opts.ExecutionId)
newExporter := func(skipVerify bool) *Exporter {
e, err := New(&Options{Host: host, Port: p, SSL: true, SSLVerification: skipVerify, Token: "secret", IndexName: "nuclei", ExecutionId: opts.ExecutionId})
require.NoError(t, err)
return e
}
err = newExporter(false).Export(&output.ResultEvent{})
require.ErrorContains(t, err, "certificate")
require.Zero(t, got.Load(), "the token went to an untrusted server")
require.NoError(t, newExporter(true).Export(&output.ResultEvent{}))
require.EqualValues(t, 1, got.Load())
}
+31
View File
@@ -414,3 +414,34 @@ http:
})
}
}
// TestFlowRuntimeIsNotSharedAcrossTemplates checks that a flow which changes the
// globals of its runtime does not change the flow of a later template.
func TestFlowRuntimeIsNotSharedAcrossTemplates(t *testing.T) {
setup()
router := httprouter.New()
router.GET("/first", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
_, _ = fmt.Fprint(w, "first-ok")
})
router.GET("/second", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
_, _ = fmt.Fprint(w, "second-ok")
})
ts := httptest.NewServer(router)
defer ts.Close()
execute := func(path string) bool {
tmpl, err := templates.Parse(path, nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.Nil(t, tmpl.Executer.Compile(), "could not compile template")
input := contextargs.NewWithInput(context.Background(), ts.URL)
gotresults, err := tmpl.Executer.Execute(scan.NewScanContext(context.Background(), input))
require.Nil(t, err, "could not execute template")
return gotresults
}
for i := 0; i < 10; i++ {
execute("testcases/flow-runtime-first.yaml")
require.True(t, execute("testcases/flow-runtime-second.yaml"), "second flow ran in a runtime changed by another template (run %d)", i)
}
}
@@ -0,0 +1,17 @@
id: flow-runtime-first
info:
name: Flow that changes the globals of its runtime
author: test
severity: info
flow: |
// Export
globalThis.leftover = true;
iterate = function () { return []; };
http(1);
http:
- method: GET
path:
- "{{BaseURL}}/first"
@@ -0,0 +1,23 @@
id: flow-runtime-second
info:
name: Flow that uses the builtins of its runtime
author: test
severity: info
flow: |
if (typeof leftover === 'undefined') {
for (let path of iterate(["second"])) {
http(1);
}
}
http:
- method: GET
path:
- "{{BaseURL}}/second"
matchers:
- type: word
words:
- "second-ok"
-3
View File
@@ -370,8 +370,6 @@ type Options struct {
// RequestSaveSize is the maximum request body kept in saved output and
// debug dumps. Matchers see the full request. 0 keeps the whole body.
RequestSaveSize int
// MaxRequestBodyInFlight bounds request-body bytes in flight; 0 disables it.
MaxRequestBodyInFlight int
// Health Check
HealthCheck bool
// Time to wait between each input read operation before closing the stream
@@ -767,7 +765,6 @@ func (options *Options) Copy() *Options {
ResponseReadSize: options.ResponseReadSize,
ResponseSaveSize: options.ResponseSaveSize,
RequestSaveSize: options.RequestSaveSize,
MaxRequestBodyInFlight: options.MaxRequestBodyInFlight,
HealthCheck: options.HealthCheck,
InputReadTimeout: options.InputReadTimeout,
DisableStdin: options.DisableStdin,
-38
View File
@@ -1,38 +0,0 @@
package httpbody
import (
"context"
"golang.org/x/sync/semaphore"
)
// MinBudgetedBody is the smallest body that waits on a Budget, so small
// requests never queue behind a heavy one.
const MinBudgetedBody = 1 << 20
// Budget bounds the request-body bytes held by requests in flight.
type Budget struct {
size int64
sem *semaphore.Weighted
}
// NewBudget returns nil when size <= 0; a nil budget never blocks.
func NewBudget(size int64) *Budget {
if size <= 0 {
return nil
}
return &Budget{size: size, sem: semaphore.NewWeighted(size)}
}
// Acquire blocks until n bytes fit or ctx is done. n larger than the budget
// is clamped, so that request runs alone.
func (b *Budget) Acquire(ctx context.Context, n int64) (release func(), err error) {
if b == nil || n < MinBudgetedBody {
return func() {}, nil
}
n = min(n, b.size)
if err := b.sem.Acquire(ctx, n); err != nil {
return nil, err
}
return func() { b.sem.Release(n) }, nil
}
-68
View File
@@ -1,68 +0,0 @@
package httpbody
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestBudgetQueuesBodiesOverSize(t *testing.T) {
budget := NewBudget(48 << 20)
first, err := budget.Acquire(t.Context(), 20<<20)
require.NoError(t, err)
second, err := budget.Acquire(t.Context(), 20<<20)
require.NoError(t, err)
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
defer cancel()
_, err = budget.Acquire(ctx, 20<<20)
require.ErrorIs(t, err, context.DeadlineExceeded)
first()
third, err := budget.Acquire(t.Context(), 20<<20)
require.NoError(t, err)
third()
second()
}
func TestBudgetRunsBodyLargerThanBudgetAlone(t *testing.T) {
budget := NewBudget(8 << 20)
huge, err := budget.Acquire(t.Context(), 64<<20)
require.NoError(t, err)
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
defer cancel()
_, err = budget.Acquire(ctx, 2<<20)
require.ErrorIs(t, err, context.DeadlineExceeded)
huge()
next, err := budget.Acquire(t.Context(), 2<<20)
require.NoError(t, err)
next()
}
func TestBudgetSkipsSmallBodies(t *testing.T) {
budget := NewBudget(MinBudgetedBody)
full, err := budget.Acquire(t.Context(), MinBudgetedBody)
require.NoError(t, err)
defer full()
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
defer cancel()
release, err := budget.Acquire(ctx, MinBudgetedBody-1)
require.NoError(t, err)
release()
}
func TestNilBudgetNeverBlocks(t *testing.T) {
require.Nil(t, NewBudget(0))
var budget *Budget
release, err := budget.Acquire(t.Context(), 1<<30)
require.NoError(t, err)
release()
}