mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2026-11-08 10:10:18 -10:00
Compare commits
33
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
55bd0bbfb6 | ||
|
|
8f1c88e1f3 | ||
|
|
8cda906559 | ||
|
|
a2e403a1d8 | ||
|
|
bbebeba8b2 | ||
|
|
578d00829a | ||
|
|
3be47b098f | ||
|
|
b29ecc6907 | ||
|
|
29559cbd2a | ||
|
|
94f7148b7e | ||
|
|
1829fca65c | ||
|
|
accfe3551b | ||
|
|
d6ec8407fb | ||
|
|
bb2da9bbea | ||
|
|
6e2b59827c | ||
|
|
875b229313 | ||
|
|
91f8bd1e4f | ||
|
|
fb0eed5122 | ||
|
|
29833da25a | ||
|
|
9b519d2264 | ||
|
|
4fed40f75a | ||
|
|
2f4ac3d9e5 | ||
|
|
27ac1e8726 | ||
|
|
5b499e0650 | ||
|
|
7748411200 | ||
|
|
0642f05241 | ||
|
|
df707d924f | ||
|
|
5f5a40c2d4 | ||
|
|
82d5a8acc2 | ||
|
|
9d3be8b22e | ||
|
|
76dd059821 | ||
|
|
6863ab16e5 | ||
|
|
0305d5468c |
No files matched your search
@@ -22,6 +22,8 @@ extend-exclude = [
|
||||
"go.sum",
|
||||
# Generated history. Commit SHAs are not words.
|
||||
"SECURITY_CONTEXT.md",
|
||||
# Generated technology graph. Entries are third-party product identifiers.
|
||||
"pkg/protocols/common/automaticscan/techgraph/tech-graph.json",
|
||||
]
|
||||
|
||||
[default.extend-identifiers]
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const headerChildEnv = "NUCLEI_HEADER_CHILD_ARGS"
|
||||
|
||||
// -H values can contain "#", on the command line, in a header file and in the
|
||||
// config file, while lines of a header file that start with "#" are skipped as
|
||||
// comments.
|
||||
func TestHeaderValuesKeepHash(t *testing.T) {
|
||||
if args := os.Getenv(headerChildEnv); args != "" {
|
||||
options = &types.Options{Logger: gologger.DefaultLogger}
|
||||
os.Args = append([]string{os.Args[0]}, strings.Split(args, "\n")...)
|
||||
readConfig()
|
||||
fmt.Printf("headers=%s\n", strings.Join(options.CustomHeaders, "|"))
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
root := t.TempDir()
|
||||
headerFile := filepath.Join(root, "headers.txt")
|
||||
require.NoError(t, os.WriteFile(headerFile, []byte("# auth headers\nX-Token: abc#def\nReferer: https://app.example.com/#/dashboard\n"), 0o600))
|
||||
configFile := filepath.Join(root, "config.yaml")
|
||||
require.NoError(t, os.WriteFile(configFile, []byte("header:\n - \"X-Config: cfg#value\"\n"), 0o600))
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{"cli and header file", []string{"-H", "Cookie: sid=1; color=#ff0000", "-H", headerFile}, "Cookie: sid=1; color=#ff0000|X-Token: abc#def|Referer: https://app.example.com/#/dashboard"},
|
||||
{"config file", []string{"-config", configFile}, "X-Config: cfg#value"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tempRoot := filepath.Join(t.TempDir(), "tmp")
|
||||
require.NoError(t, os.Mkdir(tempRoot, 0o700))
|
||||
|
||||
args := append(tt.args, "-disable-update-check")
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestHeaderValuesKeepHash$", "-test.count=1")
|
||||
cmd.Env = isolatedNucleiStorageEnv(root, tempRoot, headerChildEnv+"="+strings.Join(args, "\n"))
|
||||
output, err := cmd.CombinedOutput()
|
||||
require.NoError(t, err, string(output))
|
||||
require.Contains(t, string(output), "headers="+tt.want+"\n")
|
||||
})
|
||||
}
|
||||
}
|
||||
+18
-1
@@ -55,6 +55,17 @@ var (
|
||||
inlineSecretsTempMu sync.Mutex
|
||||
)
|
||||
|
||||
// normalizeHeaderValue trims a -H value and drops it when the whole line is a
|
||||
// "#" comment. The goflags default strips inline comments, which cut values
|
||||
// such as "Cookie: color=#fff" at the "#".
|
||||
func normalizeHeaderValue(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if strings.HasPrefix(s, "#") {
|
||||
return ""
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func main() {
|
||||
options.Logger = gologger.DefaultLogger
|
||||
|
||||
@@ -255,6 +266,11 @@ func readConfig() *goflags.FlagSet {
|
||||
var pdcpAuth string
|
||||
var fuzzFlag bool
|
||||
|
||||
// FileStringSliceOptions as set up by runner.ConfigureOptions, but keeping
|
||||
// "#" inside -H values
|
||||
headerSliceOptions := goflags.FileStringSliceOptions
|
||||
headerSliceOptions.Normalize = normalizeHeaderValue
|
||||
|
||||
flagSet := goflags.NewFlagSet()
|
||||
flagSet.CaseSensitive = true
|
||||
flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing
|
||||
@@ -291,6 +307,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"),
|
||||
flagSet.StringSliceVarP(&options.NewTemplatesWithVersion, "new-templates-version", "ntv", nil, "run new templates added in specific version", goflags.CommaSeparatedStringSliceOptions),
|
||||
flagSet.BoolVarP(&options.AutomaticScan, "automatic-scan", "as", false, "automatic web scan using wappalyzer technology detection to tags mapping"),
|
||||
flagSet.StringVarP(&options.AutomaticScanCoverage, "automatic-scan-coverage", "asc", "balanced", "automatic scan coverage tier: lean, balanced, or thorough"),
|
||||
flagSet.StringSliceVarP(&options.Templates, "templates", "t", nil, "list of template or template directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
|
||||
flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "turl", nil, "template url or list containing template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
|
||||
flagSet.StringVarP(&options.AITemplatePrompt, "prompt", "ai", "", "generate and run template using ai prompt"),
|
||||
@@ -367,7 +384,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "mr", 10, "max number of redirects to follow for http templates"),
|
||||
flagSet.BoolVarP(&options.DisableRedirects, "disable-redirects", "dr", false, "disable redirects for http templates"),
|
||||
flagSet.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking
|
||||
flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", nil, "custom header/cookie to include in all http request in header:value format (cli, file)", goflags.FileStringSliceOptions),
|
||||
flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", nil, "custom header/cookie to include in all http request in header:value format (cli, file)", headerSliceOptions),
|
||||
flagSet.RuntimeMapVarP(&options.Vars, "var", "V", nil, "custom vars in key=value format"),
|
||||
flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"),
|
||||
flagSet.BoolVarP(&options.SystemResolvers, "system-resolvers", "sr", false, "use system DNS resolving as error fallback"),
|
||||
|
||||
@@ -15,7 +15,7 @@ require (
|
||||
github.com/Mzack9999/go-rsync v0.0.0-20250821180103-81ffa574ef4d
|
||||
github.com/RedTeamPentesting/adauth v0.5.4-0.20260511073005-3d18e8a5a687
|
||||
github.com/adrg/xdg v0.5.3
|
||||
github.com/alecthomas/chroma v0.10.0
|
||||
github.com/alecthomas/chroma/v2 v2.27.0
|
||||
github.com/alexsnet/go-vnc v0.1.0
|
||||
github.com/alitto/pond v1.9.2
|
||||
github.com/andybalholm/brotli v1.2.0
|
||||
@@ -74,6 +74,7 @@ require (
|
||||
github.com/miekg/dns v1.1.73
|
||||
github.com/oiweiwei/go-msrpc v1.2.12
|
||||
github.com/olekukonko/tablewriter v1.0.8
|
||||
github.com/openrdap/rdap v0.9.1
|
||||
github.com/ory/dockertest/v3 v3.12.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/praetorian-inc/fingerprintx v1.1.15
|
||||
@@ -175,7 +176,6 @@ require (
|
||||
github.com/STARRY-S/zip v0.2.3 // indirect
|
||||
github.com/VividCortex/ewma v1.2.0 // indirect
|
||||
github.com/akrylysov/pogreb v0.10.2 // indirect
|
||||
github.com/alecthomas/chroma/v2 v2.20.0 // indirect
|
||||
github.com/alecthomas/kingpin/v2 v2.4.0 // indirect
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
|
||||
@@ -237,7 +237,6 @@ require (
|
||||
github.com/dimchansky/utfbom v1.1.1 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/djherbis/times v1.6.0 // indirect
|
||||
github.com/dlclark/regexp2 v1.11.5 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.2.1 // indirect
|
||||
github.com/docker/cli v29.2.0+incompatible // indirect
|
||||
github.com/docker/go-connections v0.7.0 // indirect
|
||||
@@ -264,7 +263,7 @@ require (
|
||||
github.com/go-fed/httpsig v1.1.0 // indirect
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.9.0 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.6 // indirect
|
||||
github.com/go-logr/logr v1.4.4 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
@@ -367,7 +366,6 @@ require (
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/opencontainers/runc v1.3.6 // indirect
|
||||
github.com/openrdap/rdap v0.9.1 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.23 // indirect
|
||||
github.com/pjbgf/sha1cd v0.6.0 // indirect
|
||||
|
||||
@@ -147,15 +147,13 @@ github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi
|
||||
github.com/akrylysov/pogreb v0.10.2/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
|
||||
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
|
||||
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
|
||||
github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
|
||||
github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
|
||||
github.com/alecthomas/chroma/v2 v2.20.0 h1:sfIHpxPyR07/Oylvmcai3X/exDlE8+FA820NTz+9sGw=
|
||||
github.com/alecthomas/chroma/v2 v2.20.0/go.mod h1:e7tViK0xh/Nf4BYHl00ycY6rV7b8iXBksI9E359yNmA=
|
||||
github.com/alecthomas/chroma/v2 v2.27.0 h1:FodwmyOBgJULFYmDqibcp9pvfDLWdtPRh9v/r5BXYZs=
|
||||
github.com/alecthomas/chroma/v2 v2.27.0/go.mod h1:NjJ3ciIgrqBNeIkWZ4e46nseoLDslxU1LmfCoL+wcY8=
|
||||
github.com/alecthomas/kingpin/v2 v2.3.1/go.mod h1:oYL5vtsvEHZGHxU7DMp32Dvx+qL+ptGn6lWaot2vCNE=
|
||||
github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY=
|
||||
github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
|
||||
github.com/alecthomas/repr v0.5.1 h1:E3G4t2QbHTSNpPKBgMTln5KLkZHLOcU7r37J4pXBuIg=
|
||||
github.com/alecthomas/repr v0.5.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
|
||||
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
@@ -358,7 +356,6 @@ github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c h1:+Zo5Ca9
|
||||
github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c/go.mod h1:HJGU9ULdREjOcVGZVPB5s6zYmHi1RxzT71l2wQyLmnE=
|
||||
github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
|
||||
github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
|
||||
github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
|
||||
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
|
||||
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/dlclark/regexp2/v2 v2.2.1 h1:mf4KkFUj0gJuarK8P+LgiS+Lit7m9N1yAwEfPbee7R0=
|
||||
@@ -455,8 +452,8 @@ github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuw
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-jose/go-jose/v4 v4.1.6 h1:sNRkftxFJxwsILiQukqp5z83QBSmntVwmgRY7AqBDO4=
|
||||
github.com/go-jose/go-jose/v4 v4.1.6/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
|
||||
|
||||
+29
-5
@@ -4,6 +4,8 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -87,11 +89,7 @@ func NewUploadWriter(ctx context.Context, logger *gologger.Logger, creds *pdcpau
|
||||
tmp.Update()
|
||||
u.uploadURL = tmp.URL
|
||||
|
||||
// create http client
|
||||
opts := retryablehttp.DefaultOptionsSingle
|
||||
opts.NoAdjustTimeout = true
|
||||
opts.Timeout = time.Duration(3) * time.Minute
|
||||
u.client = retryablehttp.NewClient(opts)
|
||||
u.client = newUploadClient()
|
||||
|
||||
// create context
|
||||
ctx, u.cancel = context.WithCancel(ctx)
|
||||
@@ -101,6 +99,32 @@ func NewUploadWriter(ctx context.Context, logger *gologger.Logger, creds *pdcpau
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// newUploadClient returns the client for cloud uploads. The upload sends the
|
||||
// API key and the scan results, so the client verifies the server certificate.
|
||||
// The retryablehttp default clients skip verification.
|
||||
func newUploadClient() *retryablehttp.Client {
|
||||
transport := retryablehttp.DefaultReusePooledTransport()
|
||||
// fastdialer DialTLS does not verify certificates, so net/http does the
|
||||
// TLS handshake over the fastdialer connection.
|
||||
transport.DialTLSContext = nil
|
||||
transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12}
|
||||
|
||||
opts := retryablehttp.DefaultOptionsSingle
|
||||
opts.NoAdjustTimeout = true
|
||||
opts.Timeout = time.Duration(3) * time.Minute
|
||||
opts.HttpClient = &http.Client{Transport: transport}
|
||||
// a certificate error does not go away on retry
|
||||
defaultRetry := retryablehttp.DefaultRetryPolicy()
|
||||
opts.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
var certErr *tls.CertificateVerificationError
|
||||
if errors.As(err, &certErr) {
|
||||
return false, nil
|
||||
}
|
||||
return defaultRetry(ctx, resp, err)
|
||||
}
|
||||
return retryablehttp.NewClient(opts)
|
||||
}
|
||||
|
||||
// SetScanID sets the scan id for the upload writer
|
||||
func (u *UploadWriter) SetScanID(id string) error {
|
||||
if !xidRegex.MatchString(id) {
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package pdcp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newTestUploadWriter(t *testing.T, gotKey *atomic.Value) (*UploadWriter, *httptest.Server) {
|
||||
t.Helper()
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotKey.Store(r.Header.Get(pdcpauth.ApiKeyHeaderName))
|
||||
_, _ = w.Write([]byte(`{"id":"abcdefghij0123456789"}`))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
creds := &pdcpauth.PDCPCredentials{APIKey: "test-key", Server: srv.URL}
|
||||
u, err := NewUploadWriter(context.Background(), gologger.DefaultLogger, creds)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(u.Close)
|
||||
return u, srv
|
||||
}
|
||||
|
||||
func TestUploadRejectsUntrustedCertificate(t *testing.T) {
|
||||
var gotKey atomic.Value
|
||||
u, _ := newTestUploadWriter(t, &gotKey)
|
||||
|
||||
err := u.upload([]byte("{}\n"))
|
||||
require.Error(t, err)
|
||||
var unknownAuthority x509.UnknownAuthorityError
|
||||
require.ErrorAs(t, err, &unknownAuthority)
|
||||
require.Nil(t, gotKey.Load(), "API key must not reach a server with an untrusted certificate")
|
||||
}
|
||||
|
||||
func TestUploadTrustedCertificate(t *testing.T) {
|
||||
var gotKey atomic.Value
|
||||
u, srv := newTestUploadWriter(t, &gotKey)
|
||||
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(srv.Certificate())
|
||||
transport, ok := u.client.HTTPClient.Transport.(*http.Transport)
|
||||
require.True(t, ok)
|
||||
transport.TLSClientConfig = &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}
|
||||
|
||||
require.NoError(t, u.upload([]byte("{}\n")))
|
||||
require.Equal(t, "test-key", gotKey.Load())
|
||||
require.Equal(t, "abcdefghij0123456789", u.scanID)
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/alecthomas/chroma/quick"
|
||||
"github.com/alecthomas/chroma/v2/quick"
|
||||
"github.com/logrusorgru/aurora/v4"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
//go:build integration
|
||||
// +build integration
|
||||
|
||||
package integration_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/ory/dockertest/v3"
|
||||
)
|
||||
|
||||
// ipmiSimSetup runs OpenIPMI's BMC simulator with an anonymous user,
|
||||
// ADMIN/ADMIN and root/calvin, and cipher suite 0 enabled.
|
||||
const ipmiSimSetup = `apt-get update -qq && apt-get install -y -qq --no-install-recommends openipmi >/dev/null && mkdir -p /etc/ipmi && cat > /etc/ipmi/lan.conf <<'EOF'
|
||||
name "bmc"
|
||||
set_working_mc 0x20
|
||||
startlan 1
|
||||
addr 0.0.0.0 623
|
||||
priv_limit admin
|
||||
allowed_auths_callback none md2 md5 straight
|
||||
allowed_auths_user none md2 md5 straight
|
||||
allowed_auths_operator none md2 md5 straight
|
||||
allowed_auths_admin none md2 md5 straight
|
||||
guid a123456789abcdefa123456789abcdef
|
||||
endlan
|
||||
user 1 true "" "" user 10 none md2 md5 straight
|
||||
user 2 true "ADMIN" "ADMIN" admin 10 none md2 md5 straight
|
||||
user 3 true "root" "calvin" admin 10 none md2 md5 straight
|
||||
EOF
|
||||
printf 'mc_setbmc 0x20\nmc_add 0x20 0 no-device-sdrs 0x23 9 8 0x9f 0x1291 0xf02 persist_sdr\nmc_enable 0x20\n' > /etc/ipmi/emu.cmd
|
||||
exec ipmi_sim -c /etc/ipmi/lan.conf -f /etc/ipmi/emu.cmd -n`
|
||||
|
||||
type javascriptIPMI struct{}
|
||||
|
||||
func (j *javascriptIPMI) Execute(filePath string) error {
|
||||
pool, err := dockertest.NewPool("")
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create docker pool: %w", err)
|
||||
}
|
||||
if err := pool.Client.Ping(); err != nil {
|
||||
return fmt.Errorf("could not connect to Docker: %w", err)
|
||||
}
|
||||
resource, err := pool.RunWithOptions(&dockertest.RunOptions{
|
||||
Repository: "debian",
|
||||
Tag: "bookworm-slim",
|
||||
Platform: "linux/amd64",
|
||||
Cmd: []string{"sh", "-c", ipmiSimSetup},
|
||||
ExposedPorts: []string{"623/udp"},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not start ipmi simulator: %w", err)
|
||||
}
|
||||
defer purge(pool, resource)
|
||||
|
||||
address := net.JoinHostPort("127.0.0.1", resource.GetPort("623/udp"))
|
||||
if err := waitForServiceCheck(address, 3*time.Minute, rmcpPingCheck); err != nil {
|
||||
return err
|
||||
}
|
||||
results, err := runSignedNucleiTemplateAndGetResults(filePath, address, debug)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
||||
// rmcpPingCheck waits for an answer to an RMCP presence ping, since the
|
||||
// port is udp and cannot be probed with a connect.
|
||||
func rmcpPingCheck(address string) error {
|
||||
conn, err := net.Dial("udp", address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
if _, err := conn.Write([]byte{0x06, 0x00, 0xff, 0x06, 0x00, 0x00, 0x11, 0xbe, 0x80, 0x00, 0x00, 0x00}); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = conn.SetReadDeadline(time.Now().Add(time.Second))
|
||||
reply := make([]byte, 64)
|
||||
n, err := conn.Read(reply)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n < 9 || reply[8] != 0x40 {
|
||||
return fmt.Errorf("unexpected rmcp reply")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -46,6 +46,7 @@ var jsTestcases = []integrationCase{
|
||||
{Path: "protocols/javascript/http-get.yaml", TestCase: &javascriptHTTPGet{}},
|
||||
{Path: "protocols/javascript/http-client-flow.yaml", TestCase: &javascriptHTTPClientFlow{}},
|
||||
{Path: "protocols/javascript/http-denied.yaml", TestCase: &javascriptHTTPDenied{}},
|
||||
{Path: "protocols/javascript/ipmi.yaml", TestCase: &javascriptIPMI{}, DisableOn: javascriptDockerDisabled, Serial: true},
|
||||
{Path: "protocols/javascript/rsync-test.yaml", TestCase: &javascriptRsyncTest{}, DisableOn: javascriptDockerDisabled, Serial: true},
|
||||
{Path: "protocols/javascript/vnc-pass-brute.yaml", TestCase: &javascriptVncPassBrute{}, DisableOn: javascriptDockerDisabled, Serial: true},
|
||||
{Path: "protocols/javascript/postgres-pass-brute.yaml", TestCase: &javascriptPostgresPassBrute{}, DisableOn: javascriptDockerDisabled, Serial: true},
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
id: ipmi-checks
|
||||
|
||||
info:
|
||||
name: IPMI Checks
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
javascript:
|
||||
- code: |
|
||||
const ipmi = require('nuclei/ipmi');
|
||||
const caps = ipmi.GetAuthCapabilities(Host, Port);
|
||||
const root = ipmi.GetRAKPHash(Host, Port, 'root');
|
||||
|
||||
// cipher suite 0 lets a wrong password in
|
||||
const anyPassword = new ipmi.Client();
|
||||
const cipherZero = anyPassword.Login(Host, Port, 'ADMIN', 'wrong', 0);
|
||||
anyPassword.Close();
|
||||
|
||||
// an encrypted, signed session, then Get Device ID and admin privilege
|
||||
const client = new ipmi.Client();
|
||||
client.Login(Host, Port, 'ADMIN', 'ADMIN', 3);
|
||||
const device = client.Command(0x06, 0x01, []);
|
||||
const admin = client.Command(0x06, 0x3b, [0x04]);
|
||||
client.Close();
|
||||
const manufacturer = device.Data[6] | device.Data[7] << 8 | device.Data[8] << 16;
|
||||
|
||||
let wrongRejected = false;
|
||||
try { new ipmi.Client().Login(Host, Port, 'ADMIN', 'wrong', 3); } catch (e) { wrongRejected = true; }
|
||||
|
||||
ipmi.Ping(Host, Port) && caps.IPMI20 && caps.AnonymousLogin &&
|
||||
ipmi.GetCipherSuites(Host, Port).includes(0) &&
|
||||
ipmi.GetSystemGUID(Host, Port).length == 32 &&
|
||||
ipmi.FindRAKPPassword(root, ['admin', 'calvin']) == 'calvin' &&
|
||||
!ipmi.CheckRAKPPassword(root, 'admin') &&
|
||||
cipherZero && wrongRejected &&
|
||||
device.CompletionCode == 0 && manufacturer == 4753 &&
|
||||
admin.CompletionCode == 0;
|
||||
|
||||
args:
|
||||
Host: "{{Host}}"
|
||||
Port: "623"
|
||||
|
||||
matchers:
|
||||
- type: dsl
|
||||
dsl:
|
||||
- "success == true"
|
||||
@@ -7,18 +7,22 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/alecthomas/chroma/quick"
|
||||
"github.com/alecthomas/chroma/v2/quick"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
)
|
||||
|
||||
const (
|
||||
var (
|
||||
aiTemplateGeneratorAPIEndpoint = "https://api.projectdiscovery.io/v1/template/ai"
|
||||
// aiTemplateHTTPClient verifies TLS, since the request carries the PDCP API key.
|
||||
aiTemplateHTTPClient = &http.Client{}
|
||||
// aiTemplateIDRegex keeps the template file name inside the pdcp directory.
|
||||
aiTemplateIDRegex = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
|
||||
)
|
||||
|
||||
type AITemplateResponse struct {
|
||||
@@ -112,7 +116,7 @@ func generateAITemplate(prompt string) (string, string, error) {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set(pdcpauth.ApiKeyHeaderName, creds.APIKey)
|
||||
|
||||
resp, err := retryablehttp.DefaultClient().Do(req)
|
||||
resp, err := aiTemplateHTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return "", "", errkit.Newf("Failed to send HTTP request: %v", err)
|
||||
}
|
||||
@@ -138,5 +142,9 @@ func generateAITemplate(prompt string) (string, string, error) {
|
||||
return "", "", errkit.Newf("Failed to generate template")
|
||||
}
|
||||
|
||||
if !aiTemplateIDRegex.MatchString(result.TemplateID) {
|
||||
return "", "", errkit.Newf("Invalid template ID in API response: %q", result.TemplateID)
|
||||
}
|
||||
|
||||
return result.Completion, result.TemplateID, nil
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newAITemplateServer(t *testing.T, templateID string, gotKey *atomic.Value) {
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotKey.Store(r.Header.Get("X-Api-Key"))
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{
|
||||
"completion": "id: test\n",
|
||||
"template_id": templateID,
|
||||
})
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
oldEndpoint, oldClient := aiTemplateGeneratorAPIEndpoint, aiTemplateHTTPClient
|
||||
t.Cleanup(func() { aiTemplateGeneratorAPIEndpoint, aiTemplateHTTPClient = oldEndpoint, oldClient })
|
||||
aiTemplateGeneratorAPIEndpoint = srv.URL
|
||||
t.Setenv("PDCP_API_KEY", "test-key")
|
||||
if templateID != "" {
|
||||
// Trust the test certificate, to check the template_id only.
|
||||
aiTemplateHTTPClient = srv.Client()
|
||||
}
|
||||
}
|
||||
|
||||
func TestAITemplateVerifiesTLS(t *testing.T) {
|
||||
var gotKey atomic.Value
|
||||
newAITemplateServer(t, "", &gotKey)
|
||||
|
||||
_, _, err := generateAITemplate("check for exposed admin panels")
|
||||
require.ErrorContains(t, err, "certificate")
|
||||
require.Nil(t, gotKey.Load(), "the API key went to an untrusted server")
|
||||
}
|
||||
|
||||
func TestAITemplateRejectsUnsafeID(t *testing.T) {
|
||||
for _, id := range []string{"../../.config/nuclei/config", "a/b", `a\b`, "a.b", "a b"} {
|
||||
var gotKey atomic.Value
|
||||
newAITemplateServer(t, id, &gotKey)
|
||||
_, _, err := generateAITemplate("check for exposed admin panels")
|
||||
require.Error(t, err, id)
|
||||
}
|
||||
|
||||
var gotKey atomic.Value
|
||||
newAITemplateServer(t, "Abc-123_x", &gotKey)
|
||||
_, templateID, err := generateAITemplate("check for exposed admin panels")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "Abc-123_x", templateID)
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package core
|
||||
import (
|
||||
"fmt"
|
||||
"net/http/cookiejar"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
@@ -32,22 +33,45 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
|
||||
templateThreads++
|
||||
}
|
||||
swg, _ := syncutil.New(syncutil.WithSize(templateThreads))
|
||||
// pending tracks every started step until it returns. swg only limits
|
||||
// concurrency and its slots can be released early, so completion is
|
||||
// tracked here instead.
|
||||
var pending sync.WaitGroup
|
||||
|
||||
for _, template := range w.Workflows {
|
||||
newCtx := scan.NewScanContext(ctx.Context(), ctx.Input.Clone())
|
||||
if err := e.runWorkflowStep(template, newCtx, results, swg, w); err != nil {
|
||||
// Root steps do not own a waitgroup slot.
|
||||
if err := e.runWorkflowStep(template, newCtx, results, swg, &pending, w, nil); err != nil {
|
||||
gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
|
||||
}
|
||||
}
|
||||
|
||||
swg.Wait()
|
||||
pending.Wait()
|
||||
|
||||
return results.Load()
|
||||
}
|
||||
|
||||
// slotRelease returns a func that gives back one swg slot. It is safe to call
|
||||
// from several goroutines and several times; only the first call releases.
|
||||
func slotRelease(swg *syncutil.AdaptiveWaitGroup) func() {
|
||||
var once sync.Once
|
||||
return func() {
|
||||
once.Do(swg.Done)
|
||||
}
|
||||
}
|
||||
|
||||
// runWorkflowStep runs a workflow step for the workflow. It executes the workflow
|
||||
// in a recursive manner running all subtemplates and matchers.
|
||||
func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *syncutil.AdaptiveWaitGroup, w *workflows.Workflow) error {
|
||||
//
|
||||
// releaseSlot, when non-nil, releases the AdaptiveWaitGroup slot held by the
|
||||
// goroutine that is running this step. It must be called before this step
|
||||
// blocks on swg.AddWithContext for child steps; otherwise every in-flight
|
||||
// parent can hold a slot while waiting for another, which deadlocks when the
|
||||
// number of matching nested steps reaches TemplateThreads (see #8074).
|
||||
//
|
||||
// pending is incremented for every child step before it starts, and each child
|
||||
// calls Done when it returns, so the root can wait for the whole tree.
|
||||
func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *syncutil.AdaptiveWaitGroup, pending *sync.WaitGroup, w *workflows.Workflow, releaseSlot func()) error {
|
||||
var firstMatched bool
|
||||
var err error
|
||||
var mainErr error
|
||||
@@ -131,18 +155,25 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
|
||||
continue
|
||||
}
|
||||
|
||||
// Free this step's slot before blocking on child acquires.
|
||||
if releaseSlot != nil {
|
||||
releaseSlot()
|
||||
}
|
||||
for _, subtemplate := range matcher.Subtemplates {
|
||||
if err := swg.AddWithContext(ctx.Context()); err != nil {
|
||||
return
|
||||
}
|
||||
pending.Add(1)
|
||||
|
||||
go func(subtemplate *workflows.WorkflowTemplate) {
|
||||
defer swg.Done()
|
||||
defer pending.Done()
|
||||
release := slotRelease(swg)
|
||||
defer release()
|
||||
|
||||
// create a new context with the same input but with unset callbacks
|
||||
// clone the Input so that other parallel executions won't overwrite the shared variables when subsequent templates are running
|
||||
subCtx := scan.NewScanContext(ctx.Context(), ctx.Input.Clone())
|
||||
if err := e.runWorkflowStep(subtemplate, subCtx, results, swg, w); err != nil {
|
||||
if err := e.runWorkflowStep(subtemplate, subCtx, results, swg, pending, w, release); err != nil {
|
||||
gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err)
|
||||
}
|
||||
}(subtemplate)
|
||||
@@ -162,18 +193,27 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
|
||||
return mainErr
|
||||
}
|
||||
if len(template.Subtemplates) > 0 && firstMatched {
|
||||
// Free this step's slot before blocking on child acquires so nested
|
||||
// matching steps cannot exhaust TemplateThreads and deadlock (#8074).
|
||||
if releaseSlot != nil {
|
||||
releaseSlot()
|
||||
}
|
||||
for _, subtemplate := range template.Subtemplates {
|
||||
if err := swg.AddWithContext(ctx.Context()); err != nil {
|
||||
return err
|
||||
}
|
||||
pending.Add(1)
|
||||
|
||||
go func(template *workflows.WorkflowTemplate) {
|
||||
defer pending.Done()
|
||||
release := slotRelease(swg)
|
||||
defer release()
|
||||
|
||||
// create a new context with the same input but with unset callbacks
|
||||
subCtx := scan.NewScanContext(ctx.Context(), ctx.Input.Clone())
|
||||
if err := e.runWorkflowStep(template, subCtx, results, swg, w); err != nil {
|
||||
if err := e.runWorkflowStep(template, subCtx, results, swg, pending, w, release); err != nil {
|
||||
gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
|
||||
}
|
||||
swg.Done()
|
||||
}(subtemplate)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -303,6 +303,89 @@ func TestWorkflowsSameStepExecutersRefreshTemplateContext(t *testing.T) {
|
||||
require.Equal(t, "foo", secondWhoami, "could not refresh workflow context into later executers in the same step")
|
||||
}
|
||||
|
||||
func TestWorkflowNestedSubtemplatesDoNotDeadlockAtConcurrencyLimit(t *testing.T) {
|
||||
// Repro for #8074: with TemplateThreads equal to the number of matching
|
||||
// nested parents, each parent used to hold a waitgroup slot while calling
|
||||
// AddWithContext for its children, so every slot was occupied by a parent
|
||||
// waiting for another slot and the workflow never finished.
|
||||
progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
|
||||
matchOutputs := []*output.InternalWrappedEvent{{
|
||||
OperatorsResult: &operators.Result{},
|
||||
Results: []*output.ResultEvent{{}},
|
||||
}}
|
||||
|
||||
const parents = 3
|
||||
var entered atomic.Int32
|
||||
allHolding := make(chan struct{})
|
||||
releaseParents := make(chan struct{})
|
||||
leafRuns := &atomic.Int32{}
|
||||
|
||||
leaf := &mockExecuter{result: true, executeScanHook: func(ctx *scan.ScanContext) {
|
||||
leafRuns.Add(1)
|
||||
}}
|
||||
parentExec := &mockExecuter{
|
||||
result: true,
|
||||
outputs: matchOutputs,
|
||||
executeScanHook: func(ctx *scan.ScanContext) {
|
||||
if entered.Add(1) == parents {
|
||||
close(allHolding)
|
||||
}
|
||||
// Stay inside ExecuteWithResults so the parent still owns its
|
||||
// waitgroup slot while every peer also holds one.
|
||||
<-releaseParents
|
||||
},
|
||||
}
|
||||
mkParent := func() *workflows.WorkflowTemplate {
|
||||
return &workflows.WorkflowTemplate{
|
||||
Executers: []*workflows.ProtocolExecuterPair{{
|
||||
Executer: parentExec,
|
||||
Options: &protocols.ExecutorOptions{Progress: progressBar},
|
||||
}},
|
||||
Subtemplates: []*workflows.WorkflowTemplate{{
|
||||
Executers: []*workflows.ProtocolExecuterPair{{
|
||||
Executer: leaf,
|
||||
Options: &protocols.ExecutorOptions{Progress: progressBar},
|
||||
}},
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
workflow := &workflows.Workflow{
|
||||
Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: parents}},
|
||||
Workflows: []*workflows.WorkflowTemplate{{
|
||||
Executers: []*workflows.ProtocolExecuterPair{{
|
||||
Executer: &mockExecuter{result: true, outputs: matchOutputs},
|
||||
Options: &protocols.ExecutorOptions{Progress: progressBar},
|
||||
}},
|
||||
Subtemplates: []*workflows.WorkflowTemplate{mkParent(), mkParent(), mkParent()},
|
||||
}},
|
||||
}
|
||||
|
||||
engine := &Engine{}
|
||||
input := contextargs.NewWithInput(context.Background(), "https://test.com")
|
||||
ctx := scan.NewScanContext(context.Background(), input)
|
||||
|
||||
done := make(chan bool, 1)
|
||||
go func() {
|
||||
done <- engine.executeWorkflow(ctx, workflow)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-allHolding:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("nested parents did not all acquire TemplateThreads slots")
|
||||
}
|
||||
close(releaseParents)
|
||||
|
||||
select {
|
||||
case matched := <-done:
|
||||
require.True(t, matched, "nested workflow should report a match")
|
||||
require.EqualValues(t, parents, leafRuns.Load(), "each nested parent should run its leaf")
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("nested workflow deadlocked at TemplateThreads == matching nested parents")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkflowSubtemplatesUnblockCancelledAcquire(t *testing.T) {
|
||||
progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
|
||||
var started atomic.Int32
|
||||
|
||||
+14
-18
@@ -155,23 +155,14 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) {
|
||||
baseInteractURLs := append([]string(nil), input.InteractURLs...)
|
||||
if rule.generator == nil {
|
||||
for _, component := range finalComponentList {
|
||||
var dataKeys map[string]struct{}
|
||||
|
||||
optionVars := rule.options.Options.Vars.AsMap()
|
||||
constants := rule.options.Constants
|
||||
dataValues := rule.options.NewVariablesScope(baseValues, optionVars, constants)
|
||||
// get vars from variables while replacing interactsh urls
|
||||
// Render variables once, allocating their interactsh URLs. The
|
||||
// output may contain runtime data, so it is never rendered again.
|
||||
evaluation := rule.options.Variables.EvaluateWithInteractshScope(dataValues, rule.options.Interactsh)
|
||||
evaluatedValues, interactURLs := evaluation.Values, evaluation.InteractURLs
|
||||
input.Values, dataKeys = mergeFuzzValueLayers(
|
||||
fuzzValueLayer{values: evaluatedValues, kind: fuzzValueTemplate},
|
||||
fuzzValueLayer{values: baseValues, kind: fuzzValueData},
|
||||
fuzzValueLayer{values: optionVars, kind: fuzzValueData},
|
||||
fuzzValueLayer{values: constants, kind: fuzzValueData},
|
||||
)
|
||||
// Replace template-text interactsh markers without evaluating runtime values.
|
||||
input.Values, interactURLs = rule.evaluateVarsWithInteractsh(input.Values, interactURLs, dataKeys)
|
||||
input.InteractURLs = mergeInteractURLs(baseInteractURLs, interactURLs)
|
||||
input.Values = generators.MergeMaps(evaluation.Values, baseValues, optionVars, constants)
|
||||
input.InteractURLs = mergeInteractURLs(baseInteractURLs, evaluation.InteractURLs)
|
||||
|
||||
err := rule.executeRuleValues(input, component)
|
||||
if err != nil {
|
||||
@@ -240,15 +231,17 @@ func (rule *Rule) prepareGeneratorValues(values, baseValues map[string]interface
|
||||
evaluation := rule.options.Variables.EvaluateWithInteractshScope(evaluationValues, rule.options.Interactsh)
|
||||
evaluatedValues, interactURLs := evaluation.Values, evaluation.InteractURLs
|
||||
inputValues, dataKeys := mergeFuzzValueLayers(
|
||||
fuzzValueLayer{values: evaluatedValues, kind: fuzzValueData},
|
||||
fuzzValueLayer{values: values, kind: fuzzValueTemplate},
|
||||
fuzzValueLayer{values: evaluatedValues, kind: fuzzValueTemplate},
|
||||
fuzzValueLayer{values: baseValues, kind: fuzzValueData},
|
||||
fuzzValueLayer{values: optionVars, kind: fuzzValueData},
|
||||
fuzzValueLayer{values: constants, kind: fuzzValueData},
|
||||
)
|
||||
|
||||
// Generator values are template-authored payload definitions until this
|
||||
// render. Runtime/base, CLI option, and constant values remain data.
|
||||
// render. Rendered variables, runtime/base, CLI option, and constant
|
||||
// values remain data. Payloads layer over variables so a payload that
|
||||
// shares a variable's name is still rendered.
|
||||
return rule.evaluateVarsWithInteractsh(inputValues, interactURLs, dataKeys)
|
||||
}
|
||||
|
||||
@@ -326,7 +319,7 @@ func (rule *Rule) evaluateVars(input string) (string, error) {
|
||||
func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, interactshUrls []string, dataKeys map[string]struct{}) (map[string]interface{}, []string) {
|
||||
// Check if Interactsh options are configured
|
||||
if rule.options.Interactsh != nil {
|
||||
data = maps.Clone(data)
|
||||
rendered := maps.Clone(data)
|
||||
|
||||
interactshUrlsMap := make(map[string]struct{})
|
||||
for _, url := range interactshUrls {
|
||||
@@ -335,7 +328,8 @@ func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, intera
|
||||
|
||||
interactshUrls = mapsutil.GetKeys(interactshUrlsMap)
|
||||
|
||||
// Iterate through template-text data to replace Interactsh URL markers.
|
||||
// Render each template-text value against the unrendered input, so
|
||||
// the result does not depend on map iteration order.
|
||||
for k, v := range data {
|
||||
if _, ok := dataKeys[k]; ok {
|
||||
continue
|
||||
@@ -349,7 +343,7 @@ func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, intera
|
||||
InteractURLs: interactshUrls,
|
||||
})
|
||||
if err == nil {
|
||||
data[k] = got.Text
|
||||
rendered[k] = got.Text
|
||||
for _, url := range got.InteractURLs {
|
||||
if _, ok := interactshUrlsMap[url]; ok {
|
||||
continue
|
||||
@@ -360,6 +354,8 @@ func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, intera
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data = rendered
|
||||
}
|
||||
|
||||
return data, interactshUrls
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
package fuzz
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/component"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
// requestBodyWithDSL is request data that looks like a DSL expression.
|
||||
requestBodyWithDSL = `PLAIN-{{to_upper("abc")}}-END`
|
||||
queryRuleTarget = "http://127.0.0.1/echo?p=SEED"
|
||||
)
|
||||
|
||||
// fakeInteractshClient hands out numbered URLs without contacting a server.
|
||||
// Fuzz rules only allocate URLs through NewURLWithData, so the embedded nil
|
||||
// IClient is never called.
|
||||
type fakeInteractshClient struct {
|
||||
interactsh.IClient
|
||||
|
||||
issued int
|
||||
}
|
||||
|
||||
func (c *fakeInteractshClient) NewURLWithData(string) (string, error) {
|
||||
c.issued++
|
||||
|
||||
return fmt.Sprintf("oast%d.example", c.issued), nil
|
||||
}
|
||||
|
||||
// newQueryRule compiles a single-mode rule that replaces the p query value
|
||||
// with fuzzValue.
|
||||
func newQueryRule(
|
||||
t *testing.T,
|
||||
fuzzValue string,
|
||||
generator *generators.PayloadGenerator,
|
||||
templateVars variables.Variable,
|
||||
client interactsh.IClient,
|
||||
) *Rule {
|
||||
t.Helper()
|
||||
|
||||
rule := &Rule{
|
||||
Part: component.RequestQueryComponent,
|
||||
Type: "replace",
|
||||
Mode: "single",
|
||||
Fuzz: SliceOrMapSlice{Value: []string{fuzzValue}},
|
||||
}
|
||||
err := rule.Compile(generator, &protocols.ExecutorOptions{
|
||||
Variables: templateVars,
|
||||
Options: &types.Options{},
|
||||
Interactsh: client,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
return rule
|
||||
}
|
||||
|
||||
// executeQueryRule runs rule against queryRuleTarget and returns the generated
|
||||
// requests.
|
||||
func executeQueryRule(
|
||||
t *testing.T,
|
||||
rule *Rule,
|
||||
method string,
|
||||
body io.Reader,
|
||||
values map[string]any,
|
||||
) []GeneratedRequest {
|
||||
t.Helper()
|
||||
|
||||
req, err := retryablehttp.NewRequest(method, queryRuleTarget, body)
|
||||
require.NoError(t, err)
|
||||
|
||||
var got []GeneratedRequest
|
||||
err = rule.Execute(&ExecuteRuleInput{
|
||||
Input: contextargs.NewWithInput(context.Background(), queryRuleTarget),
|
||||
BaseRequest: req,
|
||||
Values: values,
|
||||
Callback: func(gr GeneratedRequest) bool {
|
||||
got = append(got, gr)
|
||||
|
||||
return true
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
return got
|
||||
}
|
||||
|
||||
// newRequestDataVariableRule returns a query rule whose template declares
|
||||
// b: "{{body}}", so b captures request data.
|
||||
func newRequestDataVariableRule(t *testing.T, fuzzValue string, payloads map[string]any) *Rule {
|
||||
t.Helper()
|
||||
|
||||
templateVars := newFuzzTestVariables()
|
||||
templateVars.Set("b", "{{body}}")
|
||||
|
||||
var generator *generators.PayloadGenerator
|
||||
|
||||
if payloads != nil {
|
||||
var err error
|
||||
|
||||
generator, err = generators.New(
|
||||
payloads, generators.BatteringRamAttack, "", disk.NewCatalog(""), "", types.DefaultOptions(),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// The CLI creates an Interactsh client even with -no-interactsh.
|
||||
return newQueryRule(t, fuzzValue, generator, templateVars, &interactsh.Client{})
|
||||
}
|
||||
|
||||
// fuzzedQueryValues runs rule against a POST request whose body is
|
||||
// requestBodyWithDSL and returns the fuzzed p query value of each generated
|
||||
// request.
|
||||
func fuzzedQueryValues(t *testing.T, rule *Rule) []string {
|
||||
t.Helper()
|
||||
|
||||
generated := executeQueryRule(t, rule, http.MethodPost, strings.NewReader(requestBodyWithDSL), map[string]any{
|
||||
"body": requestBodyWithDSL,
|
||||
})
|
||||
|
||||
got := make([]string, 0, len(generated))
|
||||
for _, gr := range generated {
|
||||
got = append(got, gr.Request.URL.Query().Get("p"))
|
||||
}
|
||||
|
||||
return got
|
||||
}
|
||||
|
||||
func TestExecuteKeepsRequestDataInTemplateVariableLiteral(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("fuzz value", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
rule := newRequestDataVariableRule(t, "{{b}}", nil)
|
||||
|
||||
require.Equal(t, []string{requestBodyWithDSL}, fuzzedQueryValues(t, rule))
|
||||
})
|
||||
|
||||
t.Run("payload value", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
rule := newRequestDataVariableRule(t, "{{inj}}", map[string]any{
|
||||
"inj": []any{"{{b}}"},
|
||||
})
|
||||
|
||||
// Each run iterates a fresh value map. Repeat so the result
|
||||
// cannot depend on map iteration order.
|
||||
for range 256 {
|
||||
require.Equal(t, []string{requestBodyWithDSL}, fuzzedQueryValues(t, rule))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestExecuteRendersTemplateVariableInteractshMarker(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
templateVars := newFuzzTestVariables()
|
||||
templateVars.Set("marker", "{{interactsh-url}}")
|
||||
rule := newQueryRule(t, "{{marker}}", nil, templateVars, &fakeInteractshClient{})
|
||||
|
||||
got := executeQueryRule(t, rule, http.MethodGet, nil, nil)
|
||||
|
||||
require.Len(t, got, 1)
|
||||
require.Equal(t, []string{"oast1.example"}, got[0].InteractURLs)
|
||||
require.Equal(t, "oast1.example", got[0].Request.URL.Query().Get("p"))
|
||||
}
|
||||
|
||||
func TestPrepareGeneratorValuesRendersPayloadSharingVariableName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
templateVars := newFuzzTestVariables()
|
||||
templateVars.Set("inj", "variable")
|
||||
|
||||
rule := &Rule{
|
||||
options: &protocols.ExecutorOptions{
|
||||
Variables: templateVars,
|
||||
Options: &types.Options{},
|
||||
Interactsh: &fakeInteractshClient{},
|
||||
},
|
||||
}
|
||||
|
||||
values, _ := rule.prepareGeneratorValues(map[string]any{
|
||||
"inj": `{{to_upper("payload")}}`,
|
||||
}, nil)
|
||||
|
||||
require.Equal(t, "PAYLOAD", values["inj"])
|
||||
}
|
||||
|
||||
func TestPrepareGeneratorValuesRendersPayloadsAgainstUnrenderedInput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
rule := &Rule{
|
||||
options: &protocols.ExecutorOptions{
|
||||
Variables: newFuzzTestVariables(),
|
||||
Options: &types.Options{},
|
||||
Interactsh: &interactsh.Client{},
|
||||
},
|
||||
}
|
||||
|
||||
for range 256 {
|
||||
values, _ := rule.prepareGeneratorValues(map[string]any{
|
||||
"upper": `{{to_upper("x")}}`,
|
||||
"ref": "{{upper}}",
|
||||
}, nil)
|
||||
|
||||
require.Equal(t, "X", values["upper"])
|
||||
// ref sees the unrendered definition of upper as data.
|
||||
require.Equal(t, `{{to_upper("x")}}`, values["ref"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package compiler_test
|
||||
|
||||
//nolint:depguard // No depguard rules are configured, so its default list allows only the standard library.
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
)
|
||||
|
||||
// TestCloseWithoutConnection calls Close on clients that never connected.
|
||||
// The script must keep running past the call.
|
||||
func TestCloseWithoutConnection(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
scripts := map[string]string{
|
||||
"ssh": `
|
||||
const ssh = require('nuclei/ssh');
|
||||
ExportAs('closed', new ssh.SSHClient().Close());
|
||||
true;
|
||||
`,
|
||||
"net": `
|
||||
const net = require('nuclei/net');
|
||||
new net.NetConn().Close();
|
||||
ExportAs('closed', true);
|
||||
true;
|
||||
`,
|
||||
}
|
||||
for name, script := range scripts {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
program, err := compiler.SourceAutoMode(script, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
opts := new(compiler.ExecuteOptions)
|
||||
opts.ExecutionId = t.Name()
|
||||
opts.Source = &script // ExportAs is defined only for sources that use it.
|
||||
opts.TimeoutVariants = types.NewTimeoutVariant(5)
|
||||
|
||||
result, err := compiler.New().ExecuteWithOptions(t.Context(), program, compiler.NewExecuteArgs(), opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got := result["closed"]; got != "true" {
|
||||
t.Errorf(`result["closed"] = %v, want "true"`, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -24,12 +24,14 @@ import (
|
||||
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libhash"
|
||||
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libhttp"
|
||||
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libikev2"
|
||||
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libipmi"
|
||||
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libkerberos"
|
||||
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libkrbforge"
|
||||
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libkrbroast"
|
||||
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libldap"
|
||||
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libmssql"
|
||||
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libmysql"
|
||||
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libmqtt"
|
||||
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libnet"
|
||||
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/liboracle"
|
||||
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libpop3"
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package ipmi
|
||||
|
||||
import (
|
||||
lib_ipmi "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/ipmi"
|
||||
|
||||
"github.com/projectdiscovery/goja"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
|
||||
)
|
||||
|
||||
var (
|
||||
module = gojs.NewGojaModule("nuclei/ipmi")
|
||||
)
|
||||
|
||||
func init() {
|
||||
module.Set(
|
||||
gojs.Objects{
|
||||
// Functions
|
||||
"CheckRAKPPassword": lib_ipmi.CheckRAKPPassword,
|
||||
"FindRAKPPassword": lib_ipmi.FindRAKPPassword,
|
||||
"GetAuthCapabilities": lib_ipmi.GetAuthCapabilities,
|
||||
"GetCipherSuites": lib_ipmi.GetCipherSuites,
|
||||
"GetRAKPHash": lib_ipmi.GetRAKPHash,
|
||||
"GetSystemGUID": lib_ipmi.GetSystemGUID,
|
||||
"Ping": lib_ipmi.Ping,
|
||||
|
||||
// Var and consts
|
||||
|
||||
// Objects / Classes
|
||||
"AuthCapabilities": gojs.GetClassConstructor[lib_ipmi.AuthCapabilities](&lib_ipmi.AuthCapabilities{}),
|
||||
"Client": gojs.GetClassConstructor[lib_ipmi.Client](&lib_ipmi.Client{}),
|
||||
"CommandResponse": gojs.GetClassConstructor[lib_ipmi.CommandResponse](&lib_ipmi.CommandResponse{}),
|
||||
"RAKPHash": gojs.GetClassConstructor[lib_ipmi.RAKPHash](&lib_ipmi.RAKPHash{}),
|
||||
},
|
||||
).Register()
|
||||
}
|
||||
|
||||
func Enable(runtime *goja.Runtime) {
|
||||
module.Enable(runtime)
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package mqtt
|
||||
|
||||
import (
|
||||
lib_mqtt "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/mqtt"
|
||||
|
||||
"github.com/projectdiscovery/goja"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
|
||||
)
|
||||
|
||||
var (
|
||||
module = gojs.NewGojaModule("nuclei/mqtt")
|
||||
)
|
||||
|
||||
func init() {
|
||||
module.Set(
|
||||
gojs.Objects{
|
||||
// Functions
|
||||
"IsMQTT": lib_mqtt.IsMQTT,
|
||||
"IsAnonymousAllowed": lib_mqtt.IsAnonymousAllowed,
|
||||
|
||||
// Var and consts
|
||||
|
||||
// Objects / Classes
|
||||
"Client": gojs.GetClassConstructor[lib_mqtt.Client](&lib_mqtt.Client{}),
|
||||
"ConnectOptions": gojs.GetClassConstructor[lib_mqtt.ConnectOptions](&lib_mqtt.ConnectOptions{}),
|
||||
"ConnectResult": gojs.GetClassConstructor[lib_mqtt.ConnectResult](&lib_mqtt.ConnectResult{}),
|
||||
"Message": gojs.GetClassConstructor[lib_mqtt.Message](&lib_mqtt.Message{}),
|
||||
},
|
||||
).Register()
|
||||
}
|
||||
|
||||
func Enable(runtime *goja.Runtime) {
|
||||
module.Enable(runtime)
|
||||
}
|
||||
@@ -8,12 +8,14 @@ export * as grpc from './grpc';
|
||||
export * as hash from './hash';
|
||||
export * as http from './http';
|
||||
export * as ikev2 from './ikev2';
|
||||
export * as ipmi from './ipmi';
|
||||
export * as kerberos from './kerberos';
|
||||
export * as krbforge from './krbforge';
|
||||
export * as krbroast from './krbroast';
|
||||
export * as ldap from './ldap';
|
||||
export * as mssql from './mssql';
|
||||
export * as mysql from './mysql';
|
||||
export * as mqtt from './mqtt';
|
||||
export * as net from './net';
|
||||
export * as oracle from './oracle';
|
||||
export * as pop3 from './pop3';
|
||||
|
||||
Executable
+304
@@ -0,0 +1,304 @@
|
||||
|
||||
|
||||
/**
|
||||
* CheckRAKPPassword reports whether password produced hash. It runs offline,
|
||||
* so it can test any number of passwords without touching the BMC.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const ipmi = require('nuclei/ipmi');
|
||||
* const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
|
||||
* log(ipmi.CheckRAKPPassword(hash, 'ADMIN'));
|
||||
* ```
|
||||
*/
|
||||
export function CheckRAKPPassword(hash: RAKPHash, password: string): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* FindRAKPPassword returns the first of passwords that produced hash, or an
|
||||
* empty string.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const ipmi = require('nuclei/ipmi');
|
||||
* const hash = ipmi.GetRAKPHash('acme.com', 623, 'root');
|
||||
* log(ipmi.FindRAKPPassword(hash, ['calvin', 'root', 'admin']));
|
||||
* ```
|
||||
*/
|
||||
export function FindRAKPPassword(hash: RAKPHash, passwords: string[]): string {
|
||||
return "";
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* GetAuthCapabilities returns the authentication capabilities the BMC
|
||||
* advertises on the current channel.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const ipmi = require('nuclei/ipmi');
|
||||
* const caps = ipmi.GetAuthCapabilities('acme.com', 623);
|
||||
* log(caps.AnonymousLogin);
|
||||
* ```
|
||||
*/
|
||||
export function GetAuthCapabilities(ctx: any, host: string, port: number): AuthCapabilities | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* GetCipherSuites returns the IDs of the RMCP+ cipher suites the BMC
|
||||
* offers. Cipher suite 0 authenticates nothing, so any password logs in.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const ipmi = require('nuclei/ipmi');
|
||||
* const suites = ipmi.GetCipherSuites('acme.com', 623);
|
||||
* log(suites.includes(0));
|
||||
* ```
|
||||
*/
|
||||
export function GetCipherSuites(ctx: any, host: string, port: number): number[] | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* GetRAKPHash starts an RMCP+ session as username and returns the HMAC the
|
||||
* BMC answers with. The session is never completed, so no login is
|
||||
* attempted; the BMC drops the half-open session after its timeout. It fails
|
||||
* when the BMC rejects the username.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const ipmi = require('nuclei/ipmi');
|
||||
* const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
|
||||
* log(ipmi.CheckRAKPPassword(hash, 'ADMIN'));
|
||||
* ```
|
||||
*/
|
||||
export function GetRAKPHash(ctx: any, host: string, port: number, username: string): RAKPHash | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* GetSystemGUID returns the BMC system GUID, which vendors derive from
|
||||
* serial numbers or MAC addresses.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const ipmi = require('nuclei/ipmi');
|
||||
* const guid = ipmi.GetSystemGUID('acme.com', 623);
|
||||
* log(guid);
|
||||
* ```
|
||||
*/
|
||||
export function GetSystemGUID(ctx: any, host: string, port: number): string | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Ping reports whether the host answers an RMCP presence ping and advertises
|
||||
* IPMI support.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const ipmi = require('nuclei/ipmi');
|
||||
* const isIPMI = ipmi.Ping('acme.com', 623);
|
||||
* log(isIPMI);
|
||||
* ```
|
||||
*/
|
||||
export function Ping(ctx: any, host: string, port: number): boolean | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Client holds one RMCP+ session, for commands that need a login.
|
||||
* Close it when done: BMCs allow only a few sessions at a time.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const ipmi = require('nuclei/ipmi');
|
||||
* const client = new ipmi.Client();
|
||||
* client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
|
||||
* const id = client.Command(0x06, 0x01, []);
|
||||
* client.Close();
|
||||
* ```
|
||||
*/
|
||||
export class Client {
|
||||
|
||||
|
||||
// Constructor of Client
|
||||
constructor() {}
|
||||
/**
|
||||
* Login opens an RMCP+ session as username with the given cipher suite
|
||||
* (0, 1, 2, 3 or 6). Cipher suite 0 skips authentication, so a BMC
|
||||
* that allows it accepts any password. A rejected login is an error.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const ipmi = require('nuclei/ipmi');
|
||||
* const client = new ipmi.Client();
|
||||
* client.Login('acme.com', 623, 'ADMIN', 'not-the-password', 0);
|
||||
* ```
|
||||
*/
|
||||
public Login(ctx: any, host: string, port: number, username: string, password: string, cipherSuite: number): boolean | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Command sends an IPMI request in the session, such as Get Device ID
|
||||
* (netFn 0x06, cmd 0x01) or Set Session Privilege Level (0x06, 0x3b).
|
||||
* @example
|
||||
* ```javascript
|
||||
* const ipmi = require('nuclei/ipmi');
|
||||
* const client = new ipmi.Client();
|
||||
* client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
|
||||
* const admin = client.Command(0x06, 0x3b, [0x04]).CompletionCode == 0;
|
||||
* client.Close();
|
||||
* ```
|
||||
*/
|
||||
public Command(netFn: number, cmd: number, data: Uint8Array): CommandResponse | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Close ends the session on the BMC, freeing its slot.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const ipmi = require('nuclei/ipmi');
|
||||
* const client = new ipmi.Client();
|
||||
* client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
|
||||
* client.Close();
|
||||
* ```
|
||||
*/
|
||||
public Close(): void {
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* AuthCapabilities is what a BMC advertises before any login, from Get
|
||||
* Channel Authentication Capabilities.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const ipmi = require('nuclei/ipmi');
|
||||
* const caps = ipmi.GetAuthCapabilities('acme.com', 623);
|
||||
* log(toJSON(caps));
|
||||
* ```
|
||||
*/
|
||||
export interface AuthCapabilities {
|
||||
|
||||
/**
|
||||
* IPMI20 is true when the BMC supports IPMI 2.0 (RMCP+).
|
||||
*/
|
||||
|
||||
IPMI20?: boolean,
|
||||
|
||||
/**
|
||||
* AuthTypes are the IPMI 1.5 authentication types the channel
|
||||
* accepts: none, md2, md5, straight (cleartext password), oem.
|
||||
*/
|
||||
|
||||
AuthTypes?: string[],
|
||||
|
||||
/**
|
||||
* AnonymousLogin is true when a null username with a null password
|
||||
* is accepted.
|
||||
*/
|
||||
|
||||
AnonymousLogin?: boolean,
|
||||
|
||||
/**
|
||||
* NullUsernames is true when accounts with an empty name are enabled.
|
||||
*/
|
||||
|
||||
NullUsernames?: boolean,
|
||||
|
||||
/**
|
||||
* NonNullUsernames is true when named accounts are enabled.
|
||||
*/
|
||||
|
||||
NonNullUsernames?: boolean,
|
||||
|
||||
/**
|
||||
* PerMessageAuth is false when the BMC skips authenticating
|
||||
* messages after login.
|
||||
*/
|
||||
|
||||
PerMessageAuth?: boolean,
|
||||
|
||||
/**
|
||||
* UserLevelAuth is false when user-level commands skip
|
||||
* authentication.
|
||||
*/
|
||||
|
||||
UserLevelAuth?: boolean,
|
||||
|
||||
/**
|
||||
* OEMID is the IANA enterprise number of the BMC vendor, or 0.
|
||||
*/
|
||||
|
||||
OEMID?: number,
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* CommandResponse is the reply to an IPMI command.
|
||||
*/
|
||||
export interface CommandResponse {
|
||||
|
||||
/**
|
||||
* CompletionCode is 0 on success.
|
||||
*/
|
||||
|
||||
CompletionCode?: number,
|
||||
|
||||
Data?: Uint8Array,
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* RAKPHash is the HMAC a BMC returns for a user during RMCP+ session
|
||||
* setup (CVE-2013-4786). It can be checked offline against candidate
|
||||
* passwords, which tests credentials without a login attempt.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const ipmi = require('nuclei/ipmi');
|
||||
* const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
|
||||
* log(hash.Hashcat);
|
||||
* ```
|
||||
*/
|
||||
export interface RAKPHash {
|
||||
|
||||
Username?: string,
|
||||
|
||||
/**
|
||||
* Algorithm is the HMAC the BMC used: sha1 or md5.
|
||||
*/
|
||||
|
||||
Algorithm?: string,
|
||||
|
||||
/**
|
||||
* Salt and HMAC are hex encoded; Salt is everything the HMAC covers.
|
||||
*/
|
||||
|
||||
Salt?: string,
|
||||
|
||||
HMAC?: string,
|
||||
|
||||
/**
|
||||
* Hashcat is the hash in hashcat mode 7300 format, for sha1 only.
|
||||
*/
|
||||
|
||||
Hashcat?: string,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
|
||||
/**
|
||||
* IsMQTT checks whether the target host:port is running an MQTT broker.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const mqtt = require('nuclei/mqtt');
|
||||
* const isMqtt = mqtt.IsMQTT('acme.com', 1883);
|
||||
* log(isMqtt);
|
||||
* ```
|
||||
*/
|
||||
export function IsMQTT(host: string, port: number): boolean | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* IsAnonymousAllowed checks whether an MQTT broker permits unauthenticated / anonymous login.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const mqtt = require('nuclei/mqtt');
|
||||
* const allowed = mqtt.IsAnonymousAllowed('acme.com', 1883);
|
||||
* log(allowed);
|
||||
* ```
|
||||
*/
|
||||
export function IsAnonymousAllowed(host: string, port: number): boolean | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Message represents an MQTT published message.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const mqtt = require('nuclei/mqtt');
|
||||
* const client = new mqtt.Client();
|
||||
* client.Connect('broker.hivemq.com', 1883);
|
||||
* const msgs = client.Subscribe('test/topic', 2);
|
||||
* log(msgs[0].Topic, msgs[0].Payload);
|
||||
* ```
|
||||
*/
|
||||
export interface Message {
|
||||
Topic?: string;
|
||||
Payload?: string;
|
||||
QoS?: number;
|
||||
Retain?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* ConnectOptions contains connection options for an MQTT broker.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const mqtt = require('nuclei/mqtt');
|
||||
* const opts = new mqtt.ConnectOptions();
|
||||
* opts.Username = 'admin';
|
||||
* opts.Password = 'admin';
|
||||
* opts.CleanSession = true;
|
||||
* ```
|
||||
*/
|
||||
export interface ConnectOptions {
|
||||
Username?: string;
|
||||
Password?: string;
|
||||
ClientID?: string;
|
||||
CleanSession?: boolean;
|
||||
KeepAlive?: number;
|
||||
TLS?: boolean;
|
||||
InsecureSkipVerify?: boolean;
|
||||
Timeout?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* ConnectResult is the response returned by an MQTT connection attempt.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const mqtt = require('nuclei/mqtt');
|
||||
* const client = new mqtt.Client();
|
||||
* const res = client.Connect('acme.com', 1883);
|
||||
* log(res.Success, res.ReturnCode, res.Reason);
|
||||
* ```
|
||||
*/
|
||||
export interface ConnectResult {
|
||||
Success?: boolean;
|
||||
ReturnCode?: number;
|
||||
Reason?: string;
|
||||
SessionPresent?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Client is an MQTT client for testing MQTT broker misconfigurations and authentication.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const mqtt = require('nuclei/mqtt');
|
||||
* const client = new mqtt.Client();
|
||||
* const res = client.Connect('acme.com', 1883);
|
||||
* if (res.Success) {
|
||||
* const msgs = client.Subscribe('#', 2);
|
||||
* client.Publish('nuclei/probe', 'test');
|
||||
* }
|
||||
* client.Close();
|
||||
* ```
|
||||
*/
|
||||
export class Client {
|
||||
constructor() {}
|
||||
|
||||
/**
|
||||
* Connect connects to an MQTT broker using optional credentials and settings.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const mqtt = require('nuclei/mqtt');
|
||||
* const client = new mqtt.Client();
|
||||
* const res = client.Connect('acme.com', 1883);
|
||||
* log(res.Success);
|
||||
* ```
|
||||
*/
|
||||
public Connect(host: string, port: number, options?: ConnectOptions): ConnectResult | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* ConnectAuth connects to an MQTT broker with a username and password.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const mqtt = require('nuclei/mqtt');
|
||||
* const client = new mqtt.Client();
|
||||
* const res = client.ConnectAuth('acme.com', 1883, 'admin', 'admin');
|
||||
* log(res.Success);
|
||||
* ```
|
||||
*/
|
||||
public ConnectAuth(host: string, port: number, username: string, password: string): ConnectResult | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe subscribes to a topic filter and collects received messages up to timeoutSec.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const mqtt = require('nuclei/mqtt');
|
||||
* const client = new mqtt.Client();
|
||||
* client.Connect('acme.com', 1883);
|
||||
* const msgs = client.Subscribe('#', 3);
|
||||
* for (const m of msgs) {
|
||||
* log(m.Topic, m.Payload);
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
public Subscribe(topic: string, timeoutSec?: number): Message[] | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Publish publishes a message payload to a topic.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const mqtt = require('nuclei/mqtt');
|
||||
* const client = new mqtt.Client();
|
||||
* client.Connect('acme.com', 1883);
|
||||
* const ok = client.Publish('nuclei/test', 'probe payload');
|
||||
* log(ok);
|
||||
* ```
|
||||
*/
|
||||
public Publish(topic: string, payload: string): boolean | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ping sends an MQTT PINGREQ and awaits PINGRESP.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const mqtt = require('nuclei/mqtt');
|
||||
* const client = new mqtt.Client();
|
||||
* client.Connect('acme.com', 1883);
|
||||
* const alive = client.Ping();
|
||||
* log(alive);
|
||||
* ```
|
||||
*/
|
||||
public Ping(): boolean | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close cleanly disconnects and closes the connection.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const mqtt = require('nuclei/mqtt');
|
||||
* const client = new mqtt.Client();
|
||||
* client.Connect('acme.com', 1883);
|
||||
* client.Close();
|
||||
* ```
|
||||
*/
|
||||
public Close(): error | null {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,7 @@ export class NetConn {
|
||||
constructor() {}
|
||||
/**
|
||||
* Close closes the connection.
|
||||
* Close does nothing if no connection is open.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const net = require('nuclei/net');
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
/**
|
||||
* PGClient is a client for Postgres database.
|
||||
* Internally client uses go-pg/pg driver.
|
||||
* Internally client uses lib/pq driver.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const postgres = require('nuclei/postgres');
|
||||
|
||||
@@ -106,7 +106,8 @@ export class SSHClient {
|
||||
/**
|
||||
* Close closes the SSH connection and destroys the client
|
||||
* Returns the success state and error. If error is not nil,
|
||||
* state will be false
|
||||
* state will be false. Close does nothing and returns true
|
||||
* if no connection is open.
|
||||
* @example
|
||||
* ```javascript
|
||||
* const ssh = require('nuclei/ssh');
|
||||
|
||||
+6
-2
@@ -128,9 +128,13 @@ func (p *GojaModule) Register() Module {
|
||||
return p
|
||||
}
|
||||
|
||||
// GetClassConstructor returns a constructor for any given go struct type for goja runtime
|
||||
// GetClassConstructor returns a constructor for any given go struct type for goja runtime.
|
||||
// Each object it constructs starts as a shallow copy of instance, so objects
|
||||
// do not share state.
|
||||
func GetClassConstructor[T any](instance *T) func(call goja.ConstructorCall, runtime *goja.Runtime) *goja.Object {
|
||||
return func(call goja.ConstructorCall, runtime *goja.Runtime) *goja.Object {
|
||||
return utils.LinkConstructor[*T](call, runtime, instance)
|
||||
obj := *instance
|
||||
|
||||
return utils.LinkConstructor(call, runtime, &obj)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package gojs_test
|
||||
|
||||
//nolint:depguard // No depguard rules are configured, so its default list allows only the standard library.
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/goja"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
|
||||
)
|
||||
|
||||
type counter struct{ n int }
|
||||
|
||||
func (c *counter) Inc() int {
|
||||
c.n++
|
||||
|
||||
return c.n
|
||||
}
|
||||
|
||||
func TestGetClassConstructorReturnsNewObjects(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
prototype := new(counter)
|
||||
runtime := goja.New()
|
||||
|
||||
err := runtime.Set("Counter", gojs.GetClassConstructor(prototype))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
value, err := runtime.RunString(`
|
||||
const first = new Counter();
|
||||
first.Inc();
|
||||
new Counter().Inc();
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got := value.ToInteger(); got != 1 {
|
||||
t.Errorf("new Counter().Inc() = %d, want 1", got)
|
||||
}
|
||||
|
||||
if prototype.n != 0 {
|
||||
t.Errorf("prototype.n = %d, want 0", prototype.n)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
package ipmi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type (
|
||||
// AuthCapabilities is what a BMC advertises before any login, from Get
|
||||
// Channel Authentication Capabilities.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const ipmi = require('nuclei/ipmi');
|
||||
// const caps = ipmi.GetAuthCapabilities('acme.com', 623);
|
||||
// log(toJSON(caps));
|
||||
// ```
|
||||
AuthCapabilities struct {
|
||||
// IPMI20 is true when the BMC supports IPMI 2.0 (RMCP+).
|
||||
IPMI20 bool
|
||||
// AuthTypes are the IPMI 1.5 authentication types the channel
|
||||
// accepts: none, md2, md5, straight (cleartext password), oem.
|
||||
AuthTypes []string
|
||||
// AnonymousLogin is true when a null username with a null password
|
||||
// is accepted.
|
||||
AnonymousLogin bool
|
||||
// NullUsernames is true when accounts with an empty name are enabled.
|
||||
NullUsernames bool
|
||||
// NonNullUsernames is true when named accounts are enabled.
|
||||
NonNullUsernames bool
|
||||
// PerMessageAuth is false when the BMC skips authenticating
|
||||
// messages after login.
|
||||
PerMessageAuth bool
|
||||
// UserLevelAuth is false when user-level commands skip
|
||||
// authentication.
|
||||
UserLevelAuth bool
|
||||
// OEMID is the IANA enterprise number of the BMC vendor, or 0.
|
||||
OEMID int
|
||||
}
|
||||
|
||||
// RAKPHash is the HMAC a BMC returns for a user during RMCP+ session
|
||||
// setup (CVE-2013-4786). It can be checked offline against candidate
|
||||
// passwords, which tests credentials without a login attempt.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const ipmi = require('nuclei/ipmi');
|
||||
// const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
|
||||
// log(hash.Hashcat);
|
||||
// ```
|
||||
RAKPHash struct {
|
||||
Username string
|
||||
// Algorithm is the HMAC the BMC used: sha1 or md5.
|
||||
Algorithm string
|
||||
// Salt and HMAC are hex encoded; Salt is everything the HMAC covers.
|
||||
Salt string
|
||||
HMAC string
|
||||
// Hashcat is the hash in hashcat mode 7300 format, for sha1 only.
|
||||
Hashcat string
|
||||
}
|
||||
|
||||
// Client holds one RMCP+ session, for commands that need a login.
|
||||
// Close it when done: BMCs allow only a few sessions at a time.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const ipmi = require('nuclei/ipmi');
|
||||
// const client = new ipmi.Client();
|
||||
// client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
|
||||
// const id = client.Command(0x06, 0x01, []);
|
||||
// client.Close();
|
||||
// ```
|
||||
Client struct {
|
||||
session *established
|
||||
}
|
||||
|
||||
// CommandResponse is the reply to an IPMI command.
|
||||
CommandResponse struct {
|
||||
// CompletionCode is 0 on success.
|
||||
CompletionCode int
|
||||
Data []byte
|
||||
}
|
||||
)
|
||||
|
||||
var authTypes = []struct {
|
||||
bit byte
|
||||
name string
|
||||
}{{0x01, "none"}, {0x02, "md2"}, {0x04, "md5"}, {0x10, "straight"}, {0x20, "oem"}}
|
||||
|
||||
// Ping reports whether the host answers an RMCP presence ping and advertises
|
||||
// IPMI support.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const ipmi = require('nuclei/ipmi');
|
||||
// const isIPMI = ipmi.Ping('acme.com', 623);
|
||||
// log(isIPMI);
|
||||
// ```
|
||||
func Ping(ctx context.Context, host string, port int) (bool, error) {
|
||||
s, err := dial(ctx, host, port)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer s.close()
|
||||
return s.ping()
|
||||
}
|
||||
|
||||
// GetAuthCapabilities returns the authentication capabilities the BMC
|
||||
// advertises on the current channel.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const ipmi = require('nuclei/ipmi');
|
||||
// const caps = ipmi.GetAuthCapabilities('acme.com', 623);
|
||||
// log(caps.AnonymousLogin);
|
||||
// ```
|
||||
func GetAuthCapabilities(ctx context.Context, host string, port int) (*AuthCapabilities, error) {
|
||||
s, err := dial(ctx, host, port)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer s.close()
|
||||
|
||||
// bit 7 asks for the IPMI 2.0 extended data; an IPMI 1.5 only BMC
|
||||
// rejects it, so ask again without
|
||||
data, err := s.command(cmdGetChannelAuthCaps, []byte{0x80 | currentChannel, privAdmin})
|
||||
if err != nil {
|
||||
if data, err = s.command(cmdGetChannelAuthCaps, []byte{currentChannel, privAdmin}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return parseAuthCapabilities(data)
|
||||
}
|
||||
|
||||
// parseAuthCapabilities decodes the Get Channel Authentication Capabilities
|
||||
// response data (IPMI v2.0 table 22-15).
|
||||
func parseAuthCapabilities(data []byte) (*AuthCapabilities, error) {
|
||||
if len(data) < 8 {
|
||||
return nil, fmt.Errorf("short channel authentication capabilities response")
|
||||
}
|
||||
caps := &AuthCapabilities{
|
||||
IPMI20: data[1]&0x80 != 0 && data[3]&0x02 != 0,
|
||||
AnonymousLogin: data[2]&0x01 != 0,
|
||||
NullUsernames: data[2]&0x02 != 0,
|
||||
NonNullUsernames: data[2]&0x04 != 0,
|
||||
UserLevelAuth: data[2]&0x08 == 0,
|
||||
PerMessageAuth: data[2]&0x10 == 0,
|
||||
OEMID: int(data[4]) | int(data[5])<<8 | int(data[6])<<16,
|
||||
}
|
||||
for _, authType := range authTypes {
|
||||
if data[1]&authType.bit != 0 {
|
||||
caps.AuthTypes = append(caps.AuthTypes, authType.name)
|
||||
}
|
||||
}
|
||||
return caps, nil
|
||||
}
|
||||
|
||||
// GetSystemGUID returns the BMC system GUID, which vendors derive from
|
||||
// serial numbers or MAC addresses.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const ipmi = require('nuclei/ipmi');
|
||||
// const guid = ipmi.GetSystemGUID('acme.com', 623);
|
||||
// log(guid);
|
||||
// ```
|
||||
func GetSystemGUID(ctx context.Context, host string, port int) (string, error) {
|
||||
s, err := dial(ctx, host, port)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer s.close()
|
||||
|
||||
data, err := s.command(cmdGetSystemGUID, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(data) < 16 {
|
||||
return "", fmt.Errorf("short system guid response")
|
||||
}
|
||||
return hex.EncodeToString(data[:16]), nil
|
||||
}
|
||||
|
||||
// GetCipherSuites returns the IDs of the RMCP+ cipher suites the BMC
|
||||
// offers. Cipher suite 0 authenticates nothing, so any password logs in.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const ipmi = require('nuclei/ipmi');
|
||||
// const suites = ipmi.GetCipherSuites('acme.com', 623);
|
||||
// log(suites.includes(0));
|
||||
// ```
|
||||
func GetCipherSuites(ctx context.Context, host string, port int) ([]int, error) {
|
||||
s, err := dial(ctx, host, port)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer s.close()
|
||||
|
||||
// the records come in pages of 16 bytes; a shorter page is the last
|
||||
var records []byte
|
||||
for index := byte(0); index < 64; index++ {
|
||||
data, err := s.command(cmdGetCipherSuites, []byte{currentChannel, 0x00, 0x80 | index})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(data) > 0 {
|
||||
records = append(records, data[1:]...)
|
||||
}
|
||||
if len(data) < 17 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return parseCipherSuites(records), nil
|
||||
}
|
||||
|
||||
// parseCipherSuites returns the suite IDs in Get Channel Cipher Suites
|
||||
// record data. A record starts with 0xc0 (standard) or 0xc1 (oem, followed
|
||||
// by a 3 byte IANA); algorithm bytes are tagged below 0xc0, so they never
|
||||
// look like a record start.
|
||||
func parseCipherSuites(records []byte) []int {
|
||||
suites := []int{}
|
||||
for i := 0; i < len(records); i++ {
|
||||
switch records[i] {
|
||||
case 0xc0:
|
||||
if i+1 < len(records) {
|
||||
suites = append(suites, int(records[i+1]))
|
||||
i++
|
||||
}
|
||||
case 0xc1:
|
||||
if i+1 < len(records) {
|
||||
suites = append(suites, int(records[i+1]))
|
||||
i += 4
|
||||
}
|
||||
}
|
||||
}
|
||||
return suites
|
||||
}
|
||||
|
||||
// GetRAKPHash starts an RMCP+ session as username and returns the HMAC the
|
||||
// BMC answers with. The session is never completed, so no login is
|
||||
// attempted; the BMC drops the half-open session after its timeout. It fails
|
||||
// when the BMC rejects the username.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const ipmi = require('nuclei/ipmi');
|
||||
// const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
|
||||
// log(ipmi.CheckRAKPPassword(hash, 'ADMIN'));
|
||||
// ```
|
||||
func GetRAKPHash(ctx context.Context, host string, port int, username string) (*RAKPHash, error) {
|
||||
s, err := dial(ctx, host, port)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer s.close()
|
||||
|
||||
// the session is never completed, so no login is attempted; md5 is for
|
||||
// BMCs that refuse sha1
|
||||
var h *handshake
|
||||
for _, auth := range []byte{authHMACSHA1, authHMACMD5} {
|
||||
if h, err = s.startSession(algorithms{auth: auth}, username, privAdmin); err == nil {
|
||||
break
|
||||
}
|
||||
var refused *rakpError
|
||||
if !errors.As(err, &refused) || refused.step != "the session" {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
hash := &RAKPHash{Username: username, Algorithm: "sha1", Salt: hex.EncodeToString(h.rakp2Salt()), HMAC: hex.EncodeToString(h.bmcMAC)}
|
||||
if h.auth == authHMACMD5 {
|
||||
hash.Algorithm = "md5"
|
||||
} else {
|
||||
hash.Hashcat = hash.Salt + ":" + hash.HMAC
|
||||
}
|
||||
return hash, nil
|
||||
}
|
||||
|
||||
// CheckRAKPPassword reports whether password produced hash. It runs offline,
|
||||
// so it can test any number of passwords without touching the BMC.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const ipmi = require('nuclei/ipmi');
|
||||
// const hash = ipmi.GetRAKPHash('acme.com', 623, 'ADMIN');
|
||||
// log(ipmi.CheckRAKPPassword(hash, 'ADMIN'));
|
||||
// ```
|
||||
func CheckRAKPPassword(hash *RAKPHash, password string) bool {
|
||||
if hash == nil {
|
||||
return false
|
||||
}
|
||||
salt, err := hex.DecodeString(hash.Salt)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
want, err := hex.DecodeString(hash.HMAC)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
newHash := sha1.New
|
||||
if hash.Algorithm == "md5" {
|
||||
newHash = md5.New
|
||||
}
|
||||
return hmac.Equal(hmacOf(newHash, []byte(password), salt), want)
|
||||
}
|
||||
|
||||
// FindRAKPPassword returns the first of passwords that produced hash, or an
|
||||
// empty string.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const ipmi = require('nuclei/ipmi');
|
||||
// const hash = ipmi.GetRAKPHash('acme.com', 623, 'root');
|
||||
// log(ipmi.FindRAKPPassword(hash, ['calvin', 'root', 'admin']));
|
||||
// ```
|
||||
func FindRAKPPassword(hash *RAKPHash, passwords []string) string {
|
||||
for _, password := range passwords {
|
||||
if CheckRAKPPassword(hash, password) {
|
||||
return password
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Login opens an RMCP+ session as username with the given cipher suite
|
||||
// (0, 1, 2, 3 or 6). Cipher suite 0 skips authentication, so a BMC
|
||||
// that allows it accepts any password. A rejected login is an error.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const ipmi = require('nuclei/ipmi');
|
||||
// const client = new ipmi.Client();
|
||||
// client.Login('acme.com', 623, 'ADMIN', 'not-the-password', 0);
|
||||
// ```
|
||||
func (c *Client) Login(ctx context.Context, host string, port int, username string, password string, cipherSuite int) (bool, error) {
|
||||
alg, ok := cipherSuites[cipherSuite]
|
||||
if !ok {
|
||||
return false, fmt.Errorf("unsupported cipher suite %d", cipherSuite)
|
||||
}
|
||||
c.Close()
|
||||
s, err := dial(ctx, host, port)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
h, err := s.startSession(alg, username, privAdmin)
|
||||
if err != nil {
|
||||
s.close()
|
||||
return false, err
|
||||
}
|
||||
k, err := s.finishSession(h, password)
|
||||
if err != nil {
|
||||
s.close()
|
||||
return false, err
|
||||
}
|
||||
c.session = &established{session: s, algorithms: alg, keys: k, bmcSessionID: h.bmcSessionID, consoleSessionID: h.consoleSessionID}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Command sends an IPMI request in the session, such as Get Device ID
|
||||
// (netFn 0x06, cmd 0x01) or Set Session Privilege Level (0x06, 0x3b).
|
||||
// @example
|
||||
// ```javascript
|
||||
// const ipmi = require('nuclei/ipmi');
|
||||
// const client = new ipmi.Client();
|
||||
// client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
|
||||
// const admin = client.Command(0x06, 0x3b, [0x04]).CompletionCode == 0;
|
||||
// client.Close();
|
||||
// ```
|
||||
func (c *Client) Command(netFn int, cmd int, data []byte) (*CommandResponse, error) {
|
||||
if c.session == nil {
|
||||
return nil, fmt.Errorf("not logged in")
|
||||
}
|
||||
code, reply, err := c.session.command(byte(netFn), byte(cmd), data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &CommandResponse{CompletionCode: int(code), Data: reply}, nil
|
||||
}
|
||||
|
||||
// Close ends the session on the BMC, freeing its slot.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const ipmi = require('nuclei/ipmi');
|
||||
// const client = new ipmi.Client();
|
||||
// client.Login('acme.com', 623, 'ADMIN', 'ADMIN', 3);
|
||||
// client.Close();
|
||||
// ```
|
||||
func (c *Client) Close() {
|
||||
if c.session == nil {
|
||||
return
|
||||
}
|
||||
_, _, _ = c.session.command(netFnApp, cmdCloseSession, c.session.bmcSessionID)
|
||||
c.session.close()
|
||||
c.session = nil
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package ipmi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The fixtures were captured from OpenIPMI's ipmi_sim with an enabled user
|
||||
// with an empty name and password, ADMIN/ADMIN and root/calvin.
|
||||
|
||||
func mustHex(t *testing.T, s string) []byte {
|
||||
t.Helper()
|
||||
data, err := hex.DecodeString(s)
|
||||
require.NoError(t, err)
|
||||
return data
|
||||
}
|
||||
|
||||
func TestParseAuthCapabilities(t *testing.T) {
|
||||
caps, err := parseAuthCapabilities(mustHex(t, "0197050300000000"))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, &AuthCapabilities{
|
||||
IPMI20: true,
|
||||
AuthTypes: []string{"none", "md2", "md5", "straight"},
|
||||
AnonymousLogin: true,
|
||||
NonNullUsernames: true,
|
||||
PerMessageAuth: true,
|
||||
UserLevelAuth: true,
|
||||
}, caps)
|
||||
|
||||
_, err = parseAuthCapabilities(mustHex(t, "0197"))
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestParseCipherSuites(t *testing.T) {
|
||||
records := mustHex(t, "c000004080c001014080c002014180c003014181c004014182c005014183c006024080c007024280c008024281c009024282c00a024283c00b024380c00c024381c00d024382c00e024383")
|
||||
require.Equal(t, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, parseCipherSuites(records))
|
||||
|
||||
// an oem record carries a 3 byte IANA before its algorithms
|
||||
require.Equal(t, []int{1, 7}, parseCipherSuites(mustHex(t, "c0010140c107d70200014080")))
|
||||
}
|
||||
|
||||
func TestRAKPPasswordChecksAreOffline(t *testing.T) {
|
||||
// hashcat -m 7300 cracks this hash to calvin as well
|
||||
hash := &RAKPHash{
|
||||
Username: "root",
|
||||
Salt: "988200cd0402000069b30c7a5e3deb30504c5b76f7b1840d5e36439998d9e0f59ae49b09119a10b2a123456789abcdefa123456789abcdef1404726f6f74",
|
||||
HMAC: "23c711e5c7a737c3c0beb9f4680ef31fa6aad398",
|
||||
}
|
||||
require.True(t, CheckRAKPPassword(hash, "calvin"))
|
||||
require.False(t, CheckRAKPPassword(hash, "ADMIN"))
|
||||
require.Equal(t, "calvin", FindRAKPPassword(hash, []string{"admin", "root", "calvin"}))
|
||||
require.Empty(t, FindRAKPPassword(hash, []string{"admin"}))
|
||||
require.False(t, CheckRAKPPassword(nil, "calvin"))
|
||||
require.False(t, CheckRAKPPassword(&RAKPHash{Salt: "zz", HMAC: "00"}, "calvin"))
|
||||
}
|
||||
|
||||
func TestRequestsHonourNetworkPolicy(t *testing.T) {
|
||||
execID := "ipmi-deny"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID, ExcludeTargets: []string{"203.0.113.50"}}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
|
||||
|
||||
_, err := GetRAKPHash(ctx, "203.0.113.50", 623, "ADMIN")
|
||||
require.ErrorContains(t, err, "denied")
|
||||
require.NotErrorIs(t, err, errNoResponse, "denied before sending, not timed out")
|
||||
_, err = Ping(ctx, "203.0.113.50", 623)
|
||||
require.ErrorContains(t, err, "denied")
|
||||
}
|
||||
|
||||
func TestPayloadEncryptionRoundTrips(t *testing.T) {
|
||||
key := mustHex(t, "000102030405060708090a0b0c0d0e0f")
|
||||
for size := range 40 {
|
||||
payload := make([]byte, size)
|
||||
for i := range payload {
|
||||
payload[i] = byte(i)
|
||||
}
|
||||
sealed, err := encryptPayload(key, payload)
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, len(sealed)%16, "iv and padded blocks")
|
||||
opened, err := decryptPayload(key, sealed)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, payload, opened)
|
||||
}
|
||||
_, err := decryptPayload(key, []byte{1, 2, 3})
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestSignAlignsTheIntegrityPad(t *testing.T) {
|
||||
e := &established{algorithms: cipherSuites[3], keys: &keys{integrity: make([]byte, 20)}}
|
||||
for size := range 8 {
|
||||
packet := append(rmcpHeader(rmcpClassIPMI), make([]byte, 12+size)...)
|
||||
signed := e.sign(packet)
|
||||
// everything from the auth type through the next header is a
|
||||
// multiple of 4, then 12 bytes of HMAC-SHA1-96
|
||||
require.Zero(t, (len(signed)-12-4)%4)
|
||||
require.Equal(t, byte(0x07), signed[len(signed)-13])
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginRejectsUnsupportedCipherSuites(t *testing.T) {
|
||||
_, err := (&Client{}).Login(context.Background(), "127.0.0.1", 623, "ADMIN", "ADMIN", 17)
|
||||
require.ErrorContains(t, err, "unsupported cipher suite 17")
|
||||
_, err = (&Client{}).Command(0x06, 0x01, nil)
|
||||
require.ErrorContains(t, err, "not logged in")
|
||||
}
|
||||
|
||||
// capturedHandshake is a cipher suite 3 session setup with ipmi_sim for
|
||||
// ADMIN/ADMIN, up to and including RAKP message 4.
|
||||
func capturedHandshake(t *testing.T) (*handshake, []byte) {
|
||||
return &handshake{
|
||||
algorithms: cipherSuites[3],
|
||||
consoleSessionID: mustHex(t, "244a5ac8"),
|
||||
bmcSessionID: mustHex(t, "04170000"),
|
||||
consoleRandom: mustHex(t, "de862025bb0c5ee6c30a58d383d44cb5"),
|
||||
bmcRandom: mustHex(t, "b56fef2ee0fcab6664a019a4c88df4f6"),
|
||||
bmcGUID: mustHex(t, "a123456789abcdefa123456789abcdef"),
|
||||
role: 0x14,
|
||||
username: "ADMIN",
|
||||
bmcMAC: mustHex(t, "7fac6f776553c58df508b9ff656b5eacb6c2def9"),
|
||||
}, mustHex(t, "00000000244a5ac8231219c5fc212cdd58c09424")
|
||||
}
|
||||
|
||||
func TestBMCProofsMatchOnlyTheRealPassword(t *testing.T) {
|
||||
h, rakp4 := capturedHandshake(t)
|
||||
sik := func(password string) []byte {
|
||||
return hmacOf(sha1.New, []byte(password), h.consoleRandom, h.bmcRandom, []byte{h.role, byte(len(h.username))}, []byte(h.username))
|
||||
}
|
||||
|
||||
require.Equal(t, h.bmcMAC, hmacOf(sha1.New, []byte("ADMIN"), h.rakp2Salt()), "rakp 2 code")
|
||||
require.NotEqual(t, h.bmcMAC, hmacOf(sha1.New, []byte("wrong"), h.rakp2Salt()))
|
||||
|
||||
require.True(t, rakp4Valid(sha1.New, sik("ADMIN"), h, rakp4))
|
||||
require.False(t, rakp4Valid(sha1.New, sik("wrong"), h, rakp4))
|
||||
tampered := bytes.Clone(rakp4)
|
||||
tampered[len(tampered)-1] ^= 0xff
|
||||
require.False(t, rakp4Valid(sha1.New, sik("ADMIN"), h, tampered), "a forged success must not pass")
|
||||
require.False(t, rakp4Valid(sha1.New, sik("ADMIN"), h, rakp4[:10]))
|
||||
}
|
||||
|
||||
func TestShortCommandReplyIsRejectedNotPanicked(t *testing.T) {
|
||||
execID := "ipmi-short"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
|
||||
|
||||
// a reply that ends at the completion code, without the checksum
|
||||
bmc, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = bmc.Close() })
|
||||
go func() {
|
||||
buf := make([]byte, 512)
|
||||
for {
|
||||
n, addr, err := bmc.ReadFrom(buf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
reply := bytes.Clone(buf[:21])
|
||||
reply[19] = buf[n-2] // echo the command
|
||||
reply[20] = 0x00
|
||||
_, _ = bmc.WriteTo(reply, addr)
|
||||
}
|
||||
}()
|
||||
|
||||
port := bmc.LocalAddr().(*net.UDPAddr).Port
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
_, err = GetSystemGUID(ctx, "127.0.0.1", port)
|
||||
require.ErrorIs(t, err, errNoResponse)
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
package ipmi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
libnet "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/net"
|
||||
)
|
||||
|
||||
// Wire constants from the IPMI v2.0 specification and DSP0136 (ASF).
|
||||
const (
|
||||
rmcpVersion = 0x06
|
||||
rmcpNoAck = 0xff
|
||||
rmcpClassASF = 0x06
|
||||
rmcpClassIPMI = 0x07
|
||||
|
||||
asfIANA = 0x000011be
|
||||
asfTypePing = 0x80
|
||||
asfTypePong = 0x40
|
||||
asfEntitiesIPMI = 0x80
|
||||
|
||||
authTypeNone = 0x00
|
||||
authTypeRMCPP = 0x06
|
||||
|
||||
netFnApp = 0x06
|
||||
cmdGetSystemGUID = 0x37
|
||||
cmdGetChannelAuthCaps = 0x38
|
||||
cmdGetCipherSuites = 0x54
|
||||
|
||||
bmcAddress = 0x20
|
||||
consoleAddress = 0x81
|
||||
currentChannel = 0x0e
|
||||
privAdmin = 0x04
|
||||
|
||||
payloadOpenSessionRequest = 0x10
|
||||
payloadOpenSessionResponse = 0x11
|
||||
payloadRAKP1 = 0x12
|
||||
payloadRAKP2 = 0x13
|
||||
payloadRAKP3 = 0x14
|
||||
payloadRAKP4 = 0x15
|
||||
|
||||
// rakpNameOnlyLookup asks the BMC to find the user by name alone, so the
|
||||
// role does not have to match the one the account was created with.
|
||||
rakpNameOnlyLookup = 0x10
|
||||
|
||||
attempts = 2
|
||||
)
|
||||
|
||||
var errNoResponse = errors.New("no ipmi response")
|
||||
|
||||
// session is one UDP exchange with a BMC. UDP is lossy, so each request is
|
||||
// sent up to attempts times.
|
||||
type session struct {
|
||||
conn net.Conn
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func dial(ctx context.Context, host string, port int) (*session, error) {
|
||||
conn, err := libnet.Dial(ctx, "udp", net.JoinHostPort(host, fmt.Sprint(port)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
timeout := 3 * time.Second
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
timeout = min(timeout, time.Until(deadline)/attempts)
|
||||
}
|
||||
return &session{conn: conn, timeout: timeout}, nil
|
||||
}
|
||||
|
||||
func (s *session) close() { _ = s.conn.Close() }
|
||||
|
||||
// exchange sends packet and returns the first reply accepted by match.
|
||||
func (s *session) exchange(packet []byte, match func([]byte) bool) ([]byte, error) {
|
||||
buf := make([]byte, 1024)
|
||||
for range attempts {
|
||||
if _, err := s.conn.Write(packet); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = s.conn.SetReadDeadline(time.Now().Add(s.timeout))
|
||||
for {
|
||||
n, err := s.conn.Read(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if reply := buf[:n]; match(reply) {
|
||||
return bytes.Clone(reply), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, errNoResponse
|
||||
}
|
||||
|
||||
func rmcpHeader(class byte) []byte {
|
||||
return []byte{rmcpVersion, 0x00, rmcpNoAck, class}
|
||||
}
|
||||
|
||||
// ping sends an ASF presence ping and reports whether the pong advertises
|
||||
// IPMI support.
|
||||
func (s *session) ping() (bool, error) {
|
||||
packet := append(rmcpHeader(rmcpClassASF), 0, 0, 0, 0, asfTypePing, 0x00, 0x00, 0x00)
|
||||
binary.BigEndian.PutUint32(packet[4:8], asfIANA)
|
||||
reply, err := s.exchange(packet, func(reply []byte) bool {
|
||||
return len(reply) >= 12 && reply[3] == rmcpClassASF && reply[8] == asfTypePong
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// pong data: IANA (4), OEM (4), supported entities (1), interactions (1)
|
||||
if len(reply) < 21 {
|
||||
return false, nil
|
||||
}
|
||||
return reply[20]&asfEntitiesIPMI != 0, nil
|
||||
}
|
||||
|
||||
func checksum(data []byte) byte {
|
||||
var sum byte
|
||||
for _, b := range data {
|
||||
sum += b
|
||||
}
|
||||
return -sum
|
||||
}
|
||||
|
||||
// command sends a session-less IPMI 1.5 request and returns the response
|
||||
// data after a successful completion code.
|
||||
func (s *session) command(cmd byte, data []byte) ([]byte, error) {
|
||||
header := []byte{bmcAddress, netFnApp << 2}
|
||||
body := append([]byte{consoleAddress, 0x00, cmd}, data...)
|
||||
message := append(append(header, checksum(header)), body...)
|
||||
message = append(message, checksum(body))
|
||||
|
||||
packet := append(rmcpHeader(rmcpClassIPMI), authTypeNone, 0, 0, 0, 0, 0, 0, 0, 0, byte(len(message)))
|
||||
packet = append(packet, message...)
|
||||
|
||||
reply, err := s.exchange(packet, func(reply []byte) bool {
|
||||
// rmcp (4), session header (10), then rqAddr, netFn, checksum,
|
||||
// rsAddr, seq, cmd, completion code
|
||||
// ... completion code, then at least the trailing checksum
|
||||
return len(reply) >= 22 && reply[3] == rmcpClassIPMI && reply[4] == authTypeNone && reply[19] == cmd
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if code := reply[20]; code != 0 {
|
||||
return nil, fmt.Errorf("ipmi command 0x%02x failed with completion code 0x%02x", cmd, code)
|
||||
}
|
||||
// drop the trailing checksum
|
||||
return reply[21 : len(reply)-1], nil
|
||||
}
|
||||
|
||||
// rmcpPlus sends an IPMI 2.0 session-setup payload and returns the reply
|
||||
// payload of the expected type.
|
||||
func (s *session) rmcpPlus(payloadType byte, payload []byte, replyType byte) ([]byte, error) {
|
||||
packet := append(rmcpHeader(rmcpClassIPMI), authTypeRMCPP, payloadType, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
|
||||
binary.LittleEndian.PutUint16(packet[14:16], uint16(len(payload)))
|
||||
packet = append(packet, payload...)
|
||||
|
||||
reply, err := s.exchange(packet, func(reply []byte) bool {
|
||||
return len(reply) >= 16 && reply[3] == rmcpClassIPMI && reply[4] == authTypeRMCPP && reply[5]&0x3f == replyType
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
length := int(binary.LittleEndian.Uint16(reply[14:16]))
|
||||
if len(reply) < 16+length {
|
||||
return nil, fmt.Errorf("truncated ipmi payload")
|
||||
}
|
||||
return reply[16 : 16+length], nil
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
package ipmi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/hmac"
|
||||
"crypto/md5"
|
||||
"crypto/rand"
|
||||
"crypto/sha1"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
)
|
||||
|
||||
// RMCP+ algorithm numbers (IPMI v2.0 tables 13-17 to 13-19).
|
||||
const (
|
||||
authNone = 0x00
|
||||
authHMACSHA1 = 0x01
|
||||
authHMACMD5 = 0x02
|
||||
|
||||
integrityNone = 0x00
|
||||
integrityHMACSHA196 = 0x01
|
||||
|
||||
confidentialityNone = 0x00
|
||||
confidentialityAESCBC128 = 0x01
|
||||
|
||||
payloadIPMI = 0x00
|
||||
payloadEncrypted = 0x80
|
||||
payloadSigned = 0x40
|
||||
|
||||
cmdCloseSession = 0x3c
|
||||
|
||||
// rakp status codes worth naming in errors
|
||||
rakpInvalidRole = 0x09
|
||||
rakpUnauthorizedName = 0x0d
|
||||
)
|
||||
|
||||
type algorithms struct {
|
||||
auth, integrity, confidentiality byte
|
||||
}
|
||||
|
||||
// cipherSuites lists the suites a session can be opened with. Suites with
|
||||
// SHA-256 or MD5 message integrity are left out until they can be checked
|
||||
// against a BMC that implements them correctly.
|
||||
var cipherSuites = map[int]algorithms{
|
||||
0: {authNone, integrityNone, confidentialityNone},
|
||||
1: {authHMACSHA1, integrityNone, confidentialityNone},
|
||||
2: {authHMACSHA1, integrityHMACSHA196, confidentialityNone},
|
||||
3: {authHMACSHA1, integrityHMACSHA196, confidentialityAESCBC128},
|
||||
6: {authHMACMD5, integrityNone, confidentialityNone},
|
||||
}
|
||||
|
||||
func authHash(auth byte) func() hash.Hash {
|
||||
switch auth {
|
||||
case authHMACSHA1:
|
||||
return sha1.New
|
||||
case authHMACMD5:
|
||||
return md5.New
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func hmacOf(newHash func() hash.Hash, key []byte, parts ...[]byte) []byte {
|
||||
mac := hmac.New(newHash, key)
|
||||
for _, part := range parts {
|
||||
_, _ = mac.Write(part)
|
||||
}
|
||||
return mac.Sum(nil)
|
||||
}
|
||||
|
||||
// handshake is the state of an RMCP+ session setup up to RAKP message 2.
|
||||
type handshake struct {
|
||||
algorithms
|
||||
consoleSessionID, bmcSessionID []byte
|
||||
consoleRandom, bmcRandom []byte
|
||||
bmcGUID []byte
|
||||
role byte
|
||||
username string
|
||||
// bmcMAC is the RAKP message 2 key exchange code, empty without
|
||||
// authentication.
|
||||
bmcMAC []byte
|
||||
}
|
||||
|
||||
// rakpError is a non-zero status in an RMCP+ session setup reply.
|
||||
type rakpError struct {
|
||||
step string
|
||||
status byte
|
||||
}
|
||||
|
||||
func (e *rakpError) Error() string {
|
||||
switch e.status {
|
||||
case rakpUnauthorizedName:
|
||||
return fmt.Sprintf("bmc rejected the username in %s (status 0x%02x)", e.step, e.status)
|
||||
case rakpInvalidRole:
|
||||
return fmt.Sprintf("bmc rejected the requested privilege in %s (status 0x%02x)", e.step, e.status)
|
||||
}
|
||||
return fmt.Sprintf("bmc refused %s with status 0x%02x", e.step, e.status)
|
||||
}
|
||||
|
||||
// startSession opens an RMCP+ session and runs RAKP messages 1 and 2.
|
||||
func (s *session) startSession(alg algorithms, username string, privilege byte) (*handshake, error) {
|
||||
if len(username) > 16 {
|
||||
return nil, fmt.Errorf("ipmi usernames are at most 16 bytes")
|
||||
}
|
||||
h := &handshake{
|
||||
algorithms: alg,
|
||||
consoleSessionID: make([]byte, 4),
|
||||
consoleRandom: make([]byte, 16),
|
||||
role: privilege | rakpNameOnlyLookup,
|
||||
username: username,
|
||||
}
|
||||
if _, err := rand.Read(h.consoleSessionID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := rand.Read(h.consoleRandom); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
open := []byte{0x00, privilege, 0x00, 0x00}
|
||||
open = append(open, h.consoleSessionID...)
|
||||
open = append(open,
|
||||
0x00, 0x00, 0x00, 0x08, alg.auth, 0x00, 0x00, 0x00,
|
||||
0x01, 0x00, 0x00, 0x08, alg.integrity, 0x00, 0x00, 0x00,
|
||||
0x02, 0x00, 0x00, 0x08, alg.confidentiality, 0x00, 0x00, 0x00,
|
||||
)
|
||||
opened, err := s.rmcpPlus(payloadOpenSessionRequest, open, payloadOpenSessionResponse)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(opened) < 2 {
|
||||
return nil, fmt.Errorf("short open session response")
|
||||
}
|
||||
if opened[1] != 0 {
|
||||
return nil, &rakpError{step: "the session", status: opened[1]}
|
||||
}
|
||||
if len(opened) < 12 {
|
||||
return nil, fmt.Errorf("short open session response")
|
||||
}
|
||||
h.bmcSessionID = bytes.Clone(opened[8:12])
|
||||
|
||||
rakp1 := []byte{0x00, 0x00, 0x00, 0x00}
|
||||
rakp1 = append(rakp1, h.bmcSessionID...)
|
||||
rakp1 = append(rakp1, h.consoleRandom...)
|
||||
rakp1 = append(rakp1, h.role, 0x00, 0x00, byte(len(username)))
|
||||
rakp1 = append(rakp1, username...)
|
||||
rakp2, err := s.rmcpPlus(payloadRAKP1, rakp1, payloadRAKP2)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(rakp2) < 2 {
|
||||
return nil, fmt.Errorf("short rakp message 2")
|
||||
}
|
||||
if rakp2[1] != 0 {
|
||||
return nil, &rakpError{step: "rakp message 1", status: rakp2[1]}
|
||||
}
|
||||
if len(rakp2) < 40 {
|
||||
return nil, fmt.Errorf("short rakp message 2")
|
||||
}
|
||||
h.bmcRandom = bytes.Clone(rakp2[8:24])
|
||||
h.bmcGUID = bytes.Clone(rakp2[24:40])
|
||||
h.bmcMAC = bytes.Clone(rakp2[40:])
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// rakp2Salt is everything the RAKP message 2 code covers.
|
||||
func (h *handshake) rakp2Salt() []byte {
|
||||
salt := make([]byte, 0, 58+len(h.username))
|
||||
salt = append(salt, h.consoleSessionID...)
|
||||
salt = append(salt, h.bmcSessionID...)
|
||||
salt = append(salt, h.consoleRandom...)
|
||||
salt = append(salt, h.bmcRandom...)
|
||||
salt = append(salt, h.bmcGUID...)
|
||||
salt = append(salt, h.role, byte(len(h.username)))
|
||||
return append(salt, h.username...)
|
||||
}
|
||||
|
||||
// keys are the session keys derived from the session integrity key.
|
||||
type keys struct {
|
||||
integrity []byte // K1
|
||||
aes []byte // first 16 bytes of K2
|
||||
}
|
||||
|
||||
// finishSession runs RAKP messages 3 and 4, which is where a wrong password
|
||||
// is rejected, and derives the session keys. Both directions are proven:
|
||||
// the BMC's RAKP message 2 code and RAKP message 4 integrity value must
|
||||
// match the password, so a peer that only answers with success statuses
|
||||
// cannot pass for a login.
|
||||
func (s *session) finishSession(h *handshake, password string) (*keys, error) {
|
||||
if len(password) > 20 {
|
||||
return nil, fmt.Errorf("ipmi passwords are at most 20 bytes")
|
||||
}
|
||||
newHash := authHash(h.auth)
|
||||
if newHash != nil && !hmac.Equal(h.bmcMAC, hmacOf(newHash, []byte(password), h.rakp2Salt())) {
|
||||
return nil, errBMCProof
|
||||
}
|
||||
|
||||
rakp3 := []byte{0x00, 0x00, 0x00, 0x00}
|
||||
rakp3 = append(rakp3, h.bmcSessionID...)
|
||||
if newHash != nil {
|
||||
rakp3 = append(rakp3, hmacOf(newHash, []byte(password),
|
||||
h.bmcRandom, h.consoleSessionID, []byte{h.role, byte(len(h.username))}, []byte(h.username))...)
|
||||
}
|
||||
rakp4, err := s.rmcpPlus(payloadRAKP3, rakp3, payloadRAKP4)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(rakp4) < 2 {
|
||||
return nil, fmt.Errorf("short rakp message 4")
|
||||
}
|
||||
if rakp4[1] != 0 {
|
||||
return nil, &rakpError{step: "the login", status: rakp4[1]}
|
||||
}
|
||||
if newHash == nil {
|
||||
return &keys{}, nil
|
||||
}
|
||||
|
||||
sik := hmacOf(newHash, []byte(password),
|
||||
h.consoleRandom, h.bmcRandom, []byte{h.role, byte(len(h.username))}, []byte(h.username))
|
||||
if !rakp4Valid(newHash, sik, h, rakp4) {
|
||||
return nil, errBMCProof
|
||||
}
|
||||
return &keys{
|
||||
integrity: hmacOf(newHash, sik, bytes.Repeat([]byte{0x01}, 20)),
|
||||
aes: hmacOf(newHash, sik, bytes.Repeat([]byte{0x02}, 20))[:16],
|
||||
}, nil
|
||||
}
|
||||
|
||||
// errBMCProof means the peer's authentication code does not match the
|
||||
// password: a wrong password, or something other than the BMC answering.
|
||||
var errBMCProof = errors.New("bmc authentication code does not match the password")
|
||||
|
||||
// rakp4Valid checks the RAKP message 4 integrity value, the session key's
|
||||
// HMAC over the console random, BMC session ID and BMC GUID, truncated to
|
||||
// 12 bytes for sha1 and 16 for md5.
|
||||
func rakp4Valid(newHash func() hash.Hash, sik []byte, h *handshake, rakp4 []byte) bool {
|
||||
size := 12
|
||||
if h.auth == authHMACMD5 {
|
||||
size = 16
|
||||
}
|
||||
if len(rakp4) < 8+size {
|
||||
return false
|
||||
}
|
||||
want := hmacOf(newHash, sik, h.consoleRandom, h.bmcSessionID, h.bmcGUID)[:size]
|
||||
return hmac.Equal(rakp4[8:8+size], want)
|
||||
}
|
||||
|
||||
// established is an authenticated RMCP+ session.
|
||||
type established struct {
|
||||
*session
|
||||
algorithms
|
||||
keys *keys
|
||||
bmcSessionID []byte
|
||||
consoleSessionID []byte
|
||||
sequence uint32
|
||||
requestSeq byte
|
||||
}
|
||||
|
||||
// command sends an IPMI request inside the session and returns its
|
||||
// completion code and response data.
|
||||
func (e *established) command(netFn, cmd byte, data []byte) (byte, []byte, error) {
|
||||
// the request sequence is 6 bits on the wire
|
||||
e.requestSeq = (e.requestSeq + 1) & 0x3f
|
||||
header := []byte{bmcAddress, netFn << 2}
|
||||
body := append([]byte{consoleAddress, e.requestSeq << 2, cmd}, data...)
|
||||
message := append(append(header, checksum(header)), body...)
|
||||
message = append(message, checksum(body))
|
||||
|
||||
payloadType := byte(payloadIPMI)
|
||||
if e.confidentiality == confidentialityAESCBC128 {
|
||||
payloadType |= payloadEncrypted
|
||||
var err error
|
||||
if message, err = encryptPayload(e.keys.aes, message); err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
}
|
||||
if e.integrity != integrityNone {
|
||||
payloadType |= payloadSigned
|
||||
}
|
||||
|
||||
e.sequence++
|
||||
packet := append(rmcpHeader(rmcpClassIPMI), authTypeRMCPP, payloadType)
|
||||
packet = append(packet, e.bmcSessionID...)
|
||||
packet = binary.LittleEndian.AppendUint32(packet, e.sequence)
|
||||
packet = binary.LittleEndian.AppendUint16(packet, uint16(len(message)))
|
||||
packet = append(packet, message...)
|
||||
if e.integrity != integrityNone {
|
||||
packet = e.sign(packet)
|
||||
}
|
||||
|
||||
// a reply must be for this session, pass the integrity check when the
|
||||
// session is signed, and answer this request, not an earlier one that
|
||||
// arrived late
|
||||
var payload []byte
|
||||
_, err := e.exchange(packet, func(reply []byte) bool {
|
||||
payload = e.open(reply)
|
||||
return len(payload) >= 8 && payload[4]>>2 == e.requestSeq && payload[5] == cmd
|
||||
})
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
// rqAddr, netFn, checksum, rsAddr, seq, cmd, completion code, data,
|
||||
// checksum
|
||||
return payload[6], bytes.Clone(payload[7 : len(payload)-1]), nil
|
||||
}
|
||||
|
||||
// open returns the IPMI message in a session reply, or nil when the reply
|
||||
// is not for this session or fails its integrity check.
|
||||
func (e *established) open(reply []byte) []byte {
|
||||
if len(reply) < 16 || reply[3] != rmcpClassIPMI || reply[4] != authTypeRMCPP || reply[5]&0x3f != payloadIPMI {
|
||||
return nil
|
||||
}
|
||||
if !bytes.Equal(reply[6:10], e.consoleSessionID) {
|
||||
return nil
|
||||
}
|
||||
length := int(binary.LittleEndian.Uint16(reply[14:16]))
|
||||
if len(reply) < 16+length {
|
||||
return nil
|
||||
}
|
||||
if e.integrity != integrityNone {
|
||||
if reply[5]&payloadSigned == 0 || !e.verify(reply, 16+length) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
payload := reply[16 : 16+length]
|
||||
if reply[5]&payloadEncrypted != 0 {
|
||||
var err error
|
||||
if payload, err = decryptPayload(e.keys.aes, payload); err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
// verify checks a reply's HMAC-SHA1-96 trailer: the pad after the payload,
|
||||
// pad length and next header, then the code over everything from the auth
|
||||
// type on.
|
||||
func (e *established) verify(reply []byte, payloadEnd int) bool {
|
||||
const codeSize = 12
|
||||
if len(reply) < payloadEnd+2+codeSize {
|
||||
return false
|
||||
}
|
||||
codeStart := len(reply) - codeSize
|
||||
if reply[codeStart-1] != 0x07 || int(reply[codeStart-2]) != codeStart-2-payloadEnd {
|
||||
return false
|
||||
}
|
||||
want := hmacOf(sha1.New, e.keys.integrity, reply[4:codeStart])[:codeSize]
|
||||
return hmac.Equal(reply[codeStart:], want)
|
||||
}
|
||||
|
||||
// sign appends the HMAC-SHA1-96 integrity trailer: pad to a multiple of 4 from the auth
|
||||
// type on, pad length, next header, and the truncated HMAC over it all.
|
||||
func (e *established) sign(packet []byte) []byte {
|
||||
pad := (4 - (len(packet)-4+2)%4) % 4
|
||||
packet = append(packet, bytes.Repeat([]byte{0xff}, pad)...)
|
||||
packet = append(packet, byte(pad), 0x07)
|
||||
return append(packet, hmacOf(sha1.New, e.keys.integrity, packet[4:])[:12]...)
|
||||
}
|
||||
|
||||
// encryptPayload is AES-CBC-128 with a random IV in front and the
|
||||
// confidentiality pad (1, 2, 3, ... then its length) at the end.
|
||||
func encryptPayload(key, payload []byte) ([]byte, error) {
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pad := (aes.BlockSize - (len(payload)+1)%aes.BlockSize) % aes.BlockSize
|
||||
plain := bytes.Clone(payload)
|
||||
for i := 1; i <= pad; i++ {
|
||||
plain = append(plain, byte(i))
|
||||
}
|
||||
plain = append(plain, byte(pad))
|
||||
|
||||
out := make([]byte, aes.BlockSize+len(plain))
|
||||
if _, err := rand.Read(out[:aes.BlockSize]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cipher.NewCBCEncrypter(block, out[:aes.BlockSize]).CryptBlocks(out[aes.BlockSize:], plain)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func decryptPayload(key, payload []byte) ([]byte, error) {
|
||||
if len(payload) < 2*aes.BlockSize || len(payload)%aes.BlockSize != 0 {
|
||||
return nil, fmt.Errorf("malformed encrypted payload")
|
||||
}
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plain := make([]byte, len(payload)-aes.BlockSize)
|
||||
cipher.NewCBCDecrypter(block, payload[:aes.BlockSize]).CryptBlocks(plain, payload[aes.BlockSize:])
|
||||
pad := int(plain[len(plain)-1])
|
||||
if pad+1 > len(plain) {
|
||||
return nil, fmt.Errorf("malformed encrypted payload")
|
||||
}
|
||||
return plain[:len(plain)-pad-1], nil
|
||||
}
|
||||
@@ -0,0 +1,596 @@
|
||||
package mqtt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
libnet "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/net"
|
||||
)
|
||||
|
||||
var (
|
||||
defaultTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
const (
|
||||
maxSubscribedMessages = 1000
|
||||
maxSubscribedBytes = 10 * 1024 * 1024 // 10MB
|
||||
)
|
||||
|
||||
type (
|
||||
// Message represents an MQTT published message.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const mqtt = require('nuclei/mqtt');
|
||||
// const client = new mqtt.Client();
|
||||
// client.Connect('broker.hivemq.com', 1883);
|
||||
// const msgs = client.Subscribe('test/topic', 2);
|
||||
// log(msgs[0].Topic, msgs[0].Payload);
|
||||
// ```
|
||||
Message struct {
|
||||
Topic string `json:"topic"`
|
||||
Payload string `json:"payload"`
|
||||
QoS int `json:"qos"`
|
||||
Retain bool `json:"retain"`
|
||||
}
|
||||
|
||||
// ConnectOptions contains connection options for an MQTT broker.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const mqtt = require('nuclei/mqtt');
|
||||
// const opts = new mqtt.ConnectOptions();
|
||||
// opts.Username = 'admin';
|
||||
// opts.Password = 'admin';
|
||||
// ```
|
||||
ConnectOptions struct {
|
||||
Username string `json:"username,omitempty"`
|
||||
Password string `json:"password,omitempty"`
|
||||
ClientID string `json:"client_id,omitempty"`
|
||||
CleanSession *bool `json:"clean_session,omitempty"`
|
||||
KeepAlive int `json:"keep_alive,omitempty"`
|
||||
TLS bool `json:"tls,omitempty"`
|
||||
InsecureSkipVerify *bool `json:"insecure_skip_verify,omitempty"`
|
||||
Timeout int `json:"timeout,omitempty"`
|
||||
}
|
||||
|
||||
// ConnectResult is the response returned by an MQTT connection attempt.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const mqtt = require('nuclei/mqtt');
|
||||
// const client = new mqtt.Client();
|
||||
// const res = client.Connect('acme.com', 1883);
|
||||
// log(res.Success, res.ReturnCode, res.Reason);
|
||||
// ```
|
||||
ConnectResult struct {
|
||||
Success bool `json:"success"`
|
||||
ReturnCode int `json:"return_code"`
|
||||
Reason string `json:"reason"`
|
||||
SessionPresent bool `json:"session_present"`
|
||||
}
|
||||
|
||||
// Client is an MQTT client for testing MQTT broker misconfigurations and authentication.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const mqtt = require('nuclei/mqtt');
|
||||
// const client = new mqtt.Client();
|
||||
// const res = client.Connect('acme.com', 1883);
|
||||
// if (res.Success) {
|
||||
// const msgs = client.Subscribe('#', 2);
|
||||
// client.Publish('nuclei/probe', 'test');
|
||||
// }
|
||||
// client.Close();
|
||||
// ```
|
||||
Client struct {
|
||||
conn net.Conn
|
||||
packetID uint16
|
||||
mu sync.Mutex
|
||||
timeout time.Duration
|
||||
}
|
||||
)
|
||||
|
||||
// dial opens a network connection to host:port respecting protocolstate policy and context.
|
||||
func dial(ctx context.Context, host string, port int, useTLS bool, insecureSkipVerify bool, timeout time.Duration) (net.Conn, error) {
|
||||
if host == "" || port <= 0 {
|
||||
return nil, fmt.Errorf("invalid host or port")
|
||||
}
|
||||
|
||||
addr := net.JoinHostPort(host, strconv.Itoa(port))
|
||||
conn, err := libnet.Dial(ctx, "tcp", addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_ = conn.SetDeadline(time.Now().Add(timeout))
|
||||
|
||||
if useTLS {
|
||||
tlsConn := tls.Client(conn, &tls.Config{
|
||||
InsecureSkipVerify: insecureSkipVerify,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
ServerName: host,
|
||||
})
|
||||
if err := tlsConn.HandshakeContext(ctx); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("TLS handshake failed: %w", err)
|
||||
}
|
||||
conn = tlsConn
|
||||
}
|
||||
|
||||
_ = conn.SetDeadline(time.Now().Add(timeout))
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
// IsMQTT checks whether the target host:port is running an MQTT broker.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const mqtt = require('nuclei/mqtt');
|
||||
// const isMqtt = mqtt.IsMQTT('acme.com', 1883);
|
||||
// log(isMqtt);
|
||||
// ```
|
||||
func IsMQTT(ctx context.Context, host string, port int) (bool, error) {
|
||||
conn, err := dial(ctx, host, port, false, true, defaultTimeout)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
pkt, err := buildConnectPacket("nuclei-probe", "", "", true, 60)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
_ = conn.SetWriteDeadline(time.Now().Add(defaultTimeout))
|
||||
if _, err := conn.Write(pkt); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
header, payload, err := readPacket(conn)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Any valid CONNACK indicates an active MQTT broker
|
||||
if (header >> 4) == PacketConnack {
|
||||
_, _, err := parseConnack(payload)
|
||||
return err == nil, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// IsAnonymousAllowed checks whether an MQTT broker permits unauthenticated / anonymous login.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const mqtt = require('nuclei/mqtt');
|
||||
// const allowed = mqtt.IsAnonymousAllowed('acme.com', 1883);
|
||||
// log(allowed);
|
||||
// ```
|
||||
func IsAnonymousAllowed(ctx context.Context, host string, port int) (bool, error) {
|
||||
conn, err := dial(ctx, host, port, false, true, defaultTimeout)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
pkt, err := buildConnectPacket("", "", "", true, 60)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
_ = conn.SetWriteDeadline(time.Now().Add(defaultTimeout))
|
||||
if _, err := conn.Write(pkt); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
header, payload, err := readPacket(conn)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if (header >> 4) != PacketConnack {
|
||||
return false, fmt.Errorf("expected CONNACK, got packet type %d", header>>4)
|
||||
}
|
||||
|
||||
_, returnCode, err := parseConnack(payload)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return returnCode == ConnAccepted, nil
|
||||
}
|
||||
|
||||
// Connect connects to an MQTT broker using optional credentials and settings.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const mqtt = require('nuclei/mqtt');
|
||||
// const client = new mqtt.Client();
|
||||
// const res = client.Connect('acme.com', 1883);
|
||||
// log(res.Success);
|
||||
// ```
|
||||
func (c *Client) Connect(ctx context.Context, host string, port int, options *ConnectOptions) (*ConnectResult, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if c.conn != nil {
|
||||
_ = c.conn.Close()
|
||||
c.conn = nil
|
||||
}
|
||||
|
||||
timeout := defaultTimeout
|
||||
cleanSession := true
|
||||
keepAlive := uint16(60)
|
||||
clientID := "nuclei"
|
||||
var username, password string
|
||||
useTLS := false
|
||||
insecureSkipVerify := true
|
||||
|
||||
if options != nil {
|
||||
if options.Timeout > 0 {
|
||||
timeout = time.Duration(options.Timeout) * time.Second
|
||||
}
|
||||
if options.KeepAlive > 0 {
|
||||
if options.KeepAlive > 65535 {
|
||||
keepAlive = 65535
|
||||
} else {
|
||||
keepAlive = uint16(options.KeepAlive)
|
||||
}
|
||||
}
|
||||
if options.ClientID != "" {
|
||||
clientID = options.ClientID
|
||||
}
|
||||
if options.CleanSession != nil {
|
||||
cleanSession = *options.CleanSession
|
||||
}
|
||||
if options.InsecureSkipVerify != nil {
|
||||
insecureSkipVerify = *options.InsecureSkipVerify
|
||||
}
|
||||
username = options.Username
|
||||
password = options.Password
|
||||
useTLS = options.TLS
|
||||
}
|
||||
|
||||
c.timeout = timeout
|
||||
conn, err := dial(ctx, host, port, useTLS, insecureSkipVerify, timeout)
|
||||
if err != nil {
|
||||
return &ConnectResult{
|
||||
Success: false,
|
||||
ReturnCode: -1,
|
||||
Reason: err.Error(),
|
||||
}, err
|
||||
}
|
||||
|
||||
pkt, err := buildConnectPacket(clientID, username, password, cleanSession, keepAlive)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return &ConnectResult{
|
||||
Success: false,
|
||||
ReturnCode: -1,
|
||||
Reason: err.Error(),
|
||||
}, err
|
||||
}
|
||||
|
||||
_ = conn.SetWriteDeadline(time.Now().Add(timeout))
|
||||
if _, err := conn.Write(pkt); err != nil {
|
||||
_ = conn.Close()
|
||||
return &ConnectResult{
|
||||
Success: false,
|
||||
ReturnCode: -1,
|
||||
Reason: fmt.Sprintf("failed to send CONNECT: %v", err),
|
||||
}, err
|
||||
}
|
||||
|
||||
header, payload, err := readPacket(conn)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return &ConnectResult{
|
||||
Success: false,
|
||||
ReturnCode: -1,
|
||||
Reason: fmt.Sprintf("failed to read CONNACK: %v", err),
|
||||
}, err
|
||||
}
|
||||
|
||||
if (header >> 4) != PacketConnack {
|
||||
_ = conn.Close()
|
||||
return &ConnectResult{
|
||||
Success: false,
|
||||
ReturnCode: -1,
|
||||
Reason: fmt.Sprintf("unexpected packet %d (expected CONNACK)", header>>4),
|
||||
}, fmt.Errorf("unexpected packet %d", header>>4)
|
||||
}
|
||||
|
||||
sessPresent, returnCode, err := parseConnack(payload)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return &ConnectResult{
|
||||
Success: false,
|
||||
ReturnCode: int(returnCode),
|
||||
Reason: err.Error(),
|
||||
}, err
|
||||
}
|
||||
|
||||
res := &ConnectResult{
|
||||
Success: returnCode == ConnAccepted,
|
||||
ReturnCode: int(returnCode),
|
||||
Reason: ConnackReason(returnCode),
|
||||
SessionPresent: sessPresent,
|
||||
}
|
||||
|
||||
if res.Success {
|
||||
c.conn = conn
|
||||
} else {
|
||||
_ = conn.Close()
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// ConnectAuth connects to an MQTT broker with a username and password.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const mqtt = require('nuclei/mqtt');
|
||||
// const client = new mqtt.Client();
|
||||
// const res = client.ConnectAuth('acme.com', 1883, 'admin', 'admin');
|
||||
// log(res.Success);
|
||||
// ```
|
||||
func (c *Client) ConnectAuth(ctx context.Context, host string, port int, username, password string) (*ConnectResult, error) {
|
||||
clean := true
|
||||
return c.Connect(ctx, host, port, &ConnectOptions{
|
||||
Username: username,
|
||||
Password: password,
|
||||
CleanSession: &clean,
|
||||
})
|
||||
}
|
||||
|
||||
// Subscribe subscribes to a topic filter and collects received messages up to timeoutSec.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const mqtt = require('nuclei/mqtt');
|
||||
// const client = new mqtt.Client();
|
||||
// client.Connect('acme.com', 1883);
|
||||
// const msgs = client.Subscribe('#', 3);
|
||||
//
|
||||
// for (const m of msgs) {
|
||||
// log(m.Topic, m.Payload);
|
||||
// }
|
||||
//
|
||||
// ```
|
||||
func (c *Client) Subscribe(ctx context.Context, topic string, timeoutSec int) ([]*Message, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if c.conn == nil {
|
||||
return nil, fmt.Errorf("not connected to MQTT broker")
|
||||
}
|
||||
|
||||
c.packetID++
|
||||
if c.packetID == 0 {
|
||||
c.packetID = 1
|
||||
}
|
||||
pktID := c.packetID
|
||||
|
||||
subPkt, err := buildSubscribePacket(pktID, topic, 0)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build SUBSCRIBE: %w", err)
|
||||
}
|
||||
|
||||
_ = c.conn.SetWriteDeadline(time.Now().Add(c.timeout))
|
||||
if _, err := c.conn.Write(subPkt); err != nil {
|
||||
_ = c.conn.Close()
|
||||
c.conn = nil
|
||||
return nil, fmt.Errorf("failed to send SUBSCRIBE: %w", err)
|
||||
}
|
||||
|
||||
waitDuration := 2 * time.Second
|
||||
if timeoutSec > 0 {
|
||||
if timeoutSec > 60 {
|
||||
timeoutSec = 60
|
||||
}
|
||||
waitDuration = time.Duration(timeoutSec) * time.Second
|
||||
}
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
if remaining := time.Until(deadline); remaining < waitDuration {
|
||||
waitDuration = remaining
|
||||
}
|
||||
}
|
||||
|
||||
readDeadline := time.Now().Add(waitDuration)
|
||||
_ = c.conn.SetDeadline(readDeadline)
|
||||
defer func() {
|
||||
if c.conn != nil {
|
||||
_ = c.conn.SetDeadline(time.Time{})
|
||||
}
|
||||
}()
|
||||
|
||||
var messages []*Message
|
||||
var totalBytes int
|
||||
subackReceived := false
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = c.conn.Close()
|
||||
c.conn = nil
|
||||
return messages, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
header, payload, err := readPacket(c.conn)
|
||||
if err != nil {
|
||||
if errors.Is(err, errBrokenStream) {
|
||||
// The window closed in the middle of a packet. The stream cannot be
|
||||
// read in sync again, so the connection is dropped.
|
||||
_ = c.conn.Close()
|
||||
c.conn = nil
|
||||
if !subackReceived {
|
||||
return nil, fmt.Errorf("failed waiting for SUBACK: %w", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) && netErr.Timeout() {
|
||||
// Window finished
|
||||
break
|
||||
}
|
||||
if !subackReceived {
|
||||
_ = c.conn.Close()
|
||||
c.conn = nil
|
||||
return nil, fmt.Errorf("failed waiting for SUBACK: %w", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
packetType := header >> 4
|
||||
switch packetType {
|
||||
case PacketSuback:
|
||||
subackPID, grantedQoS, err := parseSuback(payload)
|
||||
if err != nil {
|
||||
_ = c.conn.Close()
|
||||
c.conn = nil
|
||||
return nil, err
|
||||
}
|
||||
if subackPID != pktID {
|
||||
continue // Ignore stale/mismatched SUBACK from an earlier subscription
|
||||
}
|
||||
for _, qos := range grantedQoS {
|
||||
if qos == 0x80 {
|
||||
return nil, fmt.Errorf("subscription rejected by broker for topic %q", topic)
|
||||
}
|
||||
}
|
||||
subackReceived = true
|
||||
|
||||
case PacketPublish:
|
||||
msg, err := parsePublish(header, payload)
|
||||
if err == nil {
|
||||
messages = append(messages, msg)
|
||||
totalBytes += len(msg.Topic) + len(msg.Payload)
|
||||
if len(messages) >= maxSubscribedMessages || totalBytes >= maxSubscribedBytes {
|
||||
return messages, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if time.Now().After(readDeadline) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !subackReceived {
|
||||
_ = c.conn.Close()
|
||||
c.conn = nil
|
||||
return nil, fmt.Errorf("subscription timed out waiting for SUBACK")
|
||||
}
|
||||
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
// Publish publishes a message payload to a topic.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const mqtt = require('nuclei/mqtt');
|
||||
// const client = new mqtt.Client();
|
||||
// client.Connect('acme.com', 1883);
|
||||
// const ok = client.Publish('nuclei/test', 'probe payload');
|
||||
// log(ok);
|
||||
// ```
|
||||
func (c *Client) Publish(ctx context.Context, topic string, payload string) (bool, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if c.conn == nil {
|
||||
return false, fmt.Errorf("not connected to MQTT broker")
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
pkt, err := buildPublishPacket(topic, []byte(payload), 0, false, 0)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to build PUBLISH: %w", err)
|
||||
}
|
||||
|
||||
_ = c.conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
|
||||
if _, err := c.conn.Write(pkt); err != nil {
|
||||
_ = c.conn.Close()
|
||||
c.conn = nil
|
||||
return false, fmt.Errorf("failed to send PUBLISH: %w", err)
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Ping sends an MQTT PINGREQ and awaits PINGRESP.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const mqtt = require('nuclei/mqtt');
|
||||
// const client = new mqtt.Client();
|
||||
// client.Connect('acme.com', 1883);
|
||||
// const alive = client.Ping();
|
||||
// log(alive);
|
||||
// ```
|
||||
func (c *Client) Ping(ctx context.Context) (bool, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if c.conn == nil {
|
||||
return false, fmt.Errorf("not connected to MQTT broker")
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
_ = c.conn.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
_ = c.conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
|
||||
if _, err := c.conn.Write(buildPingreqPacket()); err != nil {
|
||||
_ = c.conn.Close()
|
||||
c.conn = nil
|
||||
return false, fmt.Errorf("failed to send PINGREQ: %w", err)
|
||||
}
|
||||
|
||||
for {
|
||||
header, _, err := readPacket(c.conn)
|
||||
if err != nil {
|
||||
_ = c.conn.Close()
|
||||
c.conn = nil
|
||||
return false, fmt.Errorf("failed to read PINGRESP: %w", err)
|
||||
}
|
||||
|
||||
packetType := header >> 4
|
||||
if packetType == PacketPingresp {
|
||||
return true, nil
|
||||
}
|
||||
if packetType == PacketPublish {
|
||||
// Broker delivered an incoming publish message while ping was in flight; continue waiting for PINGRESP
|
||||
continue
|
||||
}
|
||||
return false, fmt.Errorf("unexpected ping response packet: %d", packetType)
|
||||
}
|
||||
}
|
||||
|
||||
// Close cleanly disconnects and closes the connection.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const mqtt = require('nuclei/mqtt');
|
||||
// const client = new mqtt.Client();
|
||||
// client.Connect('acme.com', 1883);
|
||||
// client.Close();
|
||||
// ```
|
||||
func (c *Client) Close() error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if c.conn != nil {
|
||||
_ = c.conn.SetWriteDeadline(time.Now().Add(3 * time.Second))
|
||||
_, _ = c.conn.Write(buildDisconnectPacket())
|
||||
err := c.conn.Close()
|
||||
c.conn = nil
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,395 @@
|
||||
package mqtt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestVarLengthEncoding(t *testing.T) {
|
||||
testCases := []int{0, 1, 64, 127, 128, 16383, 16384, 2097151}
|
||||
for _, tc := range testCases {
|
||||
encoded, err := encodeVarLength(tc)
|
||||
require.NoError(t, err)
|
||||
decoded, err := decodeVarLength(bytes.NewReader(encoded))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc, decoded)
|
||||
}
|
||||
|
||||
// Test negative or overflow
|
||||
_, err := encodeVarLength(-1)
|
||||
require.Error(t, err)
|
||||
_, err = encodeVarLength(MaxRemainingLength + 1)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestOversizedStringsAndPackets(t *testing.T) {
|
||||
hugeString := strings.Repeat("A", MaxStringLength+1)
|
||||
_, err := encodeString(hugeString)
|
||||
require.Error(t, err)
|
||||
|
||||
_, err = buildConnectPacket("id", hugeString, "pass", true, 60)
|
||||
require.Error(t, err)
|
||||
|
||||
_, err = buildSubscribePacket(1, hugeString, 0)
|
||||
require.Error(t, err)
|
||||
|
||||
_, err = buildPublishPacket(hugeString, []byte("ok"), 0, false, 0)
|
||||
require.Error(t, err)
|
||||
|
||||
// Oversized packet reading
|
||||
oversizedLen, err := encodeVarLength(MaxPacketSize + 100)
|
||||
require.NoError(t, err)
|
||||
buf := append([]byte{PacketPublish << 4}, oversizedLen...)
|
||||
_, _, err = readPacket(bytes.NewReader(buf))
|
||||
require.ErrorContains(t, err, "exceeds maximum allowed")
|
||||
}
|
||||
|
||||
func TestPacketConnect(t *testing.T) {
|
||||
pkt, err := buildConnectPacket("test-client", "user1", "pass1", true, 60)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, pkt)
|
||||
require.Equal(t, byte(PacketConnect<<4), pkt[0])
|
||||
|
||||
r := bytes.NewReader(pkt)
|
||||
header, payload, err := readPacket(r)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, byte(PacketConnect<<4), header)
|
||||
|
||||
// Verify protocol name "MQTT"
|
||||
protoLen := binary.BigEndian.Uint16(payload[0:2])
|
||||
require.Equal(t, uint16(4), protoLen)
|
||||
require.Equal(t, "MQTT", string(payload[2:6]))
|
||||
require.Equal(t, byte(0x04), payload[6]) // Protocol level 4 (MQTT 3.1.1)
|
||||
|
||||
// CleanSession (0x02) | User (0x80) | Pass (0x40) = 0xC2
|
||||
require.Equal(t, byte(0xC2), payload[7])
|
||||
}
|
||||
|
||||
func TestPacketPublishAndParse(t *testing.T) {
|
||||
pkt, err := buildPublishPacket("sensors/temperature", []byte("23.5C"), 0, true, 0)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, pkt)
|
||||
|
||||
r := bytes.NewReader(pkt)
|
||||
header, payload, err := readPacket(r)
|
||||
require.NoError(t, err)
|
||||
|
||||
msg, err := parsePublish(header, payload)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "sensors/temperature", msg.Topic)
|
||||
require.Equal(t, "23.5C", msg.Payload)
|
||||
require.Equal(t, 0, msg.QoS)
|
||||
require.True(t, msg.Retain)
|
||||
}
|
||||
|
||||
func TestPacketSubscribe(t *testing.T) {
|
||||
pkt, err := buildSubscribePacket(42, "devices/#", 0)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, pkt)
|
||||
require.Equal(t, byte((PacketSubscribe<<4)|0x02), pkt[0])
|
||||
|
||||
r := bytes.NewReader(pkt)
|
||||
header, payload, err := readPacket(r)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, byte((PacketSubscribe<<4)|0x02), header)
|
||||
|
||||
pid := binary.BigEndian.Uint16(payload[0:2])
|
||||
require.Equal(t, uint16(42), pid)
|
||||
|
||||
subackPayload := []byte{0x00, 0x2A, 0x00} // PID 42, Success QoS 0
|
||||
subackPid, granted, err := parseSuback(subackPayload)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint16(42), subackPid)
|
||||
require.Equal(t, []byte{0x00}, granted)
|
||||
}
|
||||
|
||||
func TestConnackReasons(t *testing.T) {
|
||||
require.Equal(t, "connection accepted", ConnackReason(ConnAccepted))
|
||||
require.Equal(t, "connection refused: bad user name or password", ConnackReason(ConnRefusedBadAuth))
|
||||
require.Equal(t, "connection refused: not authorized", ConnackReason(ConnRefusedNotAuthorized))
|
||||
require.Contains(t, ConnackReason(99), "connection refused: return code 99")
|
||||
}
|
||||
|
||||
func TestRequestsHonourNetworkPolicy(t *testing.T) {
|
||||
execID := "mqtt-deny"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID, ExcludeTargets: []string{"203.0.113.50"}}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
|
||||
|
||||
_, err := IsMQTT(ctx, "203.0.113.50", 1883)
|
||||
require.Error(t, err)
|
||||
|
||||
client := &Client{}
|
||||
_, err = client.Connect(ctx, "203.0.113.50", 1883, nil)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func startMockMQTTBroker(t *testing.T) (net.Listener, int) {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
port := l.Addr().(*net.TCPAddr).Port
|
||||
|
||||
go func() {
|
||||
for {
|
||||
conn, err := l.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go handleMockClient(conn)
|
||||
}
|
||||
}()
|
||||
|
||||
return l, port
|
||||
}
|
||||
|
||||
func handleMockClient(conn net.Conn) {
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
for {
|
||||
header, payload, err := readPacket(conn)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
packetType := header >> 4
|
||||
switch packetType {
|
||||
case PacketConnect:
|
||||
// Check if username is present (flags byte at offset 7)
|
||||
if len(payload) >= 8 {
|
||||
flags := payload[7]
|
||||
if (flags & 0x80) != 0 {
|
||||
// Check username
|
||||
offset := 10 // after proto(6) + level(1) + flags(1) + keepalive(2)
|
||||
if len(payload) >= offset+2 {
|
||||
clientLen := int(binary.BigEndian.Uint16(payload[offset : offset+2]))
|
||||
offset += 2 + clientLen
|
||||
if len(payload) >= offset+2 {
|
||||
userLen := int(binary.BigEndian.Uint16(payload[offset : offset+2]))
|
||||
user := string(payload[offset+2 : offset+2+userLen])
|
||||
if user == "baduser" {
|
||||
// Send CONNACK Refused Bad Auth
|
||||
_, _ = conn.Write([]byte{0x20, 0x02, 0x00, ConnRefusedBadAuth})
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Send CONNACK Accepted
|
||||
_, _ = conn.Write([]byte{0x20, 0x02, 0x00, ConnAccepted})
|
||||
|
||||
case PacketSubscribe:
|
||||
// Send SUBACK
|
||||
pid := binary.BigEndian.Uint16(payload[0:2])
|
||||
suback := []byte{0x90, 0x03, byte(pid >> 8), byte(pid & 0xFF), 0x00}
|
||||
_, _ = conn.Write(suback)
|
||||
|
||||
// Publish test message
|
||||
pubPkt, err := buildPublishPacket("nuclei/alert", []byte("vuln-detected"), 0, false, 0)
|
||||
if err == nil {
|
||||
_, _ = conn.Write(pubPkt)
|
||||
}
|
||||
|
||||
case PacketPublish:
|
||||
// QoS 0, no ack needed
|
||||
|
||||
case PacketPingreq:
|
||||
// Send PINGRESP
|
||||
_, _ = conn.Write([]byte{PacketPingresp << 4, 0x00})
|
||||
|
||||
case PacketDisconnect:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockBrokerInteractions(t *testing.T) {
|
||||
execID := "mqtt-mock-test"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
|
||||
|
||||
l, port := startMockMQTTBroker(t)
|
||||
defer func() { _ = l.Close() }()
|
||||
|
||||
// 1. Test IsMQTT
|
||||
isMQTT, err := IsMQTT(ctx, "127.0.0.1", port)
|
||||
require.NoError(t, err)
|
||||
require.True(t, isMQTT)
|
||||
|
||||
// 2. Test IsAnonymousAllowed
|
||||
anonAllowed, err := IsAnonymousAllowed(ctx, "127.0.0.1", port)
|
||||
require.NoError(t, err)
|
||||
require.True(t, anonAllowed)
|
||||
|
||||
// 3. Test Client Connect Auth Failure
|
||||
badClient := &Client{}
|
||||
badRes, _ := badClient.ConnectAuth(ctx, "127.0.0.1", port, "baduser", "wrongpass")
|
||||
require.False(t, badRes.Success)
|
||||
require.Equal(t, int(ConnRefusedBadAuth), badRes.ReturnCode)
|
||||
|
||||
// 4. Test Client Connect Success with default CleanSession preservation
|
||||
client := &Client{}
|
||||
res, err := client.Connect(ctx, "127.0.0.1", port, &ConnectOptions{KeepAlive: 70000})
|
||||
require.NoError(t, err)
|
||||
require.True(t, res.Success)
|
||||
require.Equal(t, int(ConnAccepted), res.ReturnCode)
|
||||
|
||||
// 5. Test Ping
|
||||
alive, err := client.Ping(ctx)
|
||||
require.NoError(t, err)
|
||||
require.True(t, alive)
|
||||
|
||||
// 6. Test Subscribe and Receive Message
|
||||
msgs, err := client.Subscribe(ctx, "nuclei/alert", 1)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, msgs)
|
||||
require.Equal(t, "nuclei/alert", msgs[0].Topic)
|
||||
require.Equal(t, "vuln-detected", msgs[0].Payload)
|
||||
|
||||
// 7. Test Publish
|
||||
pubOk, err := client.Publish(ctx, "nuclei/probe", "test payload")
|
||||
require.NoError(t, err)
|
||||
require.True(t, pubOk)
|
||||
|
||||
// 8. Test Close and Double Close
|
||||
require.NoError(t, client.Close())
|
||||
require.NoError(t, client.Close())
|
||||
}
|
||||
|
||||
func TestPasswordWithoutUsername(t *testing.T) {
|
||||
// MQTT 3.1.1 (MQTT-3.1.2-22): If Password flag is set, User Name flag MUST be set.
|
||||
pkt, err := buildConnectPacket("token-client", "", "my-secret-token", true, 60)
|
||||
require.NoError(t, err)
|
||||
|
||||
r := bytes.NewReader(pkt)
|
||||
_, payload, err := readPacket(r)
|
||||
require.NoError(t, err)
|
||||
|
||||
// CleanSession (0x02) | User (0x80) | Pass (0x40) = 0xC2
|
||||
require.Equal(t, byte(0xC2), payload[7])
|
||||
|
||||
// The client ID comes right after the keep alive at payload[8:10], and the
|
||||
// username length follows it.
|
||||
clientIDLen := int(binary.BigEndian.Uint16(payload[10:12]))
|
||||
off := 12 + clientIDLen
|
||||
|
||||
// Username length should be 0 (empty string)
|
||||
userLen := binary.BigEndian.Uint16(payload[off : off+2])
|
||||
require.Equal(t, uint16(0), userLen)
|
||||
off += 2
|
||||
|
||||
// Password follows the username
|
||||
passLen := binary.BigEndian.Uint16(payload[off : off+2])
|
||||
require.Equal(t, uint16(len("my-secret-token")), passLen)
|
||||
off += 2
|
||||
require.Equal(t, "my-secret-token", string(payload[off:off+int(passLen)]))
|
||||
}
|
||||
|
||||
// A window that closes in the middle of a PUBLISH must drop the connection.
|
||||
// Keeping it would let the next read start inside that packet.
|
||||
func TestSubscribePartialPacketClosesConnection(t *testing.T) {
|
||||
execID := "mqtt-partial-subscribe"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
|
||||
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = l.Close() }()
|
||||
port := l.Addr().(*net.TCPAddr).Port
|
||||
|
||||
go func() {
|
||||
conn, err := l.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
for {
|
||||
header, payload, err := readPacket(conn)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
switch header >> 4 {
|
||||
case PacketConnect:
|
||||
_, _ = conn.Write([]byte{PacketConnack << 4, 0x02, 0x00, ConnAccepted})
|
||||
case PacketSubscribe:
|
||||
// SUBACK for the packet id, then a PUBLISH that promises 10 bytes
|
||||
// and sends 5. The client's window closes in the middle of it.
|
||||
_, _ = conn.Write([]byte{PacketSuback << 4, 0x03, payload[0], payload[1], 0x00})
|
||||
_, _ = conn.Write([]byte{PacketPublish << 4, 0x0a, 0x00, 0x03, 'a', '/', 'b'})
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
client := &Client{}
|
||||
res, err := client.Connect(ctx, "127.0.0.1", port, nil)
|
||||
require.NoError(t, err)
|
||||
require.True(t, res.Success)
|
||||
|
||||
msgs, err := client.Subscribe(ctx, "a/b", 1)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, msgs)
|
||||
require.Nil(t, client.conn, "a stream cut in the middle of a packet must not be reused")
|
||||
}
|
||||
|
||||
func TestPingWithInFlightPublish(t *testing.T) {
|
||||
execID := "mqtt-ping-inflight"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
|
||||
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = l.Close() }()
|
||||
|
||||
port := l.Addr().(*net.TCPAddr).Port
|
||||
|
||||
go func() {
|
||||
conn, err := l.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
for {
|
||||
header, _, err := readPacket(conn)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
packetType := header >> 4
|
||||
switch packetType {
|
||||
case PacketConnect:
|
||||
_, _ = conn.Write([]byte{0x20, 0x02, 0x00, ConnAccepted})
|
||||
case PacketPingreq:
|
||||
// Send a PUBLISH packet first before PINGRESP
|
||||
pubPkt, _ := buildPublishPacket("sensors/temp", []byte("25.0"), 0, false, 0)
|
||||
_, _ = conn.Write(pubPkt)
|
||||
// Then send PINGRESP
|
||||
_, _ = conn.Write([]byte{PacketPingresp << 4, 0x00})
|
||||
case PacketDisconnect:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
client := &Client{}
|
||||
res, err := client.Connect(ctx, "127.0.0.1", port, nil)
|
||||
require.NoError(t, err)
|
||||
require.True(t, res.Success)
|
||||
|
||||
// Ping should skip the in-flight PUBLISH and successfully receive PINGRESP
|
||||
alive, err := client.Ping(ctx)
|
||||
require.NoError(t, err)
|
||||
require.True(t, alive)
|
||||
|
||||
_ = client.Close()
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
package mqtt
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Limits to prevent DoS and buffer overflows
|
||||
const (
|
||||
MaxStringLength = 65535
|
||||
MaxRemainingLength = 268435455
|
||||
MaxPacketSize = 10 * 1024 * 1024 // 10MB limit on incoming packets
|
||||
)
|
||||
|
||||
// MQTT 3.1.1 Control Packet Types
|
||||
const (
|
||||
PacketConnect byte = 1
|
||||
PacketConnack byte = 2
|
||||
PacketPublish byte = 3
|
||||
PacketPuback byte = 4
|
||||
PacketPubrec byte = 5
|
||||
PacketPubrel byte = 6
|
||||
PacketPubcomp byte = 7
|
||||
PacketSubscribe byte = 8
|
||||
PacketSuback byte = 9
|
||||
PacketUnsub byte = 10
|
||||
PacketUnsuback byte = 11
|
||||
PacketPingreq byte = 12
|
||||
PacketPingresp byte = 13
|
||||
PacketDisconnect byte = 14
|
||||
)
|
||||
|
||||
// CONNACK Return Codes (MQTT 3.1.1 table 3.1)
|
||||
const (
|
||||
ConnAccepted byte = 0
|
||||
ConnRefusedUnacceptableProtocol byte = 1
|
||||
ConnRefusedIdentifierRejected byte = 2
|
||||
ConnRefusedServerUnavailable byte = 3
|
||||
ConnRefusedBadAuth byte = 4
|
||||
ConnRefusedNotAuthorized byte = 5
|
||||
)
|
||||
|
||||
// ConnackReason returns a human-readable description for CONNACK return codes.
|
||||
func ConnackReason(code byte) string {
|
||||
switch code {
|
||||
case ConnAccepted:
|
||||
return "connection accepted"
|
||||
case ConnRefusedUnacceptableProtocol:
|
||||
return "connection refused: unacceptable protocol version"
|
||||
case ConnRefusedIdentifierRejected:
|
||||
return "connection refused: identifier rejected"
|
||||
case ConnRefusedServerUnavailable:
|
||||
return "connection refused: server unavailable"
|
||||
case ConnRefusedBadAuth:
|
||||
return "connection refused: bad user name or password"
|
||||
case ConnRefusedNotAuthorized:
|
||||
return "connection refused: not authorized"
|
||||
default:
|
||||
return fmt.Sprintf("connection refused: return code %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// encodeVarLength encodes a length integer as an MQTT variable byte integer.
|
||||
func encodeVarLength(length int) ([]byte, error) {
|
||||
if length < 0 || length > MaxRemainingLength {
|
||||
return nil, fmt.Errorf("remaining length %d exceeds MQTT max %d", length, MaxRemainingLength)
|
||||
}
|
||||
var out []byte
|
||||
for {
|
||||
encodedByte := byte(length % 128)
|
||||
length /= 128
|
||||
if length > 0 {
|
||||
encodedByte |= 128
|
||||
}
|
||||
out = append(out, encodedByte)
|
||||
if length == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// decodeVarLength decodes an MQTT variable byte integer from reader.
|
||||
func decodeVarLength(r io.Reader) (int, error) {
|
||||
multiplier := 1
|
||||
value := 0
|
||||
buf := make([]byte, 1)
|
||||
for i := 0; i < 4; i++ {
|
||||
if _, err := io.ReadFull(r, buf); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
encodedByte := buf[0]
|
||||
value += int(encodedByte&127) * multiplier
|
||||
if multiplier > 128*128*128 {
|
||||
return 0, errors.New("malformed remaining length")
|
||||
}
|
||||
multiplier *= 128
|
||||
if encodedByte&128 == 0 {
|
||||
return value, nil
|
||||
}
|
||||
}
|
||||
return 0, errors.New("malformed remaining length")
|
||||
}
|
||||
|
||||
// encodeString encodes a UTF-8 string with a 2-byte big-endian length prefix.
|
||||
func encodeString(s string) ([]byte, error) {
|
||||
b := []byte(s)
|
||||
if len(b) > MaxStringLength {
|
||||
return nil, fmt.Errorf("string length %d exceeds MQTT max %d", len(b), MaxStringLength)
|
||||
}
|
||||
out := make([]byte, 2+len(b))
|
||||
binary.BigEndian.PutUint16(out[0:2], uint16(len(b)))
|
||||
copy(out[2:], b)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// buildConnectPacket constructs an MQTT 3.1.1 CONNECT packet.
|
||||
func buildConnectPacket(clientID, username, password string, cleanSession bool, keepAlive uint16) ([]byte, error) {
|
||||
var varHeader []byte
|
||||
protoName, err := encodeString("MQTT")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
varHeader = append(varHeader, protoName...)
|
||||
varHeader = append(varHeader, 0x04) // Protocol Level: 4 (MQTT 3.1.1)
|
||||
|
||||
// Connect Flags
|
||||
var flags byte
|
||||
if cleanSession {
|
||||
flags |= 0x02
|
||||
}
|
||||
// Per MQTT 3.1.1 spec (MQTT-3.1.2-22): If the User Name Flag is 0, the Password Flag MUST be 0.
|
||||
// If a password is provided, the User Name Flag MUST be set (encoding an empty username if omitted).
|
||||
hasUsername := username != "" || password != ""
|
||||
if hasUsername {
|
||||
flags |= 0x80
|
||||
}
|
||||
if password != "" {
|
||||
flags |= 0x40
|
||||
}
|
||||
varHeader = append(varHeader, flags)
|
||||
|
||||
// Keep Alive (2 bytes big endian)
|
||||
ka := make([]byte, 2)
|
||||
binary.BigEndian.PutUint16(ka, keepAlive)
|
||||
varHeader = append(varHeader, ka...)
|
||||
|
||||
// Payload
|
||||
var payload []byte
|
||||
cidBytes, err := encodeString(clientID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid clientID: %w", err)
|
||||
}
|
||||
payload = append(payload, cidBytes...)
|
||||
|
||||
if hasUsername {
|
||||
userBytes, err := encodeString(username)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid username: %w", err)
|
||||
}
|
||||
payload = append(payload, userBytes...)
|
||||
}
|
||||
if password != "" {
|
||||
passBytes, err := encodeString(password)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid password: %w", err)
|
||||
}
|
||||
payload = append(payload, passBytes...)
|
||||
}
|
||||
|
||||
remainingLength := len(varHeader) + len(payload)
|
||||
lenBytes, err := encodeVarLength(remainingLength)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var pkt []byte
|
||||
pkt = append(pkt, PacketConnect<<4)
|
||||
pkt = append(pkt, lenBytes...)
|
||||
pkt = append(pkt, varHeader...)
|
||||
pkt = append(pkt, payload...)
|
||||
return pkt, nil
|
||||
}
|
||||
|
||||
// buildSubscribePacket constructs an MQTT 3.1.1 SUBSCRIBE packet for a single topic.
|
||||
func buildSubscribePacket(packetID uint16, topic string, qos byte) ([]byte, error) {
|
||||
var varHeader [2]byte
|
||||
binary.BigEndian.PutUint16(varHeader[:], packetID)
|
||||
|
||||
topicBytes, err := encodeString(topic)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid topic: %w", err)
|
||||
}
|
||||
|
||||
var payload []byte
|
||||
payload = append(payload, topicBytes...)
|
||||
payload = append(payload, qos&0x03)
|
||||
|
||||
remainingLength := len(varHeader) + len(payload)
|
||||
lenBytes, err := encodeVarLength(remainingLength)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var pkt []byte
|
||||
// SUBSCRIBE fixed header flags must be 0010 (0x02)
|
||||
pkt = append(pkt, (PacketSubscribe<<4)|0x02)
|
||||
pkt = append(pkt, lenBytes...)
|
||||
pkt = append(pkt, varHeader[:]...)
|
||||
pkt = append(pkt, payload...)
|
||||
return pkt, nil
|
||||
}
|
||||
|
||||
// buildPublishPacket constructs an MQTT 3.1.1 PUBLISH packet.
|
||||
func buildPublishPacket(topic string, payload []byte, qos byte, retain bool, packetID uint16) ([]byte, error) {
|
||||
headerByte := (PacketPublish << 4)
|
||||
if retain {
|
||||
headerByte |= 0x01
|
||||
}
|
||||
headerByte |= (qos & 0x03) << 1
|
||||
|
||||
topicBytes, err := encodeString(topic)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid topic: %w", err)
|
||||
}
|
||||
|
||||
var varHeader []byte
|
||||
varHeader = append(varHeader, topicBytes...)
|
||||
if qos > 0 {
|
||||
pid := make([]byte, 2)
|
||||
binary.BigEndian.PutUint16(pid, packetID)
|
||||
varHeader = append(varHeader, pid...)
|
||||
}
|
||||
|
||||
remainingLength := len(varHeader) + len(payload)
|
||||
lenBytes, err := encodeVarLength(remainingLength)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var pkt []byte
|
||||
pkt = append(pkt, headerByte)
|
||||
pkt = append(pkt, lenBytes...)
|
||||
pkt = append(pkt, varHeader...)
|
||||
pkt = append(pkt, payload...)
|
||||
return pkt, nil
|
||||
}
|
||||
|
||||
// buildPingreqPacket constructs an MQTT PINGREQ packet.
|
||||
func buildPingreqPacket() []byte {
|
||||
return []byte{PacketPingreq << 4, 0x00}
|
||||
}
|
||||
|
||||
// buildDisconnectPacket constructs an MQTT DISCONNECT packet.
|
||||
func buildDisconnectPacket() []byte {
|
||||
return []byte{PacketDisconnect << 4, 0x00}
|
||||
}
|
||||
|
||||
// errBrokenStream means a read failed after part of a packet was consumed, or
|
||||
// a packet was refused, so the next read would start in the middle of a packet.
|
||||
// The connection must not be used again.
|
||||
var errBrokenStream = errors.New("MQTT stream is out of sync")
|
||||
|
||||
// readPacket reads a full MQTT packet from reader, rejecting packets over MaxPacketSize.
|
||||
// Failures before the first byte is read are returned as they are. Failures after
|
||||
// it wrap errBrokenStream.
|
||||
func readPacket(r io.Reader) (headerByte byte, payload []byte, err error) {
|
||||
var header [1]byte
|
||||
if _, err := io.ReadFull(r, header[:]); err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
length, err := decodeVarLength(r)
|
||||
if err != nil {
|
||||
return 0, nil, fmt.Errorf("%w: %w", errBrokenStream, err)
|
||||
}
|
||||
if length > MaxPacketSize {
|
||||
return 0, nil, fmt.Errorf("%w: MQTT packet length %d exceeds maximum allowed %d", errBrokenStream, length, MaxPacketSize)
|
||||
}
|
||||
buf := make([]byte, length)
|
||||
if length > 0 {
|
||||
if _, err := io.ReadFull(r, buf); err != nil {
|
||||
return 0, nil, fmt.Errorf("%w: %w", errBrokenStream, err)
|
||||
}
|
||||
}
|
||||
return header[0], buf, nil
|
||||
}
|
||||
|
||||
// parseConnack decodes a CONNACK variable header.
|
||||
func parseConnack(payload []byte) (sessionPresent bool, returnCode byte, err error) {
|
||||
if len(payload) < 2 {
|
||||
return false, 0, errors.New("short CONNACK response")
|
||||
}
|
||||
sessionPresent = (payload[0] & 0x01) != 0
|
||||
returnCode = payload[1]
|
||||
return sessionPresent, returnCode, nil
|
||||
}
|
||||
|
||||
// parseSuback decodes a SUBACK packet payload.
|
||||
func parseSuback(payload []byte) (packetID uint16, grantedQoS []byte, err error) {
|
||||
if len(payload) < 3 {
|
||||
return 0, nil, errors.New("short SUBACK response")
|
||||
}
|
||||
packetID = binary.BigEndian.Uint16(payload[0:2])
|
||||
grantedQoS = payload[2:]
|
||||
return packetID, grantedQoS, nil
|
||||
}
|
||||
|
||||
// parsePublish decodes a PUBLISH packet.
|
||||
func parsePublish(headerByte byte, body []byte) (*Message, error) {
|
||||
qos := (headerByte >> 1) & 0x03
|
||||
retain := (headerByte & 0x01) != 0
|
||||
|
||||
if len(body) < 2 {
|
||||
return nil, errors.New("short PUBLISH packet")
|
||||
}
|
||||
topicLen := int(binary.BigEndian.Uint16(body[0:2]))
|
||||
if len(body) < 2+topicLen {
|
||||
return nil, errors.New("short PUBLISH topic")
|
||||
}
|
||||
topic := string(body[2 : 2+topicLen])
|
||||
offset := 2 + topicLen
|
||||
|
||||
if qos > 0 {
|
||||
if len(body) < offset+2 {
|
||||
return nil, errors.New("short PUBLISH packet ID")
|
||||
}
|
||||
offset += 2
|
||||
}
|
||||
|
||||
payload := string(body[offset:])
|
||||
return &Message{
|
||||
Topic: topic,
|
||||
Payload: payload,
|
||||
QoS: int(qos),
|
||||
Retain: retain,
|
||||
}, nil
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -20,6 +21,8 @@ import (
|
||||
|
||||
var (
|
||||
defaultTimeout = time.Duration(5) * time.Second
|
||||
|
||||
errNoConnection = errors.New("no connection: open one with net.Open or net.OpenTLS")
|
||||
)
|
||||
|
||||
// getTimeoutFromContext returns the configured TCP timeout or defaultTimeout.
|
||||
@@ -271,12 +274,14 @@ type (
|
||||
// const conn = net.Open('tcp', 'acme.com:80');
|
||||
// ```
|
||||
NetConn struct {
|
||||
// conn is nil when no connection is open.
|
||||
conn net.Conn
|
||||
timeout time.Duration
|
||||
}
|
||||
)
|
||||
|
||||
// Close closes the connection.
|
||||
// Close does nothing if no connection is open.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const net = require('nuclei/net');
|
||||
@@ -284,7 +289,13 @@ type (
|
||||
// conn.Close();
|
||||
// ```
|
||||
func (c *NetConn) Close() error {
|
||||
if c.conn == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := c.conn.Close()
|
||||
c.conn = nil
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -321,6 +332,10 @@ func (c *NetConn) unsetDeadLine() {
|
||||
// conn.SendArray(['hello', 'world']);
|
||||
// ```
|
||||
func (c *NetConn) SendArray(data []interface{}) error {
|
||||
if c.conn == nil {
|
||||
return errNoConnection
|
||||
}
|
||||
|
||||
c.setDeadLine()
|
||||
defer c.unsetDeadLine()
|
||||
input := types.ToByteSlice(data)
|
||||
@@ -342,6 +357,10 @@ func (c *NetConn) SendArray(data []interface{}) error {
|
||||
// conn.SendHex('68656c6c6f');
|
||||
// ```
|
||||
func (c *NetConn) SendHex(data string) error {
|
||||
if c.conn == nil {
|
||||
return errNoConnection
|
||||
}
|
||||
|
||||
c.setDeadLine()
|
||||
defer c.unsetDeadLine()
|
||||
bin, err := hex.DecodeString(data)
|
||||
@@ -366,6 +385,10 @@ func (c *NetConn) SendHex(data string) error {
|
||||
// conn.Send('hello');
|
||||
// ```
|
||||
func (c *NetConn) Send(data string) error {
|
||||
if c.conn == nil {
|
||||
return errNoConnection
|
||||
}
|
||||
|
||||
c.setDeadLine()
|
||||
defer c.unsetDeadLine()
|
||||
bin := []byte(data)
|
||||
@@ -389,6 +412,10 @@ func (c *NetConn) Send(data string) error {
|
||||
// const data = conn.RecvFull(1024);
|
||||
// ```
|
||||
func (c *NetConn) RecvFull(N int) ([]byte, error) {
|
||||
if c.conn == nil {
|
||||
return []byte{}, errNoConnection
|
||||
}
|
||||
|
||||
c.setDeadLine()
|
||||
defer c.unsetDeadLine()
|
||||
if N == 0 {
|
||||
@@ -414,6 +441,10 @@ func (c *NetConn) RecvFull(N int) ([]byte, error) {
|
||||
// log(`Received ${data.length} bytes from the server`)
|
||||
// ```
|
||||
func (c *NetConn) Recv(N int) ([]byte, error) {
|
||||
if c.conn == nil {
|
||||
return []byte{}, errNoConnection
|
||||
}
|
||||
|
||||
c.setDeadLine()
|
||||
defer c.unsetDeadLine()
|
||||
if N == 0 {
|
||||
|
||||
@@ -406,3 +406,41 @@ func (f *fakeConn) RemoteAddr() net.Addr { return nil }
|
||||
func (f *fakeConn) SetDeadline(time.Time) error { return nil }
|
||||
func (f *fakeConn) SetReadDeadline(time.Time) error { return nil }
|
||||
func (f *fakeConn) SetWriteDeadline(time.Time) error { return nil }
|
||||
|
||||
func TestNetConnWithoutConnection(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := map[string]func(t *testing.T) *NetConn{
|
||||
"never opened": func(*testing.T) *NetConn { return new(NetConn) },
|
||||
"already closed": func(t *testing.T) *NetConn {
|
||||
t.Helper()
|
||||
|
||||
local, remote := net.Pipe()
|
||||
|
||||
t.Cleanup(func() { _ = remote.Close() })
|
||||
|
||||
conn := new(NetConn)
|
||||
conn.conn = local
|
||||
require.NoError(t, conn.Close())
|
||||
|
||||
return conn
|
||||
},
|
||||
}
|
||||
for name, newConn := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
conn := newConn(t)
|
||||
require.NoError(t, conn.Close())
|
||||
require.ErrorContains(t, conn.Send("x"), "no connection")
|
||||
require.ErrorContains(t, conn.SendHex("78"), "no connection")
|
||||
require.ErrorContains(t, conn.SendArray([]any{120}), "no connection")
|
||||
|
||||
_, err := conn.Recv(1)
|
||||
require.ErrorContains(t, err, "no connection")
|
||||
|
||||
_, err = conn.RecvFull(1)
|
||||
require.ErrorContains(t, err, "no connection")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,12 +3,12 @@ package postgres
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-pg/pg/v10"
|
||||
"github.com/praetorian-inc/fingerprintx/pkg/plugins"
|
||||
postgres "github.com/praetorian-inc/fingerprintx/pkg/plugins/services/postgresql"
|
||||
utils "github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
|
||||
type (
|
||||
// PGClient is a client for Postgres database.
|
||||
// Internally client uses go-pg/pg driver.
|
||||
// Internally client uses lib/pq driver.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const postgres = require('nuclei/postgres');
|
||||
@@ -124,9 +124,7 @@ func executeQuery(ctx context.Context, executionId string, host string, port int
|
||||
return nil, protocolstate.ErrHostDenied.Msgf(host)
|
||||
}
|
||||
|
||||
target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
|
||||
|
||||
connStr := buildPostgresConnURL(username, password, target, dbName, executionId)
|
||||
connStr := buildPostgresDSN(host, port, username, password, dbName)
|
||||
db, err := pgwrap.OpenDB(ctx, executionId, connStr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -146,17 +144,30 @@ func executeQuery(ctx context.Context, executionId string, host string, port int
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func buildPostgresConnURL(username, password, target, dbName, executionId string) string {
|
||||
values := url.Values{}
|
||||
values.Set("sslmode", "disable")
|
||||
values.Set("executionId", executionId)
|
||||
|
||||
return fmt.Sprintf("postgres://%s@%s/%s?%s",
|
||||
url.UserPassword(username, password).String(),
|
||||
target,
|
||||
url.PathEscape(dbName),
|
||||
values.Encode(),
|
||||
)
|
||||
// buildPostgresDSN returns a lib/pq key=value connection string. user, password, dbname, service and passfile
|
||||
// are always set, also when empty: lib/pq fills unset values from the scanner's environment (PGUSER, PGPASSWORD,
|
||||
// PGDATABASE), a service file (PGSERVICE) and ~/.pgpass (PGPASSFILE), which sends the operator's credentials to the target.
|
||||
func buildPostgresDSN(host string, port int, username, password, dbName string) string {
|
||||
quote := strings.NewReplacer(`\`, `\\`, `'`, `\'`)
|
||||
params := []struct{ key, value string }{
|
||||
{"host", host},
|
||||
{"port", strconv.Itoa(port)},
|
||||
{"user", username},
|
||||
{"password", password},
|
||||
{"dbname", dbName},
|
||||
{"sslmode", "disable"},
|
||||
{"connect_timeout", "10"},
|
||||
{"service", ""},
|
||||
{"options", ""},
|
||||
{"application_name", ""},
|
||||
// a path below a file never exists, so no password file is read
|
||||
{"passfile", filepath.Join(os.DevNull, "pgpass")},
|
||||
}
|
||||
kvs := make([]string, 0, len(params))
|
||||
for _, param := range params {
|
||||
kvs = append(kvs, param.key+"='"+quote.Replace(param.value)+"'")
|
||||
}
|
||||
return strings.Join(kvs, " ")
|
||||
}
|
||||
|
||||
// ConnectWithDB connects to Postgres database using given credentials and database name.
|
||||
@@ -194,32 +205,20 @@ func connect(ctx context.Context, executionId string, host string, port int, use
|
||||
return false, protocolstate.ErrHostDenied.Msgf(host)
|
||||
}
|
||||
|
||||
target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
|
||||
|
||||
execCtx, cancel := context.WithCancel(ctx)
|
||||
execCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
dialer := protocolstate.GetDialersWithId(executionId)
|
||||
if dialer == nil {
|
||||
return false, fmt.Errorf("dialers not initialized for %s", executionId)
|
||||
// lib/pq with every credential set, as go-pg replaces an empty user, password or database
|
||||
// with PGUSER, PGPASSWORD or PGDATABASE from the scanner's environment
|
||||
db, err := pgwrap.OpenDB(execCtx, executionId, buildPostgresDSN(host, port, username, password, dbName))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
db := pg.Connect(&pg.Options{
|
||||
Addr: target,
|
||||
User: username,
|
||||
Password: password,
|
||||
Database: dbName,
|
||||
Dialer: func(dialCtx context.Context, network, addr string) (net.Conn, error) {
|
||||
return dialer.Fastdialer.Dial(dialCtx, network, addr)
|
||||
},
|
||||
IdleCheckFrequency: -1,
|
||||
}).WithTimeout(10 * time.Second)
|
||||
|
||||
defer func() {
|
||||
_ = db.Close()
|
||||
}()
|
||||
|
||||
_, err := db.ExecContext(execCtx, "select 1")
|
||||
_, err = db.ExecContext(execCtx, "select 1")
|
||||
if err != nil {
|
||||
switch true {
|
||||
case strings.Contains(err.Error(), "connect: connection refused"):
|
||||
|
||||
@@ -1,74 +1,165 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBuildPostgresConnectionURLDoesNotAllowDBNameQueryInjection(t *testing.T) {
|
||||
dbName := "testdb?sslrootcert=/etc/passwd&sslmode=verify-ca&junk="
|
||||
connStr := buildPostgresConnURL("postgres", "x", "127.0.0.1:5432", dbName, "exec-1")
|
||||
func TestBuildPostgresDSNEscapesDBName(t *testing.T) {
|
||||
dbName := "testdb' sslrootcert='/etc/passwd' sslmode='verify-ca"
|
||||
cfg, err := pq.NewConfig(buildPostgresDSN("127.0.0.1", 5432, "postgres", "x", dbName))
|
||||
require.NoError(t, err)
|
||||
|
||||
u, err := url.Parse(connStr)
|
||||
if err != nil {
|
||||
t.Fatalf("parse connection URL: %v", err)
|
||||
}
|
||||
|
||||
if got := strings.TrimPrefix(u.Path, "/"); got != dbName {
|
||||
t.Fatalf("database name = %q, want %q", got, dbName)
|
||||
}
|
||||
|
||||
values := u.Query()
|
||||
if got := values.Get("sslmode"); got != "disable" {
|
||||
t.Fatalf("sslmode = %q, want disable", got)
|
||||
}
|
||||
if got := values.Get("executionId"); got != "exec-1" {
|
||||
t.Fatalf("executionId = %q, want exec-1", got)
|
||||
}
|
||||
deniedParams := []string{"sslrootcert", "sslcert", "sslkey", "service", "junk"}
|
||||
for _, denied := range deniedParams {
|
||||
if got := values.Get(denied); got != "" {
|
||||
t.Fatalf("%s was injected with value %q", denied, got)
|
||||
}
|
||||
}
|
||||
|
||||
pqDSN, err := pq.ParseURL(connStr) //nolint:staticcheck // validates lib/pq URL parsing of the generated DSN.
|
||||
if err != nil {
|
||||
t.Fatalf("parse connection URL as lib/pq DSN: %v", err)
|
||||
}
|
||||
if !strings.Contains(pqDSN, "dbname='"+dbName+"'") {
|
||||
t.Fatalf("lib/pq DSN = %q, want dbName preserved as dbname", pqDSN)
|
||||
}
|
||||
for _, denied := range deniedParams {
|
||||
if strings.Contains(pqDSN, " "+denied+"=") {
|
||||
t.Fatalf("%s was injected into lib/pq DSN %q", denied, pqDSN)
|
||||
}
|
||||
}
|
||||
require.Equal(t, dbName, cfg.Database)
|
||||
require.Equal(t, pq.SSLModeDisable, cfg.SSLMode)
|
||||
require.Empty(t, cfg.SSLRootCert)
|
||||
require.Empty(t, cfg.Runtime)
|
||||
}
|
||||
|
||||
func TestBuildPostgresConnectionURLEscapesCredentials(t *testing.T) {
|
||||
username := "user:[email protected]"
|
||||
password := "pa:ss@word?x"
|
||||
connStr := buildPostgresConnURL(username, password, "127.0.0.1:5432", "postgres", "exec-1")
|
||||
func TestBuildPostgresDSNEscapesCredentials(t *testing.T) {
|
||||
username := `user:[email protected]' host='other`
|
||||
password := `pa:ss@word?x\' sslmode='require`
|
||||
cfg, err := pq.NewConfig(buildPostgresDSN("127.0.0.1", 5432, username, password, "postgres"))
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, username, cfg.User)
|
||||
require.Equal(t, password, cfg.Password)
|
||||
require.Equal(t, "127.0.0.1", cfg.Host)
|
||||
require.Equal(t, uint16(5432), cfg.Port)
|
||||
require.Equal(t, "postgres", cfg.Database)
|
||||
require.Equal(t, pq.SSLModeDisable, cfg.SSLMode)
|
||||
}
|
||||
|
||||
// setOperatorPostgresEnv sets every source lib/pq and go-pg read credentials from
|
||||
func setOperatorPostgresEnv(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
pgpass := filepath.Join(home, ".pgpass")
|
||||
require.NoError(t, os.WriteFile(pgpass, []byte("*:*:*:*:OPERATOR-PGPASS-SECRET\n"), 0600))
|
||||
serviceFile := filepath.Join(home, ".pg_service.conf")
|
||||
require.NoError(t, os.WriteFile(serviceFile, []byte("[operator]\npassword=OPERATOR-SERVICE-SECRET\n"), 0600))
|
||||
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("PGPASSFILE", pgpass)
|
||||
t.Setenv("PGSERVICEFILE", serviceFile)
|
||||
t.Setenv("PGSERVICE", "operator")
|
||||
t.Setenv("PGPASSWORD", "OPERATOR-ENV-PGPASSWORD")
|
||||
t.Setenv("PGUSER", "operator-user")
|
||||
t.Setenv("PGDATABASE", "operator-db")
|
||||
t.Setenv("PGOPTIONS", "-c search_path=operator")
|
||||
t.Setenv("PGAPPNAME", "operator-app")
|
||||
}
|
||||
|
||||
func TestBuildPostgresDSNIgnoresOperatorEnvironment(t *testing.T) {
|
||||
setOperatorPostgresEnv(t)
|
||||
|
||||
cfg, err := pq.NewConfig(buildPostgresDSN("127.0.0.1", 5432, "", "", ""))
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, cfg.User)
|
||||
require.Empty(t, cfg.Password)
|
||||
require.Empty(t, cfg.Database)
|
||||
require.Empty(t, cfg.Service)
|
||||
require.Empty(t, cfg.Options)
|
||||
require.Empty(t, cfg.ApplicationName)
|
||||
}
|
||||
|
||||
type startup struct {
|
||||
user, database, password string
|
||||
}
|
||||
|
||||
// fakePostgres asks every client for a cleartext password, records it and rejects the login
|
||||
func fakePostgres(t *testing.T) (int, <-chan startup) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = listener.Close() })
|
||||
|
||||
logins := make(chan startup, 16)
|
||||
go func() {
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func(conn net.Conn) {
|
||||
defer func() { _ = conn.Close() }()
|
||||
readMessage := func(typed bool) []byte {
|
||||
header := make([]byte, 4)
|
||||
if typed {
|
||||
header = make([]byte, 5)
|
||||
}
|
||||
if _, err := io.ReadFull(conn, header); err != nil {
|
||||
return nil
|
||||
}
|
||||
body := make([]byte, binary.BigEndian.Uint32(header[len(header)-4:])-4)
|
||||
if _, err := io.ReadFull(conn, body); err != nil {
|
||||
return nil
|
||||
}
|
||||
return body
|
||||
}
|
||||
params := readMessage(false)
|
||||
if len(params) < 4 {
|
||||
return
|
||||
}
|
||||
fields := strings.Split(string(params[4:]), "\x00")
|
||||
var login startup
|
||||
for i := 0; i+1 < len(fields); i += 2 {
|
||||
switch fields[i] {
|
||||
case "user":
|
||||
login.user = fields[i+1]
|
||||
case "database":
|
||||
login.database = fields[i+1]
|
||||
}
|
||||
}
|
||||
// AuthenticationCleartextPassword
|
||||
if _, err := conn.Write([]byte{'R', 0, 0, 0, 8, 0, 0, 0, 3}); err != nil {
|
||||
return
|
||||
}
|
||||
password := readMessage(true)
|
||||
login.password = strings.TrimSuffix(string(password), "\x00")
|
||||
logins <- login
|
||||
errorBody := "SFATAL\x00C28P01\x00Mpassword authentication failed\x00\x00"
|
||||
header := []byte{'E', 0, 0, 0, 0}
|
||||
binary.BigEndian.PutUint32(header[1:], uint32(len(errorBody)+4))
|
||||
_, _ = conn.Write(append(header, errorBody...))
|
||||
}(conn)
|
||||
}
|
||||
}()
|
||||
return listener.Addr().(*net.TCPAddr).Port, logins
|
||||
}
|
||||
|
||||
func TestPostgresEmptyCredentialsAreNotFilledFromEnvironment(t *testing.T) {
|
||||
setOperatorPostgresEnv(t)
|
||||
|
||||
executionID := t.Name()
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: executionID}))
|
||||
t.Cleanup(func() { protocolstate.Close(executionID) })
|
||||
|
||||
port, logins := fakePostgres(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := executeQuery(ctx, executionID, "127.0.0.1", port, "postgres", "", "postgres", "select 1")
|
||||
require.Error(t, err)
|
||||
require.Equal(t, startup{user: "postgres", database: "postgres"}, <-logins)
|
||||
|
||||
connected, err := connect(ctx, executionID, "127.0.0.1", port, "postgres", "", "postgres")
|
||||
require.NoError(t, err)
|
||||
require.False(t, connected)
|
||||
require.Equal(t, startup{user: "postgres", database: "postgres"}, <-logins)
|
||||
|
||||
_, _ = connect(ctx, executionID, "127.0.0.1", port, "", "", "")
|
||||
require.Equal(t, startup{}, <-logins)
|
||||
|
||||
_, _ = executeQuery(ctx, executionID, "127.0.0.1", port, "admin", "secret", "db", "select 1")
|
||||
require.Equal(t, startup{user: "admin", database: "db", password: "secret"}, <-logins)
|
||||
|
||||
u, err := url.Parse(connStr)
|
||||
if err != nil {
|
||||
t.Fatalf("parse connection URL: %v", err)
|
||||
}
|
||||
|
||||
if got := u.User.Username(); got != username {
|
||||
t.Fatalf("username = %q, want %q", got, username)
|
||||
}
|
||||
if got, _ := u.User.Password(); got != password {
|
||||
t.Fatalf("password = %q, want %q", got, password)
|
||||
}
|
||||
if got := u.Host; got != "127.0.0.1:5432" {
|
||||
t.Fatalf("host = %q, want 127.0.0.1:5432", got)
|
||||
}
|
||||
if got := strings.TrimPrefix(u.Path, "/"); got != "postgres" {
|
||||
t.Fatalf("database name = %q, want postgres", got)
|
||||
}
|
||||
}
|
||||
+12
-2
@@ -23,6 +23,7 @@ type (
|
||||
// const client = new ssh.SSHClient();
|
||||
// ```
|
||||
SSHClient struct {
|
||||
// connection is nil when no connection is open.
|
||||
connection *ssh.Client
|
||||
timeout time.Duration
|
||||
}
|
||||
@@ -158,7 +159,8 @@ func (c *SSHClient) Run(cmd string) (string, error) {
|
||||
|
||||
// Close closes the SSH connection and destroys the client
|
||||
// Returns the success state and error. If error is not nil,
|
||||
// state will be false
|
||||
// state will be false. Close does nothing and returns true
|
||||
// if no connection is open.
|
||||
// @example
|
||||
// ```javascript
|
||||
// const ssh = require('nuclei/ssh');
|
||||
@@ -167,9 +169,17 @@ func (c *SSHClient) Run(cmd string) (string, error) {
|
||||
// const closed = client.Close();
|
||||
// ```
|
||||
func (c *SSHClient) Close() (bool, error) {
|
||||
if err := c.connection.Close(); err != nil {
|
||||
if c.connection == nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
err := c.connection.Close()
|
||||
c.connection = nil
|
||||
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
package ssh_test
|
||||
|
||||
//nolint:depguard // No depguard rules are configured, so its default list allows only the standard library.
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
libssh "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/ssh"
|
||||
"github.com/zmap/zgrab2/lib/ssh"
|
||||
)
|
||||
|
||||
const (
|
||||
testUser = "user"
|
||||
testPassword = "secret"
|
||||
)
|
||||
|
||||
var errAccessDenied = errors.New("access denied")
|
||||
|
||||
func TestCloseWithoutConnection(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
host, port := startServer(t)
|
||||
|
||||
tests := map[string]func(t *testing.T, client *libssh.SSHClient){
|
||||
"never connected": func(*testing.T, *libssh.SSHClient) {},
|
||||
"failed connect": func(t *testing.T, client *libssh.SSHClient) {
|
||||
t.Helper()
|
||||
|
||||
_, err := client.Connect(executionContext(t), host, port, testUser, "wrong")
|
||||
if err == nil {
|
||||
t.Fatal("Connect() with a wrong password succeeded")
|
||||
}
|
||||
},
|
||||
"already closed": func(t *testing.T, client *libssh.SSHClient) {
|
||||
t.Helper()
|
||||
|
||||
connect(t, client, host, port)
|
||||
closeClient(t, client)
|
||||
},
|
||||
}
|
||||
for name, setup := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := new(libssh.SSHClient)
|
||||
setup(t, client)
|
||||
closeClient(t, client)
|
||||
|
||||
_, err := client.Run("id")
|
||||
if err == nil || !strings.Contains(err.Error(), "no connection") {
|
||||
t.Errorf("Run() error = %v, want no connection", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseConnected(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
host, port := startServer(t)
|
||||
client := new(libssh.SSHClient)
|
||||
connect(t, client, host, port)
|
||||
closeClient(t, client)
|
||||
}
|
||||
|
||||
func connect(t *testing.T, client *libssh.SSHClient, host string, port int) {
|
||||
t.Helper()
|
||||
|
||||
_, err := client.Connect(executionContext(t), host, port, testUser, testPassword)
|
||||
if err != nil {
|
||||
t.Fatalf("Connect() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func closeClient(t *testing.T, client *libssh.SSHClient) {
|
||||
t.Helper()
|
||||
|
||||
closed, err := client.Close()
|
||||
if err != nil || !closed {
|
||||
t.Fatalf("Close() = %v, %v; want true, nil", closed, err)
|
||||
}
|
||||
}
|
||||
|
||||
func executionContext(t *testing.T) context.Context {
|
||||
t.Helper()
|
||||
|
||||
return context.WithValue(t.Context(), "executionId", t.Name()) //nolint:staticcheck // Connect reads this string key.
|
||||
}
|
||||
|
||||
// startServer serves SSH on a loopback port and accepts only
|
||||
// testUser with testPassword. It rejects every channel.
|
||||
func startServer(t *testing.T) (string, int) {
|
||||
t.Helper()
|
||||
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
signer, err := ssh.NewSignerFromKey(key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
config := new(ssh.ServerConfig)
|
||||
config.PasswordCallback = checkPassword
|
||||
config.AddHostKey(signer)
|
||||
|
||||
listener, err := new(net.ListenConfig).Listen(t.Context(), "tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() { _ = listener.Close() })
|
||||
|
||||
go func() {
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
go serve(conn, config)
|
||||
}
|
||||
}()
|
||||
|
||||
addr, ok := listener.Addr().(*net.TCPAddr)
|
||||
if !ok {
|
||||
t.Fatalf("listener address %v is not TCP", listener.Addr())
|
||||
}
|
||||
|
||||
return addr.IP.String(), addr.Port
|
||||
}
|
||||
|
||||
func checkPassword(conn ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) {
|
||||
if conn.User() != testUser || string(password) != testPassword {
|
||||
return nil, errAccessDenied
|
||||
}
|
||||
|
||||
return new(ssh.Permissions), nil
|
||||
}
|
||||
|
||||
func serve(conn net.Conn, config *ssh.ServerConfig) {
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
_, chans, reqs, err := ssh.NewServerConn(conn, config)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
go ssh.DiscardRequests(reqs)
|
||||
|
||||
for ch := range chans {
|
||||
_ = ch.Reject(ssh.Prohibited, "test server")
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,10 @@
|
||||
package output
|
||||
|
||||
import json "github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
json "github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
)
|
||||
|
||||
// formatJSON formats the output for json based formatting
|
||||
func (w *StandardWriter) formatJSON(output *ResultEvent) ([]byte, error) {
|
||||
@@ -8,5 +12,50 @@ func (w *StandardWriter) formatJSON(output *ResultEvent) ([]byte, error) {
|
||||
output.Request = ""
|
||||
output.Response = ""
|
||||
}
|
||||
if len(output.Info.Metadata) > 0 {
|
||||
output.Info.Metadata = sanitizeMetadataForJSON(output.Info.Metadata)
|
||||
}
|
||||
if len(output.Metadata) > 0 {
|
||||
output.Metadata = sanitizeMetadataForJSON(output.Metadata)
|
||||
}
|
||||
return json.Marshal(output)
|
||||
}
|
||||
|
||||
// sanitizeMetadataForJSON normalizes any map[interface{}]interface{} or nested
|
||||
// collections in metadata to map[string]interface{}, ensuring JSON encoders
|
||||
// (such as sonic or standard encoding/json) can serialize them without error (#7971).
|
||||
func sanitizeMetadataForJSON(m map[string]interface{}) map[string]interface{} {
|
||||
if len(m) == 0 {
|
||||
return m
|
||||
}
|
||||
sanitized := sanitizeJSONValue(m)
|
||||
if res, ok := sanitized.(map[string]interface{}); ok {
|
||||
return res
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func sanitizeJSONValue(v interface{}) interface{} {
|
||||
switch typed := v.(type) {
|
||||
case map[interface{}]interface{}:
|
||||
converted := make(map[string]interface{}, len(typed))
|
||||
for k, val := range typed {
|
||||
converted[fmt.Sprint(k)] = sanitizeJSONValue(val)
|
||||
}
|
||||
return converted
|
||||
case map[string]interface{}:
|
||||
converted := make(map[string]interface{}, len(typed))
|
||||
for k, val := range typed {
|
||||
converted[k] = sanitizeJSONValue(val)
|
||||
}
|
||||
return converted
|
||||
case []interface{}:
|
||||
converted := make([]interface{}, len(typed))
|
||||
for i, val := range typed {
|
||||
converted[i] = sanitizeJSONValue(val)
|
||||
}
|
||||
return converted
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
@@ -209,3 +209,33 @@ type testWriteCloser struct {
|
||||
func (w testWriteCloser) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
func TestFormatJSONSanitizesNestedMetadataMaps(t *testing.T) {
|
||||
w, err := NewStandardWriter(&types.Options{})
|
||||
require.NoError(t, err)
|
||||
|
||||
event := &ResultEvent{
|
||||
TemplateID: "nested-metadata",
|
||||
Type: "file",
|
||||
Host: "target",
|
||||
Matched: "target/input.txt",
|
||||
Info: model.Info{
|
||||
Name: "Nested metadata",
|
||||
Metadata: map[string]interface{}{
|
||||
"labels": map[interface{}]interface{}{
|
||||
"en": "Hello",
|
||||
"nested_list": []interface{}{
|
||||
map[interface{}]interface{}{
|
||||
"inner_key": "inner_val",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data, err := w.formatJSON(event)
|
||||
require.NoError(t, err, "formatJSON should not fail on nested map[interface{}]interface{} (#7971)")
|
||||
require.Contains(t, string(data), `"labels":{"en":"Hello","nested_list":[{"inner_key":"inner_val"}]}`)
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/alecthomas/chroma/quick"
|
||||
"github.com/alecthomas/chroma/v2/quick"
|
||||
"github.com/ditashi/jsbeautifier-go/jsbeautifier"
|
||||
"github.com/projectdiscovery/goja"
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -20,6 +21,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan/techgraph"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
@@ -62,6 +64,14 @@ type Service struct {
|
||||
techTemplates []*templates.Template
|
||||
ServiceOpts Options
|
||||
hasResults *atomic.Bool
|
||||
|
||||
// tech-graph based selection (automatic scan v2). When selector is nil the
|
||||
// service falls back to the legacy tag-overlap selection.
|
||||
selector *techgraph.Selector
|
||||
tier techgraph.Tier
|
||||
universe map[string]*templates.Template
|
||||
universeErr error
|
||||
universeOnce sync.Once
|
||||
}
|
||||
|
||||
// New takes options and returns a new automatic scan service
|
||||
@@ -103,7 +113,7 @@ func New(opts Options) (*Service, error) {
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not get http client")
|
||||
}
|
||||
return &Service{
|
||||
service := &Service{
|
||||
opts: opts.ExecuterOpts,
|
||||
store: opts.Store,
|
||||
engine: opts.Engine,
|
||||
@@ -115,7 +125,22 @@ func New(opts Options) (*Service, error) {
|
||||
techTemplates: techDetectTemplates,
|
||||
ServiceOpts: opts,
|
||||
hasResults: &atomic.Bool{},
|
||||
}, nil
|
||||
}
|
||||
tier, err := techgraph.ParseTier(opts.ExecuterOpts.Options.AutomaticScanCoverage)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
service.tier = tier
|
||||
|
||||
// load the embedded tech-graph for v2 selection; on failure we silently fall
|
||||
// back to legacy tag-overlap so behaviour never regresses.
|
||||
if graph, err := techgraph.Embedded(); err == nil {
|
||||
service.selector = techgraph.NewSelector(graph)
|
||||
} else {
|
||||
gologger.Warning().Msgf("automatic scan: tech-graph unavailable, using legacy selection: %s", err)
|
||||
}
|
||||
|
||||
return service, nil
|
||||
}
|
||||
|
||||
// Close closes the service
|
||||
@@ -179,7 +204,9 @@ func (s *Service) executeAutomaticScanOnTarget(ctx context.Context, input *conte
|
||||
finalTags = append(finalTags, s.opts.Options.Tags...)
|
||||
finalTags = sliceutil.Dedupe(finalTags)
|
||||
|
||||
if len(finalTags) == 0 {
|
||||
// Lean has no baseline, so an empty signal set has nothing to run. Balanced
|
||||
// and thorough still run the tech-agnostic baseline.
|
||||
if len(finalTags) == 0 && (s.selector == nil || s.tier == techgraph.TierLean) {
|
||||
gologger.Warning().Msgf("Skipping automatic scan since no tags were found on %v\n", input.Input)
|
||||
return
|
||||
}
|
||||
@@ -187,9 +214,19 @@ func (s *Service) executeAutomaticScanOnTarget(ctx context.Context, input *conte
|
||||
gologger.Print().Msgf("Final tags identified for %v: %+v\n", input.Input, finalTags)
|
||||
}
|
||||
|
||||
finalTemplates, err := LoadTemplatesWithTags(s.ServiceOpts, s.templateDirs, finalTags, false)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("%v Error loading templates: %s\n", input.Input, err)
|
||||
var finalTemplates []*templates.Template
|
||||
if s.selector != nil {
|
||||
finalTemplates = s.selectWithGraph(input, finalTags)
|
||||
} else {
|
||||
var err error
|
||||
finalTemplates, err = LoadTemplatesWithTags(s.ServiceOpts, s.templateDirs, finalTags, false)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("%v Error loading templates: %s\n", input.Input, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if len(finalTemplates) == 0 {
|
||||
gologger.Warning().Msgf("Skipping automatic scan since no templates were selected on %v\n", input.Input)
|
||||
return
|
||||
}
|
||||
gologger.Info().Msgf("Executing %d templates on %v", len(finalTemplates), input.Input)
|
||||
@@ -205,12 +242,137 @@ func (s *Service) executeAutomaticScanOnTarget(ctx context.Context, input *conte
|
||||
s.hasResults.Store(tmp.Load())
|
||||
}
|
||||
|
||||
// ensureUniverse lazily loads every selectable template once and indexes it by
|
||||
// id. This is the pool from which graph-based selection draws (precise + fuzzy),
|
||||
// avoiding a per-target loader pass.
|
||||
func (s *Service) ensureUniverse() (map[string]*templates.Template, error) {
|
||||
s.universeOnce.Do(func() {
|
||||
all, err := s.store.LoadTemplates(s.templateDirs)
|
||||
if err != nil {
|
||||
s.universeErr = err
|
||||
return
|
||||
}
|
||||
s.universe = make(map[string]*templates.Template, len(all))
|
||||
for _, t := range all {
|
||||
if t != nil {
|
||||
s.universe[t.ID] = t
|
||||
}
|
||||
}
|
||||
})
|
||||
return s.universe, s.universeErr
|
||||
}
|
||||
|
||||
// selectWithGraph resolves detected signals to a precise fire-set via the
|
||||
// tech-graph, augmented (except at the lean tier) by legacy tag-overlap for any
|
||||
// signals not present in the graph, so recall never falls below legacy behaviour.
|
||||
func (s *Service) selectWithGraph(input *contextargs.MetaInput, finalTags []string) []*templates.Template {
|
||||
universe, err := s.ensureUniverse()
|
||||
if err != nil || len(universe) == 0 {
|
||||
if len(finalTags) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("automatic scan: template load failed, using tag selection: %s", err)
|
||||
}
|
||||
tpls, loadErr := LoadTemplatesWithTags(s.ServiceOpts, s.templateDirs, finalTags, false)
|
||||
if loadErr != nil {
|
||||
gologger.Error().Msgf("%v Error loading templates: %s\n", input.Input, loadErr)
|
||||
return nil
|
||||
}
|
||||
return tpls
|
||||
}
|
||||
|
||||
techIDs, unresolved := s.selector.Resolve(finalTags)
|
||||
selected := map[string]*templates.Template{}
|
||||
|
||||
graph := 0
|
||||
for _, id := range s.selector.FireSet(s.tier, techIDs) {
|
||||
if t, ok := universe[id]; ok {
|
||||
selected[id] = t
|
||||
graph++
|
||||
}
|
||||
}
|
||||
|
||||
added := 0
|
||||
if s.tier != techgraph.TierLean && len(techIDs) > 0 {
|
||||
candidates := make([]techgraph.Candidate, 0, len(universe))
|
||||
for id, t := range universe {
|
||||
if _, ok := selected[id]; ok {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, techgraph.Candidate{
|
||||
ID: id,
|
||||
Tags: t.Info.Tags.ToSlice(),
|
||||
Category: categoryFromPath(t.Path),
|
||||
})
|
||||
}
|
||||
for _, id := range s.selector.AbsentDependents(techIDs, candidates) {
|
||||
if t, ok := universe[id]; ok {
|
||||
selected[id] = t
|
||||
added++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fuzzy := 0
|
||||
if s.tier != techgraph.TierLean && len(unresolved) > 0 {
|
||||
want := map[string]struct{}{}
|
||||
for _, tag := range unresolved {
|
||||
want[strings.ToLower(tag)] = struct{}{}
|
||||
}
|
||||
for id, t := range universe {
|
||||
if _, ok := selected[id]; ok {
|
||||
continue
|
||||
}
|
||||
for _, tag := range t.Info.Tags.ToSlice() {
|
||||
if _, ok := want[strings.ToLower(tag)]; ok {
|
||||
selected[id] = t
|
||||
fuzzy++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]*templates.Template, 0, len(selected))
|
||||
for _, t := range selected {
|
||||
out = append(out, t)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
|
||||
|
||||
if s.opts.Options.VerboseVerbose {
|
||||
gologger.Print().Msgf("%v selection [tier=%s]: %d techs, %d graph + %d new + %d fuzzy = %d templates\n",
|
||||
input.Input, s.tier, len(techIDs), graph, added, fuzzy, len(out))
|
||||
}
|
||||
|
||||
if !s.opts.Options.DisableClustering {
|
||||
out, _, _ = templates.ClusterTemplates(out, s.opts)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type sharedProgress struct {
|
||||
progress.Progress
|
||||
}
|
||||
|
||||
func (p *sharedProgress) Init(hostCount int64, rulesCount int, requestCount int64) {}
|
||||
|
||||
// categoryFromPath returns the nuclei category directory for a template path
|
||||
// (http/cves/2024/x.yaml -> cves). An empty result means the category is unknown
|
||||
// and the template is not eligible for the post-graph supplement.
|
||||
func categoryFromPath(path string) string {
|
||||
parts := strings.Split(filepath.ToSlash(path), "/")
|
||||
for i, part := range parts {
|
||||
switch part {
|
||||
case "http", "network", "dns", "ssl", "file", "headless", "javascript", "code", "cloud", "dast":
|
||||
if i+1 < len(parts) {
|
||||
return parts[i+1]
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// getTagsUsingWappalyzer returns tags using wappalyzer by fingerprinting target
|
||||
// and utilizing the mapping data
|
||||
func (s *Service) getTagsUsingWappalyzer(input *contextargs.MetaInput) []string {
|
||||
|
||||
@@ -13,3 +13,9 @@ func TestNormalizeAppName(t *testing.T) {
|
||||
appName = normalizeAppName("JBoss:2.3.5")
|
||||
require.Equal(t, "jboss", appName, "could not get normalized name")
|
||||
}
|
||||
|
||||
func TestCategoryFromPath(t *testing.T) {
|
||||
require.Equal(t, "cves", categoryFromPath("/home/templates/http/cves/2024/CVE-2024-1.yaml"))
|
||||
require.Equal(t, "fuzzing", categoryFromPath("http/fuzzing/xss.yaml"))
|
||||
require.Equal(t, "", categoryFromPath("README.md"))
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package techgraph
|
||||
|
||||
import "strings"
|
||||
|
||||
// excludedCategories never participate in automatic scan (opt-in/specialized).
|
||||
var excludedCategories = map[string]struct{}{
|
||||
"fuzzing": {},
|
||||
"osint": {},
|
||||
"token-spray": {},
|
||||
"malware": {},
|
||||
"cloud": {},
|
||||
}
|
||||
|
||||
// baselineCategories are the only categories whose product-less templates may be
|
||||
// considered for the tech-agnostic baseline. Product-specific categories (cves,
|
||||
// vulnerabilities, exposed-panels, default-logins, cnvd) are intentionally absent:
|
||||
// an unmappable template there must go to unmapped (fallback), never baseline.
|
||||
var baselineCategories = map[string]struct{}{
|
||||
"exposures": {},
|
||||
"misconfiguration": {},
|
||||
"miscellaneous": {},
|
||||
"iot": {},
|
||||
"takeovers": {}, // subdomain takeover signatures are run host-agnostically
|
||||
}
|
||||
|
||||
// genericBaselineTags positively identify a tech-agnostic check.
|
||||
var genericBaselineTags = map[string]struct{}{
|
||||
"cors": {}, "crlf": {}, "open-redirect": {}, "ds_store": {}, "dsstore": {},
|
||||
"phpinfo": {}, "directory-listing": {}, "listing": {}, "debug": {},
|
||||
"robots": {}, "sitemap": {}, "git": {}, "svn": {}, "dotenv": {},
|
||||
"host-header-injection": {}, "http-trace": {}, "options-method": {},
|
||||
"well-known": {}, "trace": {},
|
||||
}
|
||||
|
||||
// genericBaselineIDSubstrings positively identify generic checks by id.
|
||||
var genericBaselineIDSubstrings = []string{
|
||||
"git-config", "gitignore", "git-credentials", "svn-", ".env", "dotenv",
|
||||
"phpinfo", "directory-listing", "dir-listing", "ds-store", "ds_store",
|
||||
"robots-txt", "sitemap-", "cors-", "crlf-", "open-redirect", "trace-method",
|
||||
"options-method", "well-known", "backup-files", "sql-dump",
|
||||
}
|
||||
|
||||
// classifyBaseline decides whether an otherwise-unmapped template is a genuinely
|
||||
// generic, tech-agnostic baseline check. It is a positive allowlist: anything not
|
||||
// recognised as generic is left for the caller to mark unmapped.
|
||||
func classifyBaseline(info templateInfo) (Tier, bool) {
|
||||
// tech-agnostic "generic" buckets (e.g. vulnerabilities/generic/* holds
|
||||
// cors/sqli/xss probes that apply to any target) qualify regardless of
|
||||
// category. The logs/files/backups subdir heuristic was removed: it leaked
|
||||
// product-specific exposures (bitrix, magento, oracle-ebs, ...) into baseline.
|
||||
if info.Subdir == "generic" {
|
||||
return TierThorough, true
|
||||
}
|
||||
if _, ok := baselineCategories[info.Category]; !ok {
|
||||
return "", false
|
||||
}
|
||||
// takeovers are wholesale host-agnostic.
|
||||
if info.Category == "takeovers" {
|
||||
return TierBalanced, true
|
||||
}
|
||||
for _, tag := range info.Tags {
|
||||
if _, ok := genericBaselineTags[normToken(tag)]; ok {
|
||||
return TierBalanced, true
|
||||
}
|
||||
}
|
||||
idl := strings.ToLower(info.ID)
|
||||
for _, sub := range genericBaselineIDSubstrings {
|
||||
if strings.Contains(idl, sub) {
|
||||
return TierBalanced, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
package techgraph
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Build walks the templates root and produces the tech-graph artifact.
|
||||
func Build(root string) (*Graph, error) {
|
||||
root = filepath.Clean(root)
|
||||
g := &Graph{
|
||||
Version: ArtifactVersion,
|
||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
Techs: map[string]*Tech{},
|
||||
Synonyms: map[string]string{},
|
||||
}
|
||||
|
||||
infos, hash, parseErr, err := collect(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g.Stats.ParseError = parseErr
|
||||
g.SourceHash = fmt.Sprintf("sha256:%x", hash)
|
||||
|
||||
var deferred []templateInfo
|
||||
|
||||
// Pass 1: build tech nodes from authoritative signals (detection anchors,
|
||||
// cpe/product, platform). Everything else is deferred so it can be resolved
|
||||
// against the fully-built node set in pass 2.
|
||||
for _, info := range infos {
|
||||
g.Stats.Total++
|
||||
|
||||
// Excluded categories never enter detection or the fire-set. Gating
|
||||
// this before detection classification keeps opt-in categories such as
|
||||
// cloud out of the automatic-scan detector list.
|
||||
if _, ok := excludedCategories[info.Category]; ok {
|
||||
g.Stats.Excluded++
|
||||
continue
|
||||
}
|
||||
if isDetection(info) && isDetectionSeverity(info) {
|
||||
id := techID(info.Vendor, info.Product)
|
||||
if id != "" {
|
||||
g.anchorTech(id, info)
|
||||
}
|
||||
g.Detection = append(g.Detection, DetectionEntry{ID: info.ID, Tech: id})
|
||||
g.Stats.Detection++
|
||||
continue
|
||||
}
|
||||
if p, ok := detectPlatform(info); ok {
|
||||
g.addDependent(p, info, "platform")
|
||||
g.Stats.Dependents++
|
||||
g.Stats.DependentsPlat++
|
||||
continue
|
||||
}
|
||||
if id := techID(info.Vendor, info.Product); id != "" {
|
||||
g.addDependent(id, info, "cpe")
|
||||
g.Stats.Dependents++
|
||||
g.Stats.DependentsCPE++
|
||||
continue
|
||||
}
|
||||
deferred = append(deferred, info)
|
||||
}
|
||||
|
||||
// Pass 2: resolve deferred templates to a known tech BEFORE considering
|
||||
// baseline. Order: directory grouping -> tag -> id token. Only genuinely
|
||||
// generic leftovers become baseline; the rest are unmapped (runtime fallback).
|
||||
aliasIdx := g.buildAliasIndex()
|
||||
for _, info := range deferred {
|
||||
if id, ok := g.resolveToken(info.Subdir, aliasIdx); ok {
|
||||
g.addDependent(id, info, "dir")
|
||||
g.Stats.Dependents++
|
||||
g.Stats.DependentsDir++
|
||||
continue
|
||||
}
|
||||
if id, ok := reconcileByTag(g, info, aliasIdx); ok {
|
||||
g.addDependent(id, info, "tag")
|
||||
g.Stats.Dependents++
|
||||
g.Stats.DependentsTag++
|
||||
continue
|
||||
}
|
||||
if id, ok := reconcileByID(g, info, aliasIdx); ok {
|
||||
g.addDependent(id, info, "id")
|
||||
g.Stats.Dependents++
|
||||
g.Stats.DependentsID++
|
||||
continue
|
||||
}
|
||||
if tier, ok := classifyBaseline(info); ok {
|
||||
g.Baseline = append(g.Baseline, BaselineEntry{
|
||||
ID: info.ID, Category: info.Category, Severity: info.Severity, Tier: tier,
|
||||
})
|
||||
g.Stats.Baseline++
|
||||
continue
|
||||
}
|
||||
g.Stats.Unmapped++
|
||||
}
|
||||
|
||||
// ship validated seed synonyms in the artifact so the runtime selector (which
|
||||
// only reads the artifact) can resolve acronyms like aem -> adobe:experience_manager.
|
||||
for alias, id := range seedSynonyms {
|
||||
if _, ok := g.Techs[id]; ok {
|
||||
g.Synonyms[alias] = id
|
||||
}
|
||||
}
|
||||
|
||||
g.Stats.Techs = len(g.Techs)
|
||||
g.sortStable()
|
||||
return g, nil
|
||||
}
|
||||
|
||||
// collect walks the tree once and returns parsed infos plus a content hash.
|
||||
func collect(root string) ([]templateInfo, []byte, int, error) {
|
||||
var infos []templateInfo
|
||||
var parseErr int
|
||||
hasher := sha256.New()
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() {
|
||||
if skipDir(d.Name()) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(path, ".yaml") && !strings.HasSuffix(path, ".yml") {
|
||||
return nil
|
||||
}
|
||||
info, ok := extractTemplate(path, root)
|
||||
if !ok {
|
||||
parseErr++
|
||||
return nil
|
||||
}
|
||||
if _, err := fmt.Fprintf(hasher, "%s|%s|%s|%s\n", info.ID, info.Vendor, info.Product, info.CPE); err != nil {
|
||||
return err
|
||||
}
|
||||
infos = append(infos, info)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
sort.Slice(infos, func(i, j int) bool { return infos[i].ID < infos[j].ID })
|
||||
return infos, hasher.Sum(nil), parseErr, nil
|
||||
}
|
||||
|
||||
// anchorTech creates/updates a tech node's metadata without adding a fireable
|
||||
// template (used by detector templates that define a tech).
|
||||
func (g *Graph) anchorTech(id string, info templateInfo) *Tech {
|
||||
tech, ok := g.Techs[id]
|
||||
if !ok {
|
||||
tech = &Tech{ID: id, Vendor: normToken(info.Vendor), Product: normToken(info.Product), CPE: info.CPE}
|
||||
g.Techs[id] = tech
|
||||
}
|
||||
if tech.CPE == "" && info.CPE != "" {
|
||||
tech.CPE = info.CPE
|
||||
}
|
||||
mergeAliases(tech, aliasesFor(info.Vendor, info.Product, info.Tags))
|
||||
for _, a := range tech.Aliases {
|
||||
if existing, ok := g.Synonyms[a]; !ok || existing == id {
|
||||
g.Synonyms[a] = id
|
||||
}
|
||||
}
|
||||
return tech
|
||||
}
|
||||
|
||||
// addDependent registers a fireable template under its canonical tech.
|
||||
//
|
||||
// Only "cpe" attachments contribute aliases (the template's own vendor/product
|
||||
// genuinely define the tech). Platform/dir/tag/id attachments must NOT pollute
|
||||
// the node's aliases with the sub-product's tokens (e.g. a wp-plugin's name must
|
||||
// not become a wordpress alias), which would corrupt reconciliation.
|
||||
func (g *Graph) addDependent(id string, info templateInfo, source string) {
|
||||
var tech *Tech
|
||||
if source == "cpe" {
|
||||
tech = g.anchorTech(id, info)
|
||||
} else {
|
||||
tech = g.ensureTech(id, info)
|
||||
}
|
||||
ref := TemplateRef{ID: info.ID, Category: info.Category, Severity: info.Severity, Source: source}
|
||||
if source == "platform" || source == "dir" {
|
||||
ref.Product = normToken(info.Product)
|
||||
}
|
||||
tech.Templates = append(tech.Templates, ref)
|
||||
}
|
||||
|
||||
// ensureTech returns an existing node, or creates a bare one without merging the
|
||||
// template's aliases (used for non-cpe attachments).
|
||||
func (g *Graph) ensureTech(id string, info templateInfo) *Tech {
|
||||
if tech, ok := g.Techs[id]; ok {
|
||||
return tech
|
||||
}
|
||||
vendor, product := "", ""
|
||||
if v, p, ok := splitTechID(id); ok {
|
||||
vendor, product = v, p
|
||||
}
|
||||
tech := &Tech{ID: id, Vendor: vendor, Product: product}
|
||||
g.Techs[id] = tech
|
||||
return tech
|
||||
}
|
||||
|
||||
// splitTechID reverses techID for bare node creation.
|
||||
func splitTechID(id string) (vendor, product string, ok bool) {
|
||||
if i := strings.IndexByte(id, ':'); i > 0 {
|
||||
return id[:i], id[i+1:], true
|
||||
}
|
||||
return "", id, true
|
||||
}
|
||||
|
||||
// buildAliasIndex maps a single alias/product token to a unique tech id. Tokens
|
||||
// that resolve to more than one tech, are too short, or are generic are dropped
|
||||
// so tag reconciliation stays conservative.
|
||||
func (g *Graph) buildAliasIndex() map[string]string {
|
||||
counts := map[string]map[string]struct{}{}
|
||||
add := func(tok, id string) {
|
||||
tok = normToken(tok)
|
||||
if len(tok) < 5 || genericTokens[tok] {
|
||||
return
|
||||
}
|
||||
if counts[tok] == nil {
|
||||
counts[tok] = map[string]struct{}{}
|
||||
}
|
||||
counts[tok][id] = struct{}{}
|
||||
}
|
||||
for id, tech := range g.Techs {
|
||||
add(id, id)
|
||||
add(tech.Product, id)
|
||||
for _, a := range tech.Aliases {
|
||||
add(a, id)
|
||||
}
|
||||
}
|
||||
idx := map[string]string{}
|
||||
for tok, ids := range counts {
|
||||
if len(ids) == 1 {
|
||||
for id := range ids {
|
||||
idx[tok] = id
|
||||
}
|
||||
}
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
// resolveToken maps a single token to a tech id via the seed synonym table
|
||||
// first (validated against existing nodes), then the alias index.
|
||||
func (g *Graph) resolveToken(tok string, idx map[string]string) (string, bool) {
|
||||
tok = normToken(tok)
|
||||
if tok == "" || genericTokens[tok] {
|
||||
return "", false
|
||||
}
|
||||
if id, ok := seedSynonyms[tok]; ok {
|
||||
if _, exists := g.Techs[id]; exists {
|
||||
return id, true
|
||||
}
|
||||
}
|
||||
if id, ok := idx[tok]; ok {
|
||||
return id, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// reconcileByTag attaches a product-less template to a known tech when its tags
|
||||
// resolve to exactly one unique tech node.
|
||||
func reconcileByTag(g *Graph, info templateInfo, idx map[string]string) (string, bool) {
|
||||
matches := map[string]struct{}{}
|
||||
for _, tag := range info.Tags {
|
||||
if id, ok := g.resolveToken(tag, idx); ok {
|
||||
matches[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(matches) == 1 {
|
||||
for id := range matches {
|
||||
return id, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// reconcileByID attaches a template when its id tokens resolve to exactly one
|
||||
// unique tech node (e.g. "aem-jcr-exposure" -> adobe:experience_manager).
|
||||
//
|
||||
// This is the riskiest signal (id tokens are common English words), so it is
|
||||
// guarded: a token must be >=6 chars AND also appear in the template name, which
|
||||
// requires the product to be the template's actual subject rather than an
|
||||
// incidental id suffix like "-dashboard"/"-gateway".
|
||||
func reconcileByID(g *Graph, info templateInfo, idx map[string]string) (string, bool) {
|
||||
name := strings.ToLower(info.Name)
|
||||
matches := map[string]struct{}{}
|
||||
for _, tok := range strings.Split(strings.ToLower(info.ID), "-") {
|
||||
if len(tok) < 6 || !strings.Contains(name, tok) {
|
||||
continue
|
||||
}
|
||||
if id, ok := g.resolveToken(tok, idx); ok {
|
||||
matches[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(matches) == 1 {
|
||||
for id := range matches {
|
||||
return id, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func mergeAliases(tech *Tech, aliases []string) {
|
||||
have := map[string]struct{}{}
|
||||
for _, a := range tech.Aliases {
|
||||
have[a] = struct{}{}
|
||||
}
|
||||
for _, a := range aliases {
|
||||
if a == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := have[a]; ok {
|
||||
continue
|
||||
}
|
||||
have[a] = struct{}{}
|
||||
tech.Aliases = append(tech.Aliases, a)
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Graph) sortStable() {
|
||||
for _, tech := range g.Techs {
|
||||
sort.Strings(tech.Aliases)
|
||||
sort.Slice(tech.Templates, func(i, j int) bool {
|
||||
return tech.Templates[i].ID < tech.Templates[j].ID
|
||||
})
|
||||
}
|
||||
sort.Slice(g.Baseline, func(i, j int) bool { return g.Baseline[i].ID < g.Baseline[j].ID })
|
||||
sort.Slice(g.Detection, func(i, j int) bool { return g.Detection[i].ID < g.Detection[j].ID })
|
||||
}
|
||||
|
||||
// WriteFile serialises the graph as indented JSON.
|
||||
func (g *Graph) WriteFile(path string) error {
|
||||
data, err := json.MarshalIndent(g, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, append(data, '\n'), 0o644)
|
||||
}
|
||||
|
||||
func skipDir(name string) bool {
|
||||
switch name {
|
||||
case ".git", ".github", "helpers", "profiles", "workflows":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package techgraph
|
||||
|
||||
import "strings"
|
||||
|
||||
// seedSynonyms maps common acronyms/aliases to a canonical tech id. This is the
|
||||
// curated seed of the synonym table that the offline AI step later expands.
|
||||
// Targets are validated against existing nodes at resolve time, so a stale entry
|
||||
// simply has no effect.
|
||||
var seedSynonyms = map[string]string{
|
||||
"aem": "adobe:experience_manager",
|
||||
"coldfusion": "adobe:coldfusion",
|
||||
"iis": "microsoft:iis",
|
||||
"k8s": "kubernetes:kubernetes",
|
||||
"springboot": "vmware:spring_boot",
|
||||
"spring-boot": "vmware:spring_boot",
|
||||
"wls": "oracle:weblogic_server",
|
||||
"weblogic": "oracle:weblogic_server",
|
||||
"adfs": "microsoft:active_directory_federation_services",
|
||||
// platform roots: ambiguous in the alias index (collide with sub-product
|
||||
// nodes like keydatas:wordpress) so pin them to the canonical platform node.
|
||||
"wordpress": "wordpress",
|
||||
"drupal": "drupal",
|
||||
"joomla": "joomla",
|
||||
"magento": "magento",
|
||||
}
|
||||
|
||||
// genericTokens are tags/aliases that are not products and must never be used as
|
||||
// a tag/id/dir-reconciliation key (they would incorrectly attach templates to random
|
||||
// techs). This list intentionally includes common English words that also happen
|
||||
// to be CPE product names (dashboard, gateway, console, ...), platform/OS tokens
|
||||
// (linux, windows, android, ...), and category labels (generic, audit, ...),
|
||||
// since those were the dominant source of false attachments.
|
||||
var genericTokens = map[string]bool{
|
||||
// vuln classes / scan vocabulary
|
||||
"cve": true, "cves": true, "vuln": true, "vulnerability": true, "tech": true,
|
||||
"detect": true, "detection": true, "panel": true, "panels": true, "login": true, "cms": true,
|
||||
"exposure": true, "exposures": true, "misconfig": true, "misconfiguration": true,
|
||||
"rce": true, "lfi": true, "xss": true, "sqli": true, "ssrf": true, "ssti": true,
|
||||
"redirect": true, "disclosure": true, "default": true, "auth": true, "bypass": true,
|
||||
"injection": true, "traversal": true, "config": true, "files": true, "file": true,
|
||||
"backup": true, "takeover": true, "oast": true, "intrusive": true, "unauth": true,
|
||||
"router": true, "iot": true, "network": true, "http": true, "json": true, "api": true,
|
||||
"server": true, "service": true, "framework": true, "plugin": true, "theme": true,
|
||||
"webserver": true, "database": true, "cloud": true, "devops": true, "tools": true,
|
||||
"edb": true, "kev": true, "packetstorm": true, "seclists": true, "osint": true,
|
||||
// category labels
|
||||
"generic": true, "audit": true, "miscellaneous": true, "misc": true, "enumeration": true,
|
||||
"vulnerabilities": true, "exposed": true, "keys": true, "secret": true, "secrets": true,
|
||||
// platform / OS / cloud-provider tokens (not products)
|
||||
"linux": true, "windows": true, "macos": true, "unix": true, "android": true, "ios": true,
|
||||
"aws": true, "azure": true, "gcp": true, "alibaba": true, "kubernetes": true, "k8s": true,
|
||||
"docker": true, "mobile": true,
|
||||
// common English words that collide with product names
|
||||
"dashboard": true, "gateway": true, "path": true, "console": true, "firewall": true,
|
||||
"monitor": true, "control": true, "permissions": true, "oauth": true, "analytics": true,
|
||||
"enterprise": true, "report": true, "reports": true, "help": true, "printer": true,
|
||||
"core": true, "commerce": true, "anchor": true, "nova": true, "element": true,
|
||||
"webclient": true, "memcached": true, "manager": true, "management": true, "admin": true,
|
||||
"client": true, "agent": true, "portal": true, "gateway2": true, "status": true,
|
||||
"user": true, "users": true, "account": true, "token": true, "tokens": true,
|
||||
"web": true, "app": true, "application": true, "system": true, "platform": true,
|
||||
}
|
||||
|
||||
// normToken lowercases and trims a vendor/product token. CPE uses '_' for
|
||||
// spaces; we keep it as-is so the id stays stable and machine-joinable.
|
||||
func normToken(s string) string {
|
||||
s = strings.ToLower(strings.TrimSpace(s))
|
||||
// CPE wildcards / NA markers are not real tokens.
|
||||
if s == "*" || s == "-" {
|
||||
return ""
|
||||
}
|
||||
// URLs / paths / multi-word strings are not clean product tokens (guards
|
||||
// against junk nodes like "https://apisix.apache.org").
|
||||
if strings.ContainsAny(s, "/ ") {
|
||||
return ""
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// techID builds the canonical technology id from vendor/product. When vendor is
|
||||
// empty or equal to product we collapse to a single token to avoid duplicate
|
||||
// nodes like "wordpress:wordpress" vs "wordpress".
|
||||
func techID(vendor, product string) string {
|
||||
vendor = normToken(vendor)
|
||||
product = normToken(product)
|
||||
switch {
|
||||
case product == "":
|
||||
return ""
|
||||
case vendor == "" || vendor == product:
|
||||
return product
|
||||
default:
|
||||
return vendor + ":" + product
|
||||
}
|
||||
}
|
||||
|
||||
// aliasesFor returns alias tokens that runtime detection (wappalyzer names,
|
||||
// tags) might surface for a tech, so they can be reconciled to the canonical id.
|
||||
// Pillar 1 derives these from product/vendor tokens; AI augments this offline.
|
||||
func aliasesFor(vendor, product string, tags []string) []string {
|
||||
seen := map[string]struct{}{}
|
||||
var out []string
|
||||
add := func(s string) {
|
||||
s = normToken(s)
|
||||
if s == "" {
|
||||
return
|
||||
}
|
||||
if _, ok := seen[s]; ok {
|
||||
return
|
||||
}
|
||||
seen[s] = struct{}{}
|
||||
out = append(out, s)
|
||||
}
|
||||
// Only whole product/vendor tokens are safe aliases. Splitting on separators
|
||||
// produces ambiguous fragments (manager, server, token, web, ...) that cause
|
||||
// false tag/id reconciliation, so we deliberately avoid it.
|
||||
add(product)
|
||||
if vendor != product {
|
||||
add(vendor)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
// Command techgraph-gen builds the offline tech-graph.json artifact from a local
|
||||
// nuclei-templates checkout. See pkg/protocols/common/automaticscan/AUTOMATIC_SCAN_V2.md.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan/techgraph"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var (
|
||||
templates = flag.String("templates", "", "path to nuclei-templates directory (required)")
|
||||
output = flag.String("output", "tech-graph.json", "output artifact path")
|
||||
htmlOut = flag.String("html", "", "also write a standalone visualizer HTML with data inlined")
|
||||
)
|
||||
flag.Parse()
|
||||
|
||||
if *templates == "" {
|
||||
fmt.Fprintln(os.Stderr, "error: -templates is required")
|
||||
flag.Usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
graph, err := techgraph.Build(*templates)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: build failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := graph.WriteFile(*output); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: write failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if *htmlOut != "" {
|
||||
html, err := techgraph.StandaloneHTML(graph)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: html render failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.WriteFile(*htmlOut, []byte(html), 0o644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: html write failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("standalone visualizer written to %s\n", *htmlOut)
|
||||
}
|
||||
|
||||
s := graph.Stats
|
||||
fmt.Printf("tech-graph written to %s\n", *output)
|
||||
fmt.Printf(" templates: %d (parse-skipped %d)\n", s.Total, s.ParseError)
|
||||
fmt.Printf(" dependents: %d (cpe %d, platform %d, dir %d, tag %d, id %d) across %d techs\n",
|
||||
s.Dependents, s.DependentsCPE, s.DependentsPlat, s.DependentsDir, s.DependentsTag, s.DependentsID, s.Techs)
|
||||
fmt.Printf(" baseline: %d\n", s.Baseline)
|
||||
fmt.Printf(" detection: %d\n", s.Detection)
|
||||
fmt.Printf(" excluded: %d\n", s.Excluded)
|
||||
fmt.Printf(" unmapped: %d\n", s.Unmapped)
|
||||
fmt.Printf(" source: %s\n", graph.SourceHash)
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package techgraph
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// embeddedGraph is the prebuilt tech-graph artifact shipped with nuclei. Run the
|
||||
// techgraph-gen command to regenerate it against a nuclei-templates checkout.
|
||||
//
|
||||
//go:embed tech-graph.json
|
||||
var embeddedGraph []byte
|
||||
|
||||
//go:embed visualizer.html
|
||||
var visualizerHTML string
|
||||
|
||||
// Embedded returns the tech-graph artifact compiled into the binary.
|
||||
func Embedded() (*Graph, error) {
|
||||
g := &Graph{}
|
||||
if err := json.Unmarshal(embeddedGraph, g); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
|
||||
// EmbeddedRaw returns the raw embedded artifact bytes.
|
||||
func EmbeddedRaw() []byte { return embeddedGraph }
|
||||
|
||||
// StandaloneHTML returns the visualizer with the given graph inlined, so it can
|
||||
// be opened directly in a browser without a server or sidecar file.
|
||||
func StandaloneHTML(g *Graph) (string, error) {
|
||||
data, err := json.Marshal(g)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
inject := "<script>window.__TECHGRAPH__=" + string(data) + ";</script>\n</head>"
|
||||
return strings.Replace(visualizerHTML, "</head>", inject, 1), nil
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package techgraph
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// templateInfo is the subset of a nuclei template we need for graph building.
|
||||
// We deliberately avoid the full template parser: only the info block is read,
|
||||
// which keeps the generator fast and dependency-light over ~13k files.
|
||||
type templateInfo struct {
|
||||
ID string
|
||||
Name string
|
||||
Severity string
|
||||
Category string // derived from directory (cves, exposed-panels, ...)
|
||||
Subdir string // directory under the category (e.g. "aem" in misconfiguration/aem/)
|
||||
Tags []string
|
||||
Vendor string
|
||||
Product string
|
||||
CPE string
|
||||
}
|
||||
|
||||
// rawTemplate mirrors the YAML structure for lenient decoding.
|
||||
type rawTemplate struct {
|
||||
ID string `yaml:"id"`
|
||||
Info struct {
|
||||
Name string `yaml:"name"`
|
||||
Severity string `yaml:"severity"`
|
||||
Tags any `yaml:"tags"`
|
||||
Metadata map[string]any `yaml:"metadata"`
|
||||
Classification struct {
|
||||
CPE string `yaml:"cpe"`
|
||||
} `yaml:"classification"`
|
||||
} `yaml:"info"`
|
||||
}
|
||||
|
||||
// extractTemplate reads a single template file and returns its info subset.
|
||||
// ok is false when the file is not a parseable template (e.g. workflow, helper).
|
||||
func extractTemplate(path, root string) (templateInfo, bool) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return templateInfo{}, false
|
||||
}
|
||||
var raw rawTemplate
|
||||
if err := yaml.Unmarshal(data, &raw); err != nil {
|
||||
return templateInfo{}, false
|
||||
}
|
||||
if raw.ID == "" {
|
||||
return templateInfo{}, false
|
||||
}
|
||||
|
||||
category, subdir := derivePath(path, root)
|
||||
info := templateInfo{
|
||||
ID: raw.ID,
|
||||
Name: raw.Info.Name,
|
||||
Severity: strings.ToLower(strings.TrimSpace(raw.Info.Severity)),
|
||||
Category: category,
|
||||
Subdir: subdir,
|
||||
Tags: coerceStringSlice(raw.Info.Tags),
|
||||
CPE: strings.TrimSpace(raw.Info.Classification.CPE),
|
||||
}
|
||||
|
||||
if raw.Info.Metadata != nil {
|
||||
info.Vendor = firstNonEmpty(coerceStringSlice(raw.Info.Metadata["vendor"]))
|
||||
info.Product = firstNonEmpty(coerceStringSlice(raw.Info.Metadata["product"]))
|
||||
}
|
||||
|
||||
// Prefer CPE for vendor/product when metadata is missing.
|
||||
if cv, cp, ok := parseCPE(info.CPE); ok {
|
||||
if info.Vendor == "" {
|
||||
info.Vendor = cv
|
||||
}
|
||||
if info.Product == "" {
|
||||
info.Product = cp
|
||||
}
|
||||
}
|
||||
return info, true
|
||||
}
|
||||
|
||||
// derivePath returns the nuclei category and the immediate subdirectory under
|
||||
// it, e.g. http/misconfiguration/aem/x.yaml -> ("misconfiguration", "aem").
|
||||
// The subdir is often a product grouping and is used as a tech signal.
|
||||
func derivePath(path, root string) (category, subdir string) {
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
rel = path
|
||||
}
|
||||
parts := strings.Split(filepath.ToSlash(rel), "/")
|
||||
switch {
|
||||
case len(parts) >= 4:
|
||||
// protocol/category/subdir/.../file
|
||||
return parts[1], parts[2]
|
||||
case len(parts) == 3:
|
||||
return parts[1], ""
|
||||
case len(parts) == 2:
|
||||
return parts[0], ""
|
||||
default:
|
||||
return "", ""
|
||||
}
|
||||
}
|
||||
|
||||
// parseCPE extracts vendor and product from CPE 2.3 (cpe:2.3:a:vendor:product:..)
|
||||
// or CPE 2.2 URI (cpe:/a:vendor:product:..) forms.
|
||||
func parseCPE(cpe string) (vendor, product string, ok bool) {
|
||||
cpe = strings.TrimSpace(cpe)
|
||||
if cpe == "" {
|
||||
return "", "", false
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(cpe, "cpe:2.3:"):
|
||||
parts := strings.Split(cpe, ":")
|
||||
// cpe:2.3:part:vendor:product:...
|
||||
if len(parts) >= 5 {
|
||||
return normToken(parts[3]), normToken(parts[4]), true
|
||||
}
|
||||
case strings.HasPrefix(cpe, "cpe:/"):
|
||||
parts := strings.Split(strings.TrimPrefix(cpe, "cpe:/"), ":")
|
||||
// part:vendor:product:...
|
||||
if len(parts) >= 3 {
|
||||
return normToken(parts[1]), normToken(parts[2]), true
|
||||
}
|
||||
}
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
// coerceStringSlice normalises tags/metadata values that may be a string, a
|
||||
// comma-separated string, or a YAML list into a clean slice.
|
||||
func coerceStringSlice(v any) []string {
|
||||
var out []string
|
||||
add := func(s string) {
|
||||
for _, part := range strings.Split(s, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
out = append(out, part)
|
||||
}
|
||||
}
|
||||
}
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
add(t)
|
||||
case []any:
|
||||
for _, item := range t {
|
||||
if s, ok := item.(string); ok {
|
||||
add(s)
|
||||
}
|
||||
}
|
||||
case []string:
|
||||
for _, s := range t {
|
||||
add(s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func firstNonEmpty(values []string) string {
|
||||
for _, v := range values {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package techgraph
|
||||
|
||||
// platforms are technologies with large plugin/extension ecosystems whose
|
||||
// add-on templates carry the add-on as `product` (often with the platform as
|
||||
// `vendor` or only as a tag). Runtime detection sees the platform, not each
|
||||
// plugin, so those templates must be reachable from the platform node.
|
||||
var platforms = map[string]string{
|
||||
"wordpress": "wordpress",
|
||||
"joomla": "joomla",
|
||||
"drupal": "drupal",
|
||||
"magento": "magento",
|
||||
"prestashop": "prestashop",
|
||||
"opencart": "opencart",
|
||||
"moodle": "moodle",
|
||||
"typo3": "typo3",
|
||||
"dotnetnuke": "dotnetnuke",
|
||||
"concrete5": "concrete5",
|
||||
"shopware": "shopware",
|
||||
"vbulletin": "vbulletin",
|
||||
"mybb": "mybb",
|
||||
"phpbb": "phpbb",
|
||||
}
|
||||
|
||||
// platformTagAliases maps secondary tag tokens to a platform key.
|
||||
var platformTagAliases = map[string]string{
|
||||
"wp": "wordpress",
|
||||
"wp-plugin": "wordpress",
|
||||
"wp-theme": "wordpress",
|
||||
}
|
||||
|
||||
// detectPlatform returns the platform node id when a template belongs to a
|
||||
// platform's add-on ecosystem (and is not the platform core itself).
|
||||
func detectPlatform(info templateInfo) (string, bool) {
|
||||
vendor := normToken(info.Vendor)
|
||||
product := normToken(info.Product)
|
||||
|
||||
if p, ok := platforms[vendor]; ok && product != "" && product != p {
|
||||
return p, true
|
||||
}
|
||||
for _, tag := range info.Tags {
|
||||
t := normToken(tag)
|
||||
if p, ok := platforms[t]; ok {
|
||||
if product != "" && product != p {
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
if p, ok := platformTagAliases[t]; ok {
|
||||
if product != p {
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// isDetection reports whether a template is a phase-1 detector.
|
||||
func isDetection(info templateInfo) bool {
|
||||
for _, tag := range info.Tags {
|
||||
switch normToken(tag) {
|
||||
case "tech", "detect", "favicon":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isDetectionSeverity guards the detection bucket against vuln/exposure templates
|
||||
// that merely carry a "tech"/"detect" tag. Real fingerprint templates are
|
||||
// informational; medium+ severity indicates an actual finding, not detection.
|
||||
func isDetectionSeverity(info templateInfo) bool {
|
||||
switch info.Severity {
|
||||
case "", "info", "low":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
package techgraph
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Selector resolves detected technology signals into a concrete fire-set of
|
||||
// template ids using a tech-graph artifact. It is pure and deterministic.
|
||||
type Selector struct {
|
||||
g *Graph
|
||||
byAlias map[string]string // alias/synonym/id/product -> canonical tech id
|
||||
}
|
||||
|
||||
// NewSelector builds a selector with an alias index for fast signal resolution.
|
||||
func NewSelector(g *Graph) *Selector {
|
||||
s := &Selector{g: g, byAlias: make(map[string]string)}
|
||||
// synonyms first (authoritative), then ids/products/aliases without clobbering.
|
||||
for a, id := range g.Synonyms {
|
||||
s.byAlias[normToken(a)] = id
|
||||
}
|
||||
for id, t := range g.Techs {
|
||||
if _, ok := s.byAlias[normToken(id)]; !ok {
|
||||
s.byAlias[normToken(id)] = id
|
||||
}
|
||||
if t.Product != "" {
|
||||
if _, ok := s.byAlias[normToken(t.Product)]; !ok {
|
||||
s.byAlias[normToken(t.Product)] = id
|
||||
}
|
||||
}
|
||||
for _, a := range t.Aliases {
|
||||
if _, ok := s.byAlias[normToken(a)]; !ok {
|
||||
s.byAlias[normToken(a)] = id
|
||||
}
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Resolve maps detected signal tokens (wappalyzer names, detection tags, cpe
|
||||
// products) to canonical tech ids. Tokens that match no node are returned as
|
||||
// unresolved so the caller can apply tag-overlap fallback.
|
||||
func (s *Selector) Resolve(signals []string) (techIDs, unresolved []string) {
|
||||
seenTech := map[string]struct{}{}
|
||||
seenUnres := map[string]struct{}{}
|
||||
for _, sig := range signals {
|
||||
tok := normToken(sig)
|
||||
if tok == "" || genericTokens[tok] {
|
||||
continue
|
||||
}
|
||||
if id, ok := s.byAlias[tok]; ok {
|
||||
if _, dup := seenTech[id]; !dup {
|
||||
seenTech[id] = struct{}{}
|
||||
techIDs = append(techIDs, id)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, dup := seenUnres[tok]; !dup {
|
||||
seenUnres[tok] = struct{}{}
|
||||
unresolved = append(unresolved, tok)
|
||||
}
|
||||
}
|
||||
sort.Strings(techIDs)
|
||||
sort.Strings(unresolved)
|
||||
return techIDs, unresolved
|
||||
}
|
||||
|
||||
// FireSet returns the precise template ids to execute for the given techs at the
|
||||
// requested coverage tier (dependents + tier-gated baseline).
|
||||
func (s *Selector) FireSet(tier Tier, techIDs []string) []string {
|
||||
ids := map[string]struct{}{}
|
||||
for _, id := range techIDs {
|
||||
if t, ok := s.g.Techs[id]; ok {
|
||||
for _, r := range t.Templates {
|
||||
ids[r.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, b := range s.g.Baseline {
|
||||
if tierIncludesBaseline(tier, b.Tier) {
|
||||
ids[b.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(ids))
|
||||
for id := range ids {
|
||||
out = append(out, id)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// BaselineIDs returns just the tier-gated baseline ids.
|
||||
func (s *Selector) BaselineIDs(tier Tier) []string {
|
||||
var out []string
|
||||
for _, b := range s.g.Baseline {
|
||||
if tierIncludesBaseline(tier, b.Tier) {
|
||||
out = append(out, b.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// tierIncludesBaseline gates baseline membership by coverage tier:
|
||||
// - lean: no baseline (dependents only)
|
||||
// - balanced: balanced baseline only
|
||||
// - thorough: balanced + thorough baseline
|
||||
func tierIncludesBaseline(scan, entry Tier) bool {
|
||||
switch scan {
|
||||
case TierLean:
|
||||
return false
|
||||
case TierThorough:
|
||||
return true
|
||||
default: // balanced
|
||||
return entry == TierBalanced
|
||||
}
|
||||
}
|
||||
|
||||
// NormalizeTier coerces arbitrary input to a valid tier, defaulting to balanced.
|
||||
// Case and surrounding whitespace are ignored.
|
||||
func NormalizeTier(s string) Tier {
|
||||
tier, err := ParseTier(s)
|
||||
if err != nil {
|
||||
return TierBalanced
|
||||
}
|
||||
return tier
|
||||
}
|
||||
|
||||
// ParseTier validates a coverage tier. Empty input defaults to balanced.
|
||||
// Unknown values return an error instead of silently changing behavior.
|
||||
func ParseTier(s string) (Tier, error) {
|
||||
switch Tier(strings.ToLower(strings.TrimSpace(s))) {
|
||||
case "", TierBalanced:
|
||||
return TierBalanced, nil
|
||||
case TierLean:
|
||||
return TierLean, nil
|
||||
case TierThorough:
|
||||
return TierThorough, nil
|
||||
default:
|
||||
return "", fmt.Errorf("invalid automatic scan coverage %q (allowed: lean, balanced, thorough)", s)
|
||||
}
|
||||
}
|
||||
|
||||
// Candidate is a loaded template that the artifact may not know about yet.
|
||||
type Candidate struct {
|
||||
ID string
|
||||
Tags []string
|
||||
Category string
|
||||
}
|
||||
|
||||
// AbsentDependents returns ids of templates the graph does not contain whose
|
||||
// tags resolve to exactly one of the detected techs. Ambiguous tags and
|
||||
// excluded categories are left out, so this only recovers templates added
|
||||
// after the artifact was generated.
|
||||
func (s *Selector) AbsentDependents(detected []string, candidates []Candidate) []string {
|
||||
if s == nil || s.g == nil || len(detected) == 0 || len(candidates) == 0 {
|
||||
return nil
|
||||
}
|
||||
want := make(map[string]struct{}, len(detected))
|
||||
for _, id := range detected {
|
||||
want[id] = struct{}{}
|
||||
}
|
||||
known := s.knownIDs()
|
||||
var out []string
|
||||
for _, c := range candidates {
|
||||
if c.ID == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := known[c.ID]; ok {
|
||||
continue
|
||||
}
|
||||
// An unknown category is skipped: excluded categories such as fuzzing
|
||||
// and cloud must not be pulled in just because the path did not parse.
|
||||
if _, excluded := excludedCategories[c.Category]; excluded || c.Category == "" {
|
||||
continue
|
||||
}
|
||||
matches := map[string]struct{}{}
|
||||
for _, tag := range c.Tags {
|
||||
tok := normToken(tag)
|
||||
if len(tok) < 5 || genericTokens[tok] {
|
||||
continue
|
||||
}
|
||||
id, ok := s.byAlias[tok]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
matches[id] = struct{}{}
|
||||
}
|
||||
if len(matches) != 1 {
|
||||
continue
|
||||
}
|
||||
var only string
|
||||
for id := range matches {
|
||||
only = id
|
||||
}
|
||||
if _, ok := want[only]; !ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, c.ID)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// knownIDs is every template id the artifact already classifies.
|
||||
func (s *Selector) knownIDs() map[string]struct{} {
|
||||
ids := make(map[string]struct{})
|
||||
for _, tech := range s.g.Techs {
|
||||
for _, ref := range tech.Templates {
|
||||
ids[ref.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, b := range s.g.Baseline {
|
||||
ids[b.ID] = struct{}{}
|
||||
}
|
||||
for _, d := range s.g.Detection {
|
||||
ids[d.ID] = struct{}{}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package techgraph
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func testGraph() *Graph {
|
||||
return &Graph{
|
||||
Techs: map[string]*Tech{
|
||||
"wordpress": {ID: "wordpress", Product: "wordpress", Aliases: []string{"wordpress"},
|
||||
Templates: []TemplateRef{{ID: "wp-cve-1"}, {ID: "wp-plugin-1", Source: "platform"}}},
|
||||
"atlassian:confluence": {ID: "atlassian:confluence", Product: "confluence", Aliases: []string{"confluence"},
|
||||
Templates: []TemplateRef{{ID: "confluence-cve-1"}}},
|
||||
},
|
||||
Baseline: []BaselineEntry{
|
||||
{ID: "git-config", Tier: TierBalanced},
|
||||
{ID: "noisy-osinty", Tier: TierThorough},
|
||||
},
|
||||
Synonyms: map[string]string{"aem": "adobe:experience_manager"},
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolve(t *testing.T) {
|
||||
s := NewSelector(testGraph())
|
||||
tech, unres := s.Resolve([]string{"WordPress", "confluence", "php", "cve"})
|
||||
if len(tech) != 2 {
|
||||
t.Fatalf("want 2 techs, got %v", tech)
|
||||
}
|
||||
if len(unres) != 1 || unres[0] != "php" {
|
||||
t.Fatalf("want [php] unresolved, got %v", unres)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFireSetTiers(t *testing.T) {
|
||||
s := NewSelector(testGraph())
|
||||
techs := []string{"wordpress"}
|
||||
|
||||
lean := s.FireSet(TierLean, techs)
|
||||
if want := []string{"wp-cve-1", "wp-plugin-1"}; !reflect.DeepEqual(lean, want) {
|
||||
t.Fatalf("lean: want %v, got %v", want, lean)
|
||||
}
|
||||
bal := s.FireSet(TierBalanced, techs)
|
||||
if want := []string{"git-config", "wp-cve-1", "wp-plugin-1"}; !reflect.DeepEqual(bal, want) {
|
||||
t.Fatalf("balanced: want %v, got %v", want, bal)
|
||||
}
|
||||
tho := s.FireSet(TierThorough, techs)
|
||||
if want := []string{"git-config", "noisy-osinty", "wp-cve-1", "wp-plugin-1"}; !reflect.DeepEqual(tho, want) {
|
||||
t.Fatalf("thorough: want %v, got %v", want, tho)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeTier(t *testing.T) {
|
||||
for in, want := range map[string]Tier{"lean": TierLean, " THOROUGH ": TierThorough, "": TierBalanced, "x": TierBalanced} {
|
||||
if got := NormalizeTier(in); got != want {
|
||||
t.Fatalf("NormalizeTier(%q)=%q want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTier(t *testing.T) {
|
||||
if _, err := ParseTier("not-a-tier"); err == nil {
|
||||
t.Fatal("expected error for unknown tier")
|
||||
}
|
||||
got, err := ParseTier(" Lean ")
|
||||
if err != nil || got != TierLean {
|
||||
t.Fatalf("ParseTier(Lean)=%q %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAbsentDependents(t *testing.T) {
|
||||
s := NewSelector(testGraph())
|
||||
candidates := []Candidate{
|
||||
{ID: "new-wp", Tags: []string{"wordpress"}, Category: "cves"},
|
||||
{ID: "ambiguous", Tags: []string{"wordpress", "confluence"}, Category: "cves"},
|
||||
{ID: "fuzzed", Tags: []string{"wordpress"}, Category: "fuzzing"},
|
||||
{ID: "wp-cve-1", Tags: []string{"wordpress"}, Category: "cves"},
|
||||
{ID: "short", Tags: []string{"wp"}, Category: "cves"},
|
||||
{ID: "other-tech", Tags: []string{"wordpress"}, Category: "cves"},
|
||||
}
|
||||
got := s.AbsentDependents([]string{"wordpress"}, candidates[:5])
|
||||
if want := []string{"new-wp"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("want %v, got %v", want, got)
|
||||
}
|
||||
if got := s.AbsentDependents([]string{"atlassian:confluence"}, candidates[5:]); len(got) != 0 {
|
||||
t.Fatalf("wordpress tag must not attach to confluence, got %v", got)
|
||||
}
|
||||
if got := s.AbsentDependents([]string{"wordpress"}, []Candidate{{ID: "no-path", Tags: []string{"wordpress"}}}); len(got) != 0 {
|
||||
t.Fatalf("unknown category must not be added, got %v", got)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,84 @@
|
||||
// Package techgraph builds the offline technology->template mapping artifact
|
||||
// (tech-graph.json) consumed by automatic scan v2. See AUTOMATIC_SCAN_V2.md.
|
||||
package techgraph
|
||||
|
||||
// ArtifactVersion is the schema version of the emitted artifact.
|
||||
const ArtifactVersion = 1
|
||||
|
||||
// Tier gates how aggressively a template is selected at runtime.
|
||||
type Tier string
|
||||
|
||||
const (
|
||||
TierLean Tier = "lean"
|
||||
TierBalanced Tier = "balanced"
|
||||
TierThorough Tier = "thorough"
|
||||
)
|
||||
|
||||
// Graph is the top-level tech-graph artifact.
|
||||
type Graph struct {
|
||||
Version int `json:"version"`
|
||||
GeneratedAt string `json:"generated_at"`
|
||||
SourceHash string `json:"source_hash,omitempty"`
|
||||
Techs map[string]*Tech `json:"techs"`
|
||||
Baseline []BaselineEntry `json:"baseline"`
|
||||
Detection []DetectionEntry `json:"detection"`
|
||||
Synonyms map[string]string `json:"synonyms"`
|
||||
Stats Stats `json:"stats"`
|
||||
}
|
||||
|
||||
// DetectionEntry is a phase-1 detector template (tagged tech/detect/favicon).
|
||||
// Running these fingerprints the target before selection.
|
||||
type DetectionEntry struct {
|
||||
ID string `json:"id"`
|
||||
Tech string `json:"tech,omitempty"` // canonical tech this detector anchors, if any
|
||||
}
|
||||
|
||||
// Tech is a single canonical technology and the templates that depend on it.
|
||||
type Tech struct {
|
||||
ID string `json:"id"`
|
||||
Vendor string `json:"vendor,omitempty"`
|
||||
Product string `json:"product,omitempty"`
|
||||
CPE string `json:"cpe,omitempty"`
|
||||
Aliases []string `json:"aliases,omitempty"`
|
||||
Templates []TemplateRef `json:"templates"`
|
||||
}
|
||||
|
||||
// TemplateRef references a template that depends on a tech.
|
||||
type TemplateRef struct {
|
||||
ID string `json:"id"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Severity string `json:"severity,omitempty"`
|
||||
// Product is the original sub-product when attached to a platform node via
|
||||
// platform grouping (e.g. plugin "wp-optimize" under platform "wordpress").
|
||||
Product string `json:"product,omitempty"`
|
||||
// Source records how the template was attached: "cpe", "platform", "tag".
|
||||
Source string `json:"source,omitempty"`
|
||||
// VersionRange is reserved for the AI version-parsing milestone. Empty means
|
||||
// "fire regardless of detected version".
|
||||
VersionRange string `json:"version_range,omitempty"`
|
||||
}
|
||||
|
||||
// BaselineEntry is a tech-agnostic template that runs on every applicable target.
|
||||
type BaselineEntry struct {
|
||||
ID string `json:"id"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Severity string `json:"severity,omitempty"`
|
||||
Tier Tier `json:"tier"`
|
||||
}
|
||||
|
||||
// Stats summarises a build run.
|
||||
type Stats struct {
|
||||
Total int `json:"total"`
|
||||
Dependents int `json:"dependents"`
|
||||
DependentsCPE int `json:"dependents_cpe"`
|
||||
DependentsPlat int `json:"dependents_platform"`
|
||||
DependentsDir int `json:"dependents_dir"`
|
||||
DependentsTag int `json:"dependents_tag"`
|
||||
DependentsID int `json:"dependents_id"`
|
||||
Baseline int `json:"baseline"`
|
||||
Detection int `json:"detection"`
|
||||
Excluded int `json:"excluded"`
|
||||
Unmapped int `json:"unmapped"`
|
||||
Techs int `json:"techs"`
|
||||
ParseError int `json:"parse_error"`
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>nuclei tech-graph visualizer</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0c0f17; --panel: #141926; --panel2: #1b2233; --line: #263149;
|
||||
--txt: #e6edf6; --muted: #8b97ad; --accent: #5b8cff;
|
||||
--cpe:#36c98a; --platform:#b07cff; --dir:#3bc4c4; --tag:#5b8cff; --id:#f0a23c;
|
||||
--crit:#ff5470; --high:#ff8a3d; --med:#f5c451; --low:#5b8cff; --info:#7d8aa3;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body { margin:0; background:var(--bg); color:var(--txt);
|
||||
font:14px/1.5 ui-sans-serif,system-ui,-apple-system,"Segoe UI",Roboto,sans-serif; }
|
||||
header { padding:18px 24px; border-bottom:1px solid var(--line);
|
||||
display:flex; align-items:center; gap:16px; flex-wrap:wrap;
|
||||
background:linear-gradient(180deg,#10141f,#0c0f17); position:sticky; top:0; z-index:5; }
|
||||
h1 { font-size:16px; margin:0; font-weight:600; letter-spacing:.3px; }
|
||||
h1 span { color:var(--accent); }
|
||||
.meta { color:var(--muted); font-size:12px; }
|
||||
.btn { background:var(--panel2); color:var(--txt); border:1px solid var(--line);
|
||||
padding:7px 13px; border-radius:8px; cursor:pointer; font-size:13px; }
|
||||
.btn:hover { border-color:var(--accent); }
|
||||
main { padding:20px 24px; max-width:1240px; margin:0 auto; }
|
||||
.cards { display:grid; grid-template-columns:repeat(auto-fit,minmax(140px,1fr)); gap:12px; margin-bottom:18px; }
|
||||
.card { background:var(--panel); border:1px solid var(--line); border-radius:12px; padding:14px 16px; }
|
||||
.card .n { font-size:24px; font-weight:700; }
|
||||
.card .l { color:var(--muted); font-size:12px; text-transform:uppercase; letter-spacing:.5px; }
|
||||
.srcbar { display:flex; height:10px; border-radius:6px; overflow:hidden; margin:4px 0 16px; border:1px solid var(--line); }
|
||||
.srcbar div { height:100%; }
|
||||
.legend { display:flex; gap:14px; flex-wrap:wrap; font-size:12px; color:var(--muted); margin-bottom:18px; }
|
||||
.legend i { display:inline-block; width:10px; height:10px; border-radius:3px; margin-right:5px; vertical-align:middle; }
|
||||
.tabs { display:flex; gap:6px; border-bottom:1px solid var(--line); margin-bottom:16px; }
|
||||
.tab { padding:9px 15px; cursor:pointer; color:var(--muted); border-bottom:2px solid transparent; }
|
||||
.tab.active { color:var(--txt); border-bottom-color:var(--accent); }
|
||||
.toolbar { display:flex; gap:10px; margin-bottom:14px; flex-wrap:wrap; align-items:center; }
|
||||
input[type=search], select { background:var(--panel2); border:1px solid var(--line); color:var(--txt);
|
||||
padding:8px 12px; border-radius:8px; font-size:13px; }
|
||||
input[type=search] { flex:1; min-width:220px; }
|
||||
.tech { background:var(--panel); border:1px solid var(--line); border-radius:10px; margin-bottom:8px; overflow:hidden; }
|
||||
.tech > summary { padding:11px 15px; cursor:pointer; display:flex; align-items:center; gap:12px; list-style:none; }
|
||||
.tech > summary::-webkit-details-marker { display:none; }
|
||||
.tech .id { font-weight:600; }
|
||||
.tech .cnt { margin-left:auto; color:var(--muted); font-size:12px; background:var(--panel2);
|
||||
padding:2px 9px; border-radius:20px; }
|
||||
.cpe { color:var(--muted); font-size:11px; font-family:ui-monospace,monospace; }
|
||||
.tpls { border-top:1px solid var(--line); padding:6px 15px 12px; }
|
||||
.row { display:flex; align-items:center; gap:8px; padding:5px 0; border-bottom:1px dashed #1e2740; font-size:13px; }
|
||||
.row:last-child { border-bottom:none; }
|
||||
.row .tid { font-family:ui-monospace,monospace; font-size:12px; }
|
||||
.badge { font-size:10px; padding:1px 7px; border-radius:20px; text-transform:uppercase; letter-spacing:.4px; font-weight:600; }
|
||||
.b-cpe{background:rgba(54,201,138,.16);color:var(--cpe)} .b-platform{background:rgba(176,124,255,.16);color:var(--platform)}
|
||||
.b-dir{background:rgba(59,196,196,.16);color:var(--dir)} .b-tag{background:rgba(91,140,255,.16);color:var(--tag)}
|
||||
.b-id{background:rgba(240,162,60,.16);color:var(--id)}
|
||||
.sev { width:7px; height:7px; border-radius:50%; flex:none; }
|
||||
.s-critical{background:var(--crit)} .s-high{background:var(--high)} .s-medium{background:var(--med)}
|
||||
.s-low{background:var(--low)} .s-info{background:var(--info)} .s-unknown{background:#444}
|
||||
.cat { color:var(--muted); font-size:11px; }
|
||||
.drop { border:2px dashed var(--line); border-radius:14px; padding:60px; text-align:center; color:var(--muted); }
|
||||
.drop.hot { border-color:var(--accent); color:var(--txt); }
|
||||
.muted { color:var(--muted); }
|
||||
.hidden { display:none; }
|
||||
.alias { color:var(--muted); font-size:11px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>nuclei <span>tech-graph</span> visualizer</h1>
|
||||
<span class="meta" id="meta"></span>
|
||||
<button class="btn" id="pick" style="margin-left:auto">Load tech-graph.json</button>
|
||||
<input type="file" id="file" accept="application/json,.json" class="hidden" />
|
||||
</header>
|
||||
<main>
|
||||
<div id="empty" class="drop">
|
||||
Drop <b>tech-graph.json</b> here, or click <b>Load</b>.<br/>
|
||||
<span class="muted">Generate it with <code>techgraph-gen -templates <dir> -output tech-graph.json</code></span>
|
||||
</div>
|
||||
|
||||
<div id="app" class="hidden">
|
||||
<div class="cards" id="cards"></div>
|
||||
<div class="srcbar" id="srcbar"></div>
|
||||
<div class="legend" id="legend"></div>
|
||||
|
||||
<div class="tabs">
|
||||
<div class="tab active" data-tab="techs">Technologies</div>
|
||||
<div class="tab" data-tab="baseline">Baseline</div>
|
||||
<div class="tab" data-tab="detection">Detection</div>
|
||||
<div class="tab" data-tab="synonyms">Synonyms</div>
|
||||
</div>
|
||||
|
||||
<div class="toolbar">
|
||||
<input type="search" id="q" placeholder="Search tech, template id, alias, cpe…" />
|
||||
<select id="sort">
|
||||
<option value="count">Sort: most templates</option>
|
||||
<option value="name">Sort: name</option>
|
||||
</select>
|
||||
<select id="srcFilter">
|
||||
<option value="">Source: all</option>
|
||||
<option value="cpe">cpe</option><option value="platform">platform</option>
|
||||
<option value="dir">dir</option><option value="tag">tag</option><option value="id">id</option>
|
||||
</select>
|
||||
<span class="muted" id="resultCount"></span>
|
||||
</div>
|
||||
|
||||
<div id="view"></div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<script>
|
||||
const SEV = ["critical","high","medium","low","info"];
|
||||
const SRC = ["cpe","platform","dir","tag","id"];
|
||||
const SRC_COLOR = {cpe:"--cpe",platform:"--platform",dir:"--dir",tag:"--tag",id:"--id"};
|
||||
let G = null;
|
||||
|
||||
const $ = s => document.querySelector(s);
|
||||
const el = (t,c,h) => { const e=document.createElement(t); if(c)e.className=c; if(h!=null)e.innerHTML=h; return e; };
|
||||
const esc = s => String(s==null?"":s).replace(/[&<>]/g,m=>({"&":"&","<":"<",">":">"}[m]));
|
||||
|
||||
function load(json){
|
||||
G = json;
|
||||
$("#empty").classList.add("hidden");
|
||||
$("#app").classList.remove("hidden");
|
||||
$("#meta").textContent = `v${json.version} · ${json.generated_at||""} · ${json.source_hash? json.source_hash.slice(0,18)+"…":""}`;
|
||||
renderCards(); renderTabs(); render();
|
||||
}
|
||||
|
||||
function renderCards(){
|
||||
const s = G.stats || {};
|
||||
const cards = [
|
||||
["templates", s.total], ["dependents", s.dependents], ["techs", s.techs],
|
||||
["baseline", s.baseline], ["detection", s.detection], ["excluded", s.excluded], ["unmapped", s.unmapped],
|
||||
];
|
||||
$("#cards").innerHTML = cards.map(([l,n])=>`<div class="card"><div class="n">${n??0}</div><div class="l">${l}</div></div>`).join("");
|
||||
// source bar from per-source dependent counts
|
||||
const bySrc = {cpe:s.dependents_cpe,platform:s.dependents_platform,dir:s.dependents_dir,tag:s.dependents_tag,id:s.dependents_id};
|
||||
const tot = SRC.reduce((a,k)=>a+(bySrc[k]||0),0)||1;
|
||||
$("#srcbar").innerHTML = SRC.map(k=>`<div style="width:${100*(bySrc[k]||0)/tot}%;background:var(${SRC_COLOR[k]})" title="${k}: ${bySrc[k]||0}"></div>`).join("");
|
||||
$("#legend").innerHTML = SRC.map(k=>`<span><i style="background:var(${SRC_COLOR[k]})"></i>${k} ${bySrc[k]||0}</span>`).join("")
|
||||
+ ` <span style="margin-left:auto" class="muted">dependent attachment source</span>`;
|
||||
}
|
||||
|
||||
let TAB = "techs";
|
||||
function renderTabs(){
|
||||
document.querySelectorAll(".tab").forEach(t=>{
|
||||
t.onclick = ()=>{ TAB=t.dataset.tab; document.querySelectorAll(".tab").forEach(x=>x.classList.toggle("active",x===t)); render(); };
|
||||
});
|
||||
}
|
||||
["#q","#sort","#srcFilter"].forEach(s=>document.addEventListener("input",e=>{ if(e.target.matches(s)) render(); }));
|
||||
|
||||
function render(){
|
||||
const q = $("#q").value.trim().toLowerCase();
|
||||
const v = $("#view"); v.innerHTML="";
|
||||
if(TAB==="techs") return renderTechs(q,v);
|
||||
if(TAB==="baseline") return renderList(G.baseline||[], q, v, b=>[b.id,b.category], b=>rowHTML(b.id,null,b.category,b.severity,b.tier));
|
||||
if(TAB==="detection") return renderList(G.detection||[], q, v, d=>[d.id,d.tech], d=>rowHTML(d.id,null,"detector → "+(d.tech||"—")));
|
||||
if(TAB==="synonyms"){
|
||||
const ents=Object.entries(G.synonyms||{}).filter(([a,t])=>!q||a.includes(q)||t.includes(q)).sort();
|
||||
$("#resultCount").textContent = ents.length+" synonyms";
|
||||
ents.forEach(([a,t])=>{ const r=el("div","row"); r.innerHTML=`<span class="tid">${esc(a)}</span><span class="muted">→</span><span class="badge b-tag">${esc(t)}</span>`; v.appendChild(r); });
|
||||
}
|
||||
}
|
||||
|
||||
function renderTechs(q,v){
|
||||
const srcF = $("#srcFilter").value;
|
||||
let techs = Object.values(G.techs||{});
|
||||
techs = techs.filter(t=>{
|
||||
const tpls = t.templates||[];
|
||||
if(srcF && !tpls.some(x=>x.source===srcF)) return false;
|
||||
if(!q) return true;
|
||||
if(t.id.toLowerCase().includes(q) || (t.cpe||"").toLowerCase().includes(q)) return true;
|
||||
if((t.aliases||[]).some(a=>a.includes(q))) return true;
|
||||
return tpls.some(x=>x.id.toLowerCase().includes(q));
|
||||
});
|
||||
techs.sort($("#sort").value==="name"
|
||||
? (a,b)=>a.id.localeCompare(b.id)
|
||||
: (a,b)=>(b.templates||[]).length-(a.templates||[]).length);
|
||||
$("#resultCount").textContent = `${techs.length} techs · ${techs.reduce((n,t)=>n+(t.templates||[]).length,0)} templates`;
|
||||
const frag=document.createDocumentFragment();
|
||||
for(const t of techs.slice(0,400)){
|
||||
const d=el("details","tech");
|
||||
const tpls=(t.templates||[]);
|
||||
d.innerHTML=`<summary><span class="id">${esc(t.id)}</span>`
|
||||
+(t.cpe?`<span class="cpe">${esc(t.cpe)}</span>`:"")
|
||||
+(t.aliases&&t.aliases.length?`<span class="alias">${esc(t.aliases.slice(0,4).join(", "))}</span>`:"")
|
||||
+`<span class="cnt">${tpls.length}</span></summary>`;
|
||||
const body=el("div","tpls");
|
||||
let list=tpls; if(srcF) list=list.filter(x=>x.source===srcF);
|
||||
if(q) list=list.filter(x=>x.id.toLowerCase().includes(q)||t.id.toLowerCase().includes(q)||(t.aliases||[]).some(a=>a.includes(q)));
|
||||
body.innerHTML = list.slice(0,300).map(x=>rowHTML(x.id,x.source,x.category,x.severity)).join("");
|
||||
d.appendChild(body); frag.appendChild(d);
|
||||
}
|
||||
v.appendChild(frag);
|
||||
if(techs.length>400) v.appendChild(el("div","muted",`Showing first 400 of ${techs.length}. Refine your search.`));
|
||||
}
|
||||
|
||||
function renderList(items,q,v,keys,rowFn){
|
||||
const f=items.filter(it=>!q||keys(it).some(k=>String(k||"").toLowerCase().includes(q)));
|
||||
$("#resultCount").textContent = f.length+" items";
|
||||
v.innerHTML = `<div class="tech"><div class="tpls">`+f.slice(0,1000).map(rowFn).join("")+`</div></div>`;
|
||||
}
|
||||
|
||||
function rowHTML(id,source,category,severity,extra){
|
||||
const sev=(severity||"unknown").toLowerCase();
|
||||
return `<div class="row"><span class="sev s-${SEV.includes(sev)?sev:"unknown"}"></span>`
|
||||
+ `<span class="tid">${esc(id)}</span>`
|
||||
+ (source?`<span class="badge b-${source}">${esc(source)}</span>`:"")
|
||||
+ (category?`<span class="cat">${esc(category)}</span>`:"")
|
||||
+ (extra?`<span class="cat">· ${esc(extra)}</span>`:"")
|
||||
+ `</div>`;
|
||||
}
|
||||
|
||||
// ---- loading: file picker, drag/drop, auto-fetch ----
|
||||
$("#pick").onclick=()=>$("#file").click();
|
||||
$("#file").onchange=e=>{ const f=e.target.files[0]; if(f) f.text().then(t=>load(JSON.parse(t))); };
|
||||
const dz=$("#empty");
|
||||
["dragover","dragenter"].forEach(ev=>dz.addEventListener(ev,e=>{e.preventDefault();dz.classList.add("hot");}));
|
||||
["dragleave","drop"].forEach(ev=>dz.addEventListener(ev,e=>{e.preventDefault();dz.classList.remove("hot");}));
|
||||
dz.addEventListener("drop",e=>{ const f=e.dataTransfer.files[0]; if(f) f.text().then(t=>load(JSON.parse(t))); });
|
||||
if(window.__TECHGRAPH__){ load(window.__TECHGRAPH__); }
|
||||
else { fetch("tech-graph.json").then(r=>r.ok?r.json():Promise.reject()).then(load).catch(()=>{}); }
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -2,6 +2,7 @@ package expressions
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/govaluate"
|
||||
@@ -48,55 +49,75 @@ func EvaluateWithOptions(data string, base map[string]interface{}, options *type
|
||||
return evaluate(data, base, options)
|
||||
}
|
||||
|
||||
// evaluate renders data, the template text, with base, the data values.
|
||||
//
|
||||
// Expressions are located in data before anything is substituted, and each
|
||||
// result is written at its expression's position. The text between
|
||||
// expressions only has its placeholders replaced. Nothing that evaluate
|
||||
// writes is scanned again, so a value or a helper result that looks like an
|
||||
// expression stays literal and never takes another expression's result. On
|
||||
// error, data is returned unchanged.
|
||||
//
|
||||
// Expressions can be simple (base keys) or complex (helper functions and base
|
||||
// keys). Literals such as {{2+2}} are not expressions.
|
||||
func evaluate(data string, base map[string]interface{}, options *types.Options) (string, error) {
|
||||
expressions := FindExpressions(data, marker.ParenthesisOpen, marker.ParenthesisClose, base)
|
||||
|
||||
// replace simple placeholders (key => value) MarkerOpen + key + MarkerClose and General + key + General to value
|
||||
data = replacer.Replace(data, base)
|
||||
|
||||
// expressions can be:
|
||||
// - simple: containing base values keys (variables)
|
||||
// - complex: containing helper functions [ + variables]
|
||||
// literals like {{2+2}} are not considered expressions
|
||||
for _, expression := range expressions {
|
||||
originalExpression := expression
|
||||
// data has already had simple placeholders replaced; keep the same
|
||||
// marker shape for output replacement, but never compile this string.
|
||||
replacedExpression := replacer.Replace(expression, base)
|
||||
expression = replaceStringPlaceholders(expression, base)
|
||||
|
||||
// turns expressions (either helper functions+base values or base values)
|
||||
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions)
|
||||
if err != nil {
|
||||
return data, fmt.Errorf("failed to compile expression %q: %w", originalExpression, err)
|
||||
}
|
||||
|
||||
result, err := dsl.EvalWithOptions(compiled, base, options)
|
||||
if err != nil {
|
||||
return data, fmt.Errorf("failed to evaluate expression %q: %w", originalExpression, err)
|
||||
}
|
||||
|
||||
replacement := result
|
||||
// Preserve unresolved markers only when a helper call would otherwise
|
||||
// hide them from downstream validation. Plain expressions such as
|
||||
// comparisons should evaluate normally.
|
||||
if markers := unresolvedVarMarkers(compiled.Vars(), base); markers != "" {
|
||||
usesFunctions := false
|
||||
for _, token := range compiled.Tokens() {
|
||||
if token.Kind == govaluate.FUNCTION {
|
||||
usesFunctions = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if usesFunctions && ContainsUnresolvedVariables(fmt.Sprint(result)) == nil {
|
||||
replacement = markers
|
||||
}
|
||||
}
|
||||
|
||||
// replace incrementally
|
||||
data = replacer.ReplaceOne(data, replacedExpression, replacement)
|
||||
spans := findExpressionSpans(data, marker.ParenthesisOpen, marker.ParenthesisClose, base)
|
||||
if len(spans) == 0 {
|
||||
return replacer.Replace(data, base), nil
|
||||
}
|
||||
return data, nil
|
||||
|
||||
var (
|
||||
builder strings.Builder
|
||||
last int
|
||||
)
|
||||
|
||||
builder.Grow(len(data))
|
||||
|
||||
for _, span := range spans {
|
||||
result, err := evaluateExpression(span.expression, base, options)
|
||||
if err != nil {
|
||||
return data, err
|
||||
}
|
||||
|
||||
builder.WriteString(replacer.Replace(data[last:span.start], base))
|
||||
builder.WriteString(result)
|
||||
|
||||
last = span.end
|
||||
}
|
||||
|
||||
builder.WriteString(replacer.Replace(data[last:], base))
|
||||
|
||||
return builder.String(), nil
|
||||
}
|
||||
|
||||
// evaluateExpression compiles and evaluates one template-authored expression,
|
||||
// the text between its markers, and returns the result as text.
|
||||
func evaluateExpression(expression string, base map[string]any, options *types.Options) (string, error) {
|
||||
source := replaceStringPlaceholders(expression, base)
|
||||
|
||||
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(source, dsl.HelperFunctions)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to compile expression %q: %w", expression, err)
|
||||
}
|
||||
|
||||
result, err := dsl.EvalWithOptions(compiled, base, options)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to evaluate expression %q: %w", expression, err)
|
||||
}
|
||||
|
||||
// Preserve unresolved markers only when a helper call would otherwise
|
||||
// hide them from downstream validation. Plain expressions such as
|
||||
// comparisons should evaluate normally.
|
||||
if markers := unresolvedVarMarkers(compiled.Vars(), base); markers != "" {
|
||||
usesFunctions := slices.ContainsFunc(compiled.Tokens(), func(token govaluate.ExpressionToken) bool {
|
||||
return token.Kind == govaluate.FUNCTION
|
||||
})
|
||||
if usesFunctions && ContainsUnresolvedVariables(fmt.Sprint(result)) == nil {
|
||||
return markers, nil
|
||||
}
|
||||
}
|
||||
|
||||
return types.ToString(result), nil
|
||||
}
|
||||
|
||||
func replaceStringPlaceholders(expression string, base map[string]interface{}) string {
|
||||
@@ -206,59 +227,63 @@ func EscapeStringValue(value string, quote byte) string {
|
||||
// maxIterations to avoid infinite loop
|
||||
const maxIterations = 250
|
||||
|
||||
// FindExpressions returns the expressions in data, the text between each
|
||||
// matching OpenMarker and CloseMarker pair, in order.
|
||||
func FindExpressions(data, OpenMarker, CloseMarker string, base map[string]interface{}) []string {
|
||||
var (
|
||||
iterations int
|
||||
exps []string
|
||||
)
|
||||
for iterations <= maxIterations {
|
||||
// check if we reached the maximum number of iterations
|
||||
spans := findExpressionSpans(data, OpenMarker, CloseMarker, base)
|
||||
if len(spans) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
iterations++
|
||||
// attempt to find open markers
|
||||
indexOpenMarker := strings.Index(data, OpenMarker)
|
||||
// exits if not found
|
||||
if indexOpenMarker < 0 {
|
||||
exps := make([]string, 0, len(spans))
|
||||
for _, span := range spans {
|
||||
exps = append(exps, span.expression)
|
||||
}
|
||||
|
||||
return exps
|
||||
}
|
||||
|
||||
// expressionSpan is one expression in template text: data[start:end] is the
|
||||
// whole marker and expression is the text between its markers.
|
||||
type expressionSpan struct {
|
||||
start, end int
|
||||
expression string
|
||||
}
|
||||
|
||||
// findExpressionSpans finds the expressions in data and where each one sits.
|
||||
// For each open marker, it takes the nearest close marker that makes the text
|
||||
// between them an expression. When none does, the search resumes right after
|
||||
// the open marker.
|
||||
func findExpressionSpans(data, openMarker, closeMarker string, base map[string]any) []expressionSpan {
|
||||
var spans []expressionSpan
|
||||
|
||||
for offset, iterations := 0, 0; iterations <= maxIterations; iterations++ {
|
||||
openIndex := strings.Index(data[offset:], openMarker)
|
||||
if openIndex < 0 {
|
||||
break
|
||||
}
|
||||
|
||||
indexOpenMarkerOffset := indexOpenMarker + len(OpenMarker)
|
||||
start := offset + openIndex
|
||||
inner := start + len(openMarker)
|
||||
offset = inner
|
||||
|
||||
shouldSearchCloseMarker := true
|
||||
closeMarkerFound := false
|
||||
innerData := data
|
||||
var potentialMatch string
|
||||
var indexCloseMarker, indexCloseMarkerOffset int
|
||||
skip := indexOpenMarkerOffset
|
||||
for shouldSearchCloseMarker {
|
||||
// attempt to find close marker
|
||||
indexCloseMarker = stringsutil.IndexAt(innerData, CloseMarker, skip)
|
||||
// if no close markers are found exit
|
||||
if indexCloseMarker < 0 {
|
||||
shouldSearchCloseMarker = false
|
||||
continue
|
||||
for search := inner; ; {
|
||||
closeIndex := stringsutil.IndexAt(data, closeMarker, search)
|
||||
if closeIndex < 0 {
|
||||
break
|
||||
}
|
||||
indexCloseMarkerOffset = indexCloseMarker + len(CloseMarker)
|
||||
|
||||
potentialMatch = innerData[indexOpenMarkerOffset:indexCloseMarker]
|
||||
if isExpression(potentialMatch, base) {
|
||||
closeMarkerFound = true
|
||||
shouldSearchCloseMarker = false
|
||||
exps = append(exps, potentialMatch)
|
||||
} else {
|
||||
skip = indexCloseMarkerOffset
|
||||
search = closeIndex + len(closeMarker)
|
||||
if expression := data[inner:closeIndex]; isExpression(expression, base) {
|
||||
spans = append(spans, expressionSpan{start: start, end: search, expression: expression})
|
||||
offset = search
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if closeMarkerFound {
|
||||
// move after the close marker
|
||||
data = data[indexCloseMarkerOffset:]
|
||||
} else {
|
||||
// move after the open marker
|
||||
data = data[indexOpenMarkerOffset:]
|
||||
}
|
||||
}
|
||||
return exps
|
||||
|
||||
return spans
|
||||
}
|
||||
|
||||
func isExpression(data string, base map[string]interface{}) bool {
|
||||
|
||||
@@ -109,6 +109,71 @@ func TestEvaluateDoesNotReinterpretResolvedValues(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateReplacesExpressionsInTheirOwnPosition(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// tpl and echo stand for response values that repeat an expression's text.
|
||||
values := map[string]any{
|
||||
"seed": "hello",
|
||||
"tpl": "{{to_upper(seed)}}",
|
||||
"echo": "{{base64('127.0.0.1')}}",
|
||||
"Host": "127.0.0.1",
|
||||
}
|
||||
|
||||
items := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "resolved value echoes a later expression",
|
||||
input: "[{{tpl}}] {{to_upper(seed)}}",
|
||||
expected: "[{{to_upper(seed)}}] HELLO",
|
||||
},
|
||||
{
|
||||
name: "general marker value echoes a later expression",
|
||||
input: "[§tpl§] {{to_upper(seed)}}",
|
||||
expected: "[{{to_upper(seed)}}] HELLO",
|
||||
},
|
||||
{
|
||||
name: "resolved value echoes an expression with substituted arguments",
|
||||
input: "{{echo}} {{base64('{{Host}}')}}",
|
||||
expected: "{{base64('127.0.0.1')}} MTI3LjAuMC4x",
|
||||
},
|
||||
{
|
||||
name: "helper result echoes a later expression",
|
||||
input: "{{concat('{{', 'to_upper(seed)', '}}')}} {{to_upper(seed)}}",
|
||||
expected: "{{to_upper(seed)}} HELLO",
|
||||
},
|
||||
{
|
||||
name: "repeated expression is evaluated in each position",
|
||||
input: "{{to_upper(seed)}}-{{to_upper(seed)}}",
|
||||
expected: "HELLO-HELLO",
|
||||
},
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
t.Run(item.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
value, err := Evaluate(item.input, values)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, item.expected, value)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateDoesNotResolveGeneralMarkersFromResolvedValues(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
value, err := Evaluate("Authorization: {{body}}", map[string]any{
|
||||
"body": "§secret_token§",
|
||||
"secret_token": "top-secret-cia-mi6-kgb-mossad-classified",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "Authorization: §secret_token§", value)
|
||||
}
|
||||
|
||||
func TestEvaluateDoesNotExecuteHelpersFromResolvedValues(t *testing.T) {
|
||||
var calls int
|
||||
|
||||
@@ -206,6 +271,18 @@ func TestEvaluateReturnsErrorForInvalidTemplateExpression(t *testing.T) {
|
||||
require.ErrorContains(t, err, `failed to evaluate expression "base64()"`)
|
||||
}
|
||||
|
||||
func TestEvaluateReturnsInputOnError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := "{{secret_token}} {{base64()}}"
|
||||
|
||||
value, err := Evaluate(input, map[string]any{
|
||||
"secret_token": "top-secret-cia-mi6-kgb-mossad-classified",
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.Equal(t, input, value)
|
||||
}
|
||||
|
||||
func TestEvaluateErrorDoesNotLeakResolvedValues(t *testing.T) {
|
||||
_, err := Evaluate("{{base64('{{secret_token}}', 'extra')}}", map[string]interface{}{
|
||||
"secret_token": "top-secret-cia-mi6-kgb-mossad-classified",
|
||||
|
||||
@@ -115,6 +115,26 @@ func ContainsVariablesWithIgnoreList(skipNames map[string]interface{}, items ...
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReferencesVariables reports whether data contains a {{...}} marker that
|
||||
// refers to a variable in names, either alone or inside an expression. It
|
||||
// parses the marker but does not evaluate it.
|
||||
func ReferencesVariables(data string, names map[string]interface{}) bool {
|
||||
for _, match := range unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) {
|
||||
expr, err := govaluate.NewEvaluableExpressionWithFunctions(match[1], dsl.HelperFunctions)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, name := range expr.Vars() {
|
||||
if _, ok := names[name]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func hasLiteralsOnly(data string) bool {
|
||||
expr, err := govaluate.NewEvaluableExpressionWithFunctions(data, dsl.HelperFunctions)
|
||||
if err != nil {
|
||||
|
||||
@@ -44,6 +44,26 @@ func TestUnresolvedVariablesCheck(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferencesVariables(t *testing.T) {
|
||||
names := map[string]interface{}{"Hostname": "", "Port": ""}
|
||||
|
||||
tests := []struct {
|
||||
data string
|
||||
want bool
|
||||
}{
|
||||
{data: "user@{{Hostname}}", want: true},
|
||||
{data: "{{to_lower(Hostname)}}:{{Port}}", want: true},
|
||||
{data: "{{Hostname_suffix}}", want: false},
|
||||
{data: "{{secret}}-{{to_upper(seed)}}", want: false},
|
||||
{data: "{{7*7}}", want: false},
|
||||
{data: "Hostname", want: false},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
require.Equal(t, test.want, ReferencesVariables(test.data, names), test.data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnresolvedVariablesCheckDoesNotExecuteHelpers(t *testing.T) {
|
||||
var calls int
|
||||
withTestHelperFunction(t, "test_side_effect", func(args ...interface{}) (interface{}, error) {
|
||||
|
||||
@@ -35,5 +35,7 @@ func Init(options *types.Options) error {
|
||||
}
|
||||
|
||||
func Close(executionId string) {
|
||||
dnsclientpool.Close(executionId)
|
||||
rdapclientpool.Close(executionId)
|
||||
protocolstate.Close(executionId)
|
||||
}
|
||||
@@ -3,27 +3,186 @@ package replacer
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/fasttemplate"
|
||||
convUtil "github.com/projectdiscovery/utils/conversion"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/marker"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
)
|
||||
|
||||
// Replace replaces placeholders in template with values on the fly.
|
||||
// Replace substitutes each {{key}} and §key§ placeholder in template whose key
|
||||
// is in values, scanning template once from left to right.
|
||||
//
|
||||
// Values are data: Replace never scans an inserted value for placeholders, so
|
||||
// a value cannot pull in another value, alone or together with the template
|
||||
// text around it. Text that does not form a known placeholder is kept as is.
|
||||
func Replace(template string, values map[string]interface{}) string {
|
||||
valuesMap := make(map[string]interface{}, len(values))
|
||||
for k, v := range values {
|
||||
// fasttemplate copies string values to []byte on every write, so
|
||||
// pass a read-only []byte view instead
|
||||
valuesMap[k] = convUtil.Bytes(types.ToString(v))
|
||||
if len(values) == 0 {
|
||||
return template
|
||||
}
|
||||
replaced := fasttemplate.ExecuteStringStd(template, marker.ParenthesisOpen, marker.ParenthesisClose, valuesMap)
|
||||
final := fasttemplate.ExecuteStringStd(replaced, marker.General, marker.General, valuesMap)
|
||||
return final
|
||||
|
||||
scan := newScanner(template, values)
|
||||
|
||||
start := scan.nextPlaceholder(0)
|
||||
if start < 0 {
|
||||
return template
|
||||
}
|
||||
|
||||
var builder strings.Builder
|
||||
|
||||
builder.Grow(len(template))
|
||||
|
||||
written := 0 // template[:written] is already in builder
|
||||
|
||||
for pos := start; pos >= 0; {
|
||||
value, end, ok := scan.resolve(pos)
|
||||
if !ok {
|
||||
// Not a known placeholder: keep it as text and look again one
|
||||
// byte later, so a placeholder nested in it still resolves.
|
||||
pos = scan.nextPlaceholder(pos + 1)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
builder.WriteString(template[written:pos])
|
||||
builder.WriteString(value)
|
||||
|
||||
written = end
|
||||
pos = scan.nextPlaceholder(end)
|
||||
}
|
||||
|
||||
builder.WriteString(template[written:])
|
||||
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
// Replace replaces one placeholder in template with one value on the fly.
|
||||
// directLookupLimit is the longest candidate key that resolve looks up in
|
||||
// values without first comparing it to the longest key there.
|
||||
const directLookupLimit = 32
|
||||
|
||||
// scanner finds the placeholders of one template. Replace asks for positions
|
||||
// that never decrease, so each finder searches every byte at most once. A
|
||||
// candidate key longer than every key in values is rejected without hashing
|
||||
// it. Together, these keep Replace linear in the template length.
|
||||
type scanner struct {
|
||||
template string
|
||||
values map[string]any
|
||||
longestKey int // length of the longest key in values; -1 until needed
|
||||
braceOpen, braceClose finder
|
||||
generalOpen, generalClose finder
|
||||
}
|
||||
|
||||
func newScanner(template string, values map[string]any) scanner {
|
||||
return scanner{
|
||||
template: template,
|
||||
values: values,
|
||||
longestKey: -1,
|
||||
braceOpen: newFinder(marker.ParenthesisOpen),
|
||||
braceClose: newFinder(marker.ParenthesisClose),
|
||||
generalOpen: newFinder(marker.General),
|
||||
generalClose: newFinder(marker.General),
|
||||
}
|
||||
}
|
||||
|
||||
// nextPlaceholder returns the index of the next {{ or § at or after from that
|
||||
// can still be closed, or -1 when there is none. Once no closing marker of a
|
||||
// kind is left, no later opening marker of that kind can close.
|
||||
func (s *scanner) nextPlaceholder(from int) int {
|
||||
brace, general := -1, -1
|
||||
|
||||
if !s.braceClose.exhausted() {
|
||||
brace = s.braceOpen.index(s.template, from)
|
||||
}
|
||||
|
||||
if !s.generalClose.exhausted() {
|
||||
general = s.generalOpen.index(s.template, from)
|
||||
}
|
||||
|
||||
if brace < 0 || (general >= 0 && general < brace) {
|
||||
return general
|
||||
}
|
||||
|
||||
return brace
|
||||
}
|
||||
|
||||
// resolve returns the value of the placeholder that starts at index start,
|
||||
// and the index just past it, when its key is in values.
|
||||
func (s *scanner) resolve(start int) (string, int, bool) {
|
||||
open, closing := marker.ParenthesisOpen, &s.braceClose
|
||||
if strings.HasPrefix(s.template[start:], marker.General) {
|
||||
open, closing = marker.General, &s.generalClose
|
||||
}
|
||||
|
||||
keyStart := start + len(open)
|
||||
|
||||
keyEnd := closing.index(s.template, keyStart)
|
||||
if keyEnd < 0 {
|
||||
return "", 0, false
|
||||
}
|
||||
|
||||
key := s.template[keyStart:keyEnd]
|
||||
if len(key) > directLookupLimit && len(key) > s.longestKeyLen() {
|
||||
return "", 0, false
|
||||
}
|
||||
|
||||
value, ok := s.values[key]
|
||||
if !ok {
|
||||
return "", 0, false
|
||||
}
|
||||
|
||||
return types.ToString(value), keyEnd + len(closing.needle), true
|
||||
}
|
||||
|
||||
// longestKeyLen returns the length of the longest key in values.
|
||||
func (s *scanner) longestKeyLen() int {
|
||||
if s.longestKey < 0 {
|
||||
s.longestKey = 0
|
||||
for key := range s.values {
|
||||
s.longestKey = max(s.longestKey, len(key))
|
||||
}
|
||||
}
|
||||
|
||||
return s.longestKey
|
||||
}
|
||||
|
||||
// finder finds one marker in a template. Its callers pass positions that
|
||||
// never decrease, so it keeps the last match until the scan passes it, and a
|
||||
// failed search answers every later call.
|
||||
type finder struct {
|
||||
needle string
|
||||
next int // last match, or -1 when none is left
|
||||
searched bool
|
||||
}
|
||||
|
||||
func newFinder(needle string) finder {
|
||||
return finder{needle: needle, next: -1, searched: false}
|
||||
}
|
||||
|
||||
// index returns the index of the first needle at or after from, or -1.
|
||||
func (f *finder) index(template string, from int) int {
|
||||
if f.searched && (f.next < 0 || f.next >= from) {
|
||||
return f.next
|
||||
}
|
||||
|
||||
f.searched = true
|
||||
|
||||
f.next = strings.Index(template[from:], f.needle)
|
||||
if f.next >= 0 {
|
||||
f.next += from
|
||||
}
|
||||
|
||||
return f.next
|
||||
}
|
||||
|
||||
// exhausted reports whether a search found no match left in the template.
|
||||
func (f *finder) exhausted() bool {
|
||||
return f.searched && f.next < 0
|
||||
}
|
||||
|
||||
// ReplaceOne replaces the first {{key}} and the first §key§ in template with
|
||||
// value.
|
||||
//
|
||||
// Deprecated: ReplaceOne finds the placeholder by searching template, so when
|
||||
// template already holds substituted values it can replace text that a value
|
||||
// introduced instead of the placeholder the caller meant. Render by position
|
||||
// with Replace instead.
|
||||
func ReplaceOne(template string, key string, value interface{}) string {
|
||||
data := replaceOneWithMarkers(template, key, value, marker.ParenthesisOpen, marker.ParenthesisClose)
|
||||
return replaceOneWithMarkers(data, key, value, marker.General, marker.General)
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package replacer_test
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
|
||||
)
|
||||
|
||||
func BenchmarkReplace(b *testing.B) {
|
||||
// An HTTP request renders with about 60 values, enough that map lookups
|
||||
// hash their keys.
|
||||
values := map[string]any{"Hostname": "example.com", "token": "abc"}
|
||||
for i := range 60 {
|
||||
values["var"+strconv.Itoa(i)] = i
|
||||
}
|
||||
|
||||
header := "POST /?t={{token}} HTTP/1.1\r\nHost: {{Hostname}}\r\n\r\n"
|
||||
|
||||
for _, item := range []struct {
|
||||
name string
|
||||
template string
|
||||
}{
|
||||
{name: "raw request", template: header},
|
||||
{name: "64KB JSON body", template: header + strings.Repeat(`{"k": "v", "n": [1, 2]} `, 2730)},
|
||||
{name: "64KB non-ASCII body", template: header + strings.Repeat("Привет, мир! ", 2730)},
|
||||
{name: "64KB unclosed markers", template: strings.Repeat(`{{`, 32768)},
|
||||
{name: "64KB markers closed at the end", template: strings.Repeat(`{`, 65536) + `}}`},
|
||||
} {
|
||||
b.Run(item.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
|
||||
for b.Loop() {
|
||||
replacer.Replace(item.template, values)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -73,6 +73,24 @@ func TestReplacerReplace(t *testing.T) {
|
||||
values: map[string]interface{}{"var1": "first variable", "var2": "second variable"},
|
||||
expected: "first variable and first variable and another second variable",
|
||||
},
|
||||
{
|
||||
name: "Extra braces around a placeholder",
|
||||
template: "${{{num}}*{{{num}}",
|
||||
values: map[string]any{"num": "7"},
|
||||
expected: "${7*{7",
|
||||
},
|
||||
{
|
||||
name: "Unknown general marker before a known one",
|
||||
template: "§unknown§var1§",
|
||||
values: map[string]any{"var1": "variable 1"},
|
||||
expected: "§unknownvariable 1",
|
||||
},
|
||||
{
|
||||
name: "Unclosed markers",
|
||||
template: "{{{{ §§ {{var1}",
|
||||
values: map[string]any{"var1": "variable 1"},
|
||||
expected: "{{{{ §§ {{var1}",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
@@ -82,6 +100,48 @@ func TestReplacerReplace(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplacerReplaceDoesNotRescanValues(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
values := map[string]any{
|
||||
"secret": "leaked",
|
||||
"token": "leaked",
|
||||
"general": "§secret§",
|
||||
"parenthesis": "{{secret}}",
|
||||
"prefix": "§",
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
template string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "general marker inside a parenthesis value",
|
||||
template: "{{general}}",
|
||||
expected: "§secret§",
|
||||
},
|
||||
{
|
||||
name: "parenthesis marker inside a general value",
|
||||
template: "§parenthesis§",
|
||||
expected: "{{secret}}",
|
||||
},
|
||||
{
|
||||
name: "value completes a marker with the template text after it",
|
||||
template: "{{prefix}}token§",
|
||||
expected: "§token§",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
require.Equal(t, test.expected, Replace(test.template, values))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplacerReplaceOne(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool"
|
||||
"github.com/projectdiscovery/retryabledns"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
)
|
||||
|
||||
@@ -87,7 +86,7 @@ type Request struct {
|
||||
generator *generators.PayloadGenerator
|
||||
|
||||
CompiledOperators *operators.Operators `yaml:"-" json:"-"`
|
||||
dnsClient *retryabledns.Client
|
||||
dnsClient *dnsclientpool.Client
|
||||
options *protocols.ExecutorOptions
|
||||
|
||||
// cache any variables that may be needed for operation.
|
||||
@@ -205,7 +204,7 @@ func resolverHost(resolver string) string {
|
||||
return r
|
||||
}
|
||||
|
||||
func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadata map[string]interface{}) (*retryabledns.Client, error) {
|
||||
func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadata map[string]interface{}) (*dnsclientpool.Client, error) {
|
||||
dnsClientOptions := &dnsclientpool.Configuration{
|
||||
Retries: request.Retries,
|
||||
Proxy: options.Options.AliveSocksProxy,
|
||||
|
||||
@@ -12,9 +12,9 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
clientPool *mapsutil.SyncLockMap[string, *retryabledns.Client]
|
||||
clientPool *mapsutil.SyncLockMap[string, *Client]
|
||||
|
||||
normalClient *retryabledns.Client
|
||||
normalClient *Client
|
||||
m sync.Mutex
|
||||
)
|
||||
|
||||
@@ -46,20 +46,21 @@ func resolversFor(options *types.Options) []string {
|
||||
return defaultResolvers
|
||||
}
|
||||
|
||||
func cachedClient(resolvers []string) (*retryabledns.Client, error) {
|
||||
func cachedClient(resolvers []string) (*Client, error) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
if clientPool == nil {
|
||||
clientPool = mapsutil.NewSyncLockMap[string, *retryabledns.Client]()
|
||||
clientPool = mapsutil.NewSyncLockMap[string, *Client]()
|
||||
}
|
||||
key := strings.Join(resolvers, ",")
|
||||
if client, ok := clientPool.Get(key); ok {
|
||||
return client, nil
|
||||
}
|
||||
client, err := retryabledns.New(resolvers, 1)
|
||||
dnsClient, err := retryabledns.New(resolvers, 1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client := &Client{Client: dnsClient}
|
||||
_ = clientPool.Set(key, client)
|
||||
return client, nil
|
||||
}
|
||||
@@ -88,10 +89,13 @@ func (c *Configuration) Hash() string {
|
||||
}
|
||||
|
||||
// Get creates or gets a client for the protocol based on custom configuration
|
||||
func Get(options *types.Options, configuration *Configuration) (*retryabledns.Client, error) {
|
||||
func Get(options *types.Options, configuration *Configuration) (*Client, error) {
|
||||
if (configuration.Retries <= 1) && len(configuration.Resolvers) == 0 {
|
||||
return cachedClient(resolversFor(options))
|
||||
}
|
||||
if len(options.InternalResolversList) == 0 && hasDohResolver(configuration.Resolvers) {
|
||||
return getPolicyClient(options, configuration)
|
||||
}
|
||||
hash := configuration.Hash()
|
||||
if client, ok := clientPool.Get(hash); ok {
|
||||
return client, nil
|
||||
@@ -103,7 +107,7 @@ func Get(options *types.Options, configuration *Configuration) (*retryabledns.Cl
|
||||
} else if len(configuration.Resolvers) > 0 {
|
||||
resolvers = configuration.Resolvers
|
||||
}
|
||||
client, err := retryabledns.NewWithOptions(retryabledns.Options{
|
||||
dnsClient, err := retryabledns.NewWithOptions(retryabledns.Options{
|
||||
BaseResolvers: resolvers,
|
||||
MaxRetries: configuration.Retries,
|
||||
Proxy: options.AliveSocksProxy,
|
||||
@@ -111,6 +115,7 @@ func Get(options *types.Options, configuration *Configuration) (*retryabledns.Cl
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not create dns client")
|
||||
}
|
||||
client := &Client{Client: dnsClient}
|
||||
_ = clientPool.Set(hash, client)
|
||||
|
||||
return client, nil
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
package dnsclientpool
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/retryabledns"
|
||||
"github.com/projectdiscovery/retryabledns/doh"
|
||||
)
|
||||
|
||||
// Client sends the DNS queries of a template. It sends queries for template
|
||||
// DoH resolvers itself, because the retryabledns DoH client dials and follows
|
||||
// redirects without the network policy of the scan.
|
||||
type Client struct {
|
||||
*retryabledns.Client
|
||||
|
||||
resolvers []resolver
|
||||
retries int
|
||||
index uint32
|
||||
doh *doh.Client
|
||||
transport *http.Transport
|
||||
}
|
||||
|
||||
// resolver is one template resolver. url is empty for a resolver that
|
||||
// retryabledns queries over UDP, TCP or DoT.
|
||||
type resolver struct {
|
||||
url string
|
||||
method doh.Method
|
||||
}
|
||||
|
||||
var (
|
||||
// policyClients holds the clients with DoH resolvers of each execution,
|
||||
// because each one dials with the dialer of its execution.
|
||||
policyClients = map[string]map[string]*Client{}
|
||||
policyMu sync.Mutex
|
||||
)
|
||||
|
||||
func hasDohResolver(resolvers []string) bool {
|
||||
for _, r := range resolvers {
|
||||
if strings.HasPrefix(r, "doh:") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// parseDohResolver returns the URL and method of a resolver such as
|
||||
// doh:https://dns.example/dns-query:get, in the same way as retryabledns.
|
||||
func parseDohResolver(r string) (resolver, bool) {
|
||||
if !strings.HasPrefix(r, "doh:") {
|
||||
return resolver{}, false
|
||||
}
|
||||
u := strings.TrimPrefix(r, "doh:")
|
||||
method := doh.MethodPost
|
||||
for _, suffix := range []string{":get", ":post", ":jsonapi"} {
|
||||
if strings.HasSuffix(u, suffix) {
|
||||
u = strings.TrimSuffix(u, suffix)
|
||||
if suffix == ":get" {
|
||||
method = doh.MethodGet
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return resolver{url: u, method: method}, true
|
||||
}
|
||||
|
||||
func getPolicyClient(options *types.Options, configuration *Configuration) (*Client, error) {
|
||||
executionID := options.ExecutionId
|
||||
hash := configuration.Hash()
|
||||
|
||||
policyMu.Lock()
|
||||
defer policyMu.Unlock()
|
||||
if client, ok := policyClients[executionID][hash]; ok {
|
||||
return client, nil
|
||||
}
|
||||
|
||||
client := &Client{retries: configuration.Retries}
|
||||
var networkResolvers []string
|
||||
for _, r := range configuration.Resolvers {
|
||||
if parsed, ok := parseDohResolver(r); ok {
|
||||
client.resolvers = append(client.resolvers, parsed)
|
||||
continue
|
||||
}
|
||||
client.resolvers = append(client.resolvers, resolver{})
|
||||
networkResolvers = append(networkResolvers, r)
|
||||
}
|
||||
// Trace and the non DoH resolvers still go through retryabledns.
|
||||
if len(networkResolvers) == 0 {
|
||||
networkResolvers = defaultResolvers
|
||||
}
|
||||
dnsClient, err := retryabledns.NewWithOptions(retryabledns.Options{
|
||||
BaseResolvers: networkResolvers,
|
||||
MaxRetries: 1,
|
||||
Proxy: options.AliveSocksProxy,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not create dns client")
|
||||
}
|
||||
client.Client = dnsClient
|
||||
|
||||
// The scan dialer checks every address it connects to, so redirects
|
||||
// and names that resolve to a denied address are refused too. It also
|
||||
// goes through the scan proxy when one is set.
|
||||
client.transport = &http.Transport{
|
||||
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
dialers := protocolstate.GetDialersWithId(executionID)
|
||||
if dialers == nil || dialers.Fastdialer == nil {
|
||||
return nil, fmt.Errorf("dialers not initialized for %s", executionID)
|
||||
}
|
||||
return dialers.Fastdialer.Dial(ctx, network, address)
|
||||
},
|
||||
// same as the retryabledns DoH client
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec G402
|
||||
}
|
||||
client.doh = doh.NewWithOptions(doh.Options{
|
||||
HttpClient: &http.Client{
|
||||
Transport: client.transport,
|
||||
Timeout: options.GetTimeouts().HttpTimeout,
|
||||
},
|
||||
})
|
||||
|
||||
if policyClients[executionID] == nil {
|
||||
policyClients[executionID] = map[string]*Client{}
|
||||
}
|
||||
policyClients[executionID][hash] = client
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// Do sends a DNS message to the resolvers in turn until one answers with
|
||||
// NOERROR, like retryabledns.Client.Do.
|
||||
func (c *Client) Do(msg *dns.Msg) (*dns.Msg, error) {
|
||||
if len(c.resolvers) == 0 {
|
||||
return c.Client.Do(msg)
|
||||
}
|
||||
var resp *dns.Msg
|
||||
var err error
|
||||
for i := 0; i < c.retries; i++ {
|
||||
index := atomic.AddUint32(&c.index, 1)
|
||||
r := c.resolvers[index%uint32(len(c.resolvers))]
|
||||
if r.url != "" {
|
||||
resp, err = c.doh.QueryWithDOHMsg(r.method, doh.Resolver{URL: r.url}, msg)
|
||||
} else {
|
||||
resp, err = c.Client.Do(msg)
|
||||
}
|
||||
if err == nil && resp != nil && resp.Rcode == dns.RcodeSuccess {
|
||||
return resp, nil
|
||||
}
|
||||
}
|
||||
return resp, retryabledns.ErrRetriesExceeded
|
||||
}
|
||||
|
||||
// Close drops the clients with DoH resolvers of an execution.
|
||||
func Close(executionID string) {
|
||||
policyMu.Lock()
|
||||
clients := policyClients[executionID]
|
||||
delete(policyClients, executionID)
|
||||
policyMu.Unlock()
|
||||
for _, client := range clients {
|
||||
client.transport.CloseIdleConnections()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package dnsclientpool
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newDohServer answers every DoH query with a TXT record and counts the hits.
|
||||
func newDohServer(t *testing.T, hits *atomic.Int32) *httptest.Server {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
m := new(dns.Msg)
|
||||
m.SetQuestion("example.com.", dns.TypeTXT)
|
||||
m.Response = true
|
||||
rr, err := dns.NewRR(`example.com. 60 IN TXT "doh-answer"`)
|
||||
require.NoError(t, err)
|
||||
m.Answer = append(m.Answer, rr)
|
||||
packed, err := m.Pack()
|
||||
require.NoError(t, err)
|
||||
w.Header().Set("Content-Type", "application/dns-message")
|
||||
_, _ = w.Write(packed)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return server
|
||||
}
|
||||
|
||||
func dohOptions(t *testing.T, excludeTargets []string) *types.Options {
|
||||
t.Helper()
|
||||
options := types.DefaultOptions()
|
||||
options.ExecutionId = t.Name()
|
||||
options.ExcludeTargets = excludeTargets
|
||||
require.NoError(t, protocolstate.Init(options))
|
||||
t.Cleanup(func() { protocolstate.Close(options.ExecutionId) })
|
||||
require.NoError(t, Init(options))
|
||||
return options
|
||||
}
|
||||
|
||||
func txtQuery() *dns.Msg {
|
||||
msg := new(dns.Msg)
|
||||
msg.SetQuestion("example.com.", dns.TypeTXT)
|
||||
return msg
|
||||
}
|
||||
|
||||
func TestDohResolverObeysNetworkPolicy(t *testing.T) {
|
||||
t.Run("denied address", func(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
internal := newDohServer(t, &hits)
|
||||
options := dohOptions(t, []string{"127.0.0.1"})
|
||||
|
||||
client, err := Get(options, &Configuration{Retries: 1, Resolvers: []string{"doh:" + internal.URL + "/dns-query:get"}})
|
||||
require.NoError(t, err)
|
||||
_, _ = client.Do(txtQuery())
|
||||
require.Zero(t, hits.Load(), "DoH query reached a denied address")
|
||||
})
|
||||
|
||||
t.Run("redirect to denied address", func(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
internal := newDohServer(t, &hits)
|
||||
redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, internal.URL+"/internal", http.StatusFound)
|
||||
}))
|
||||
t.Cleanup(redirector.Close)
|
||||
// localhost passes the resolver check through ::1, while 127.0.0.1 is denied
|
||||
_, port, err := net.SplitHostPort(strings.TrimPrefix(redirector.URL, "http://"))
|
||||
require.NoError(t, err)
|
||||
options := dohOptions(t, []string{"127.0.0.1"})
|
||||
require.True(t, protocolstate.IsHostAllowed(options.ExecutionId, "localhost"))
|
||||
|
||||
client, err := Get(options, &Configuration{Retries: 1, Resolvers: []string{"doh:http://localhost:" + port + "/dns-query:get"}})
|
||||
require.NoError(t, err)
|
||||
_, _ = client.Do(txtQuery())
|
||||
require.Zero(t, hits.Load(), "DoH redirect reached a denied address")
|
||||
})
|
||||
|
||||
t.Run("allowed address", func(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
server := newDohServer(t, &hits)
|
||||
options := dohOptions(t, nil)
|
||||
|
||||
client, err := Get(options, &Configuration{Retries: 2, Resolvers: []string{"doh:" + server.URL + "/dns-query:get"}})
|
||||
require.NoError(t, err)
|
||||
resp, err := client.Do(txtQuery())
|
||||
require.NoError(t, err)
|
||||
require.Len(t, resp.Answer, 1)
|
||||
require.Equal(t, int32(1), hits.Load())
|
||||
})
|
||||
}
|
||||
@@ -31,6 +31,11 @@ type Browser struct {
|
||||
// use getHTTPClient to get the http client
|
||||
httpClient *http.Client
|
||||
httpClientOnce *sync.Once
|
||||
|
||||
// use getScreenshotDir to get the screenshot directory
|
||||
screenshotDir string
|
||||
screenshotDirErr error
|
||||
screenshotDirOnce sync.Once
|
||||
}
|
||||
|
||||
// New creates a new nuclei headless browser module
|
||||
@@ -236,6 +241,16 @@ func (b *Browser) getHTTPClient() (*http.Client, error) {
|
||||
return b.httpClient, err
|
||||
}
|
||||
|
||||
// getScreenshotDir returns the directory that holds the screenshots of
|
||||
// templates when local file access is off. It is created once per browser
|
||||
// and kept after Close, so the screenshots stay available.
|
||||
func (b *Browser) getScreenshotDir() (string, error) {
|
||||
b.screenshotDirOnce.Do(func() {
|
||||
b.screenshotDir, b.screenshotDirErr = os.MkdirTemp("", "nuclei-screenshots-*")
|
||||
})
|
||||
return b.screenshotDir, b.screenshotDirErr
|
||||
}
|
||||
|
||||
// Close closes the browser engine
|
||||
//
|
||||
// When connected over CDP, it does NOT close the browsers.
|
||||
|
||||
@@ -43,6 +43,7 @@ type Page struct {
|
||||
variables map[string]interface{}
|
||||
lastActionNavigate *Action
|
||||
cancel context.CancelFunc
|
||||
screenshots int
|
||||
}
|
||||
|
||||
// HistoryData contains the page request/response pairs
|
||||
|
||||
@@ -526,6 +526,13 @@ func (p *Page) RightClickElement(act *Action, out ActionData) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// maxScreenshotsPerPage and maxScreenshotSize bound the disk space that the
|
||||
// screenshots of one page can use when local file access is off.
|
||||
const (
|
||||
maxScreenshotsPerPage = 10
|
||||
maxScreenshotSize = 10 * 1024 * 1024
|
||||
)
|
||||
|
||||
// Screenshot executes screenshot action on a page
|
||||
func (p *Page) Screenshot(act *Action, out ActionData) error {
|
||||
to, err := p.getActionArg(act, "to")
|
||||
@@ -533,13 +540,52 @@ func (p *Page) Screenshot(act *Action, out ActionData) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if to == "" {
|
||||
generatedName := to == ""
|
||||
if generatedName {
|
||||
to = ksuid.New().String()
|
||||
if act.Name != "" {
|
||||
out[act.Name] = to
|
||||
}
|
||||
}
|
||||
|
||||
lfaAllowed := protocolstate.IsLfaAllowed(p.options.Options)
|
||||
if !lfaAllowed {
|
||||
if p.screenshots >= maxScreenshotsPerPage {
|
||||
return errkit.Newf("screenshot limit of %d per page reached, use -allow-local-file-access flag to remove it", maxScreenshotsPerPage)
|
||||
}
|
||||
p.screenshots++
|
||||
}
|
||||
|
||||
var filePath string
|
||||
if lfaAllowed {
|
||||
filePath, err = fileutil.CleanPath(to)
|
||||
if err != nil {
|
||||
return errkit.Newf("could not clean output screenshot path %s", to)
|
||||
}
|
||||
if !strings.HasSuffix(filePath, ".png") {
|
||||
filePath += ".png"
|
||||
}
|
||||
} else {
|
||||
if err := p.isScreenshotPathAllowed(to); err != nil {
|
||||
return err
|
||||
}
|
||||
// Without local file access the template only names a file inside a
|
||||
// directory that nuclei creates, so it cannot choose where files land.
|
||||
baseDir, err := p.instance.browser.getScreenshotDir()
|
||||
if err != nil {
|
||||
return errkit.Wrap(err, "could not create screenshot directory")
|
||||
}
|
||||
filePath = filepath.Join(baseDir, filepath.Clean(to))
|
||||
// Add .png before the containment check, so that a name such as "."
|
||||
// cannot turn into a sibling of the directory.
|
||||
if !strings.HasSuffix(filePath, ".png") {
|
||||
filePath += ".png"
|
||||
}
|
||||
if !isScreenshotPathWithinDirectory(filePath, baseDir) {
|
||||
return ErrLFAccessDenied
|
||||
}
|
||||
}
|
||||
|
||||
var data []byte
|
||||
|
||||
fullpage, err := p.getActionArg(act, "fullpage")
|
||||
@@ -555,23 +601,8 @@ func (p *Page) Screenshot(act *Action, out ActionData) error {
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not take screenshot")
|
||||
}
|
||||
|
||||
to, err = fileutil.CleanPath(to)
|
||||
if err != nil {
|
||||
return errkit.Newf("could not clean output screenshot path %s", to)
|
||||
}
|
||||
|
||||
// Build the final write path (with .png) BEFORE running any containment
|
||||
// gate. Otherwise inputs like "." or a bare directory name pass the gate
|
||||
// against `to` and then the .png suffix moves the actual write outside
|
||||
// cwd (e.g. <cwd>.png is a sibling of cwd and not contained by it).
|
||||
filePath := to
|
||||
if !strings.HasSuffix(filePath, ".png") {
|
||||
filePath += ".png"
|
||||
}
|
||||
|
||||
if err := p.isScreenshotPathAllowed(filePath); err != nil {
|
||||
return err
|
||||
if !lfaAllowed && len(data) > maxScreenshotSize {
|
||||
return errkit.Newf("screenshot of %d bytes is over the limit of %d bytes, use -allow-local-file-access flag to remove it", len(data), maxScreenshotSize)
|
||||
}
|
||||
|
||||
mkdir, err := p.getActionArg(act, "mkdir")
|
||||
@@ -596,25 +627,29 @@ func (p *Page) Screenshot(act *Action, out ActionData) error {
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not write screenshot")
|
||||
}
|
||||
// The generated name alone does not locate a file inside the screenshot directory.
|
||||
if generatedName && act.Name != "" && !lfaAllowed {
|
||||
out[act.Name] = filePath
|
||||
}
|
||||
gologger.Info().Msgf("Screenshot successfully saved at %v\n", filePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// isScreenshotPathAllowed checks the path a template gives for a screenshot.
|
||||
// Without local file access it must be relative and must not contain "..".
|
||||
func (p *Page) isScreenshotPathAllowed(to string) error {
|
||||
if protocolstate.IsLfaAllowed(p.options.Options) {
|
||||
return nil
|
||||
}
|
||||
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
return errkit.Wrap(err, "could not get current working directory")
|
||||
}
|
||||
|
||||
if !isScreenshotPathWithinDirectory(to, cwd) {
|
||||
// writing outside of cwd requires -lfa flag
|
||||
if filepath.IsAbs(to) || filepath.VolumeName(to) != "" || strings.HasPrefix(to, "/") || strings.HasPrefix(to, `\`) {
|
||||
return ErrLFAccessDenied
|
||||
}
|
||||
|
||||
for _, part := range strings.FieldsFunc(to, func(r rune) bool { return r == '/' || r == '\\' }) {
|
||||
if part == ".." {
|
||||
return ErrLFAccessDenied
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -449,7 +449,7 @@ func TestActionScreenshotDeniesSiblingPrefixPathWithoutLFA(t *testing.T) {
|
||||
err = page.isScreenshotPathAllowed(filePath)
|
||||
require.ErrorIs(t, err, ErrLFAccessDenied)
|
||||
|
||||
err = page.isScreenshotPathAllowed(filepath.Join(cwd, "test.png"))
|
||||
err = page.isScreenshotPathAllowed("test.png")
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
@@ -541,8 +541,8 @@ func TestActionScreenshotDeniesPostExtensionEscape(t *testing.T) {
|
||||
"a sibling .png in cwd's parent must be rejected")
|
||||
|
||||
// Sanity: a path inside cwd remains allowed.
|
||||
require.NoError(t, page.isScreenshotPathAllowed(filepath.Join(cwd, "ok.png")),
|
||||
"a path inside cwd must still be allowed")
|
||||
require.NoError(t, page.isScreenshotPathAllowed("ok.png"),
|
||||
"a relative path must still be allowed")
|
||||
}
|
||||
|
||||
func TestActionScreenshotDeniesSymlinkedParentOutsideCWDWithoutLFA(t *testing.T) {
|
||||
@@ -1051,6 +1051,10 @@ func TestMain(m *testing.M) {
|
||||
code := m.Run()
|
||||
|
||||
browser.Close()
|
||||
// screenshots of tests without local file access land here
|
||||
if browser.screenshotDir != "" {
|
||||
_ = os.RemoveAll(browser.screenshotDir)
|
||||
}
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
)
|
||||
|
||||
// chdirTemp moves the test into a fresh working directory and returns it.
|
||||
func chdirTemp(t *testing.T) string {
|
||||
t.Helper()
|
||||
cwd := t.TempDir()
|
||||
originalWd, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.Chdir(cwd))
|
||||
t.Cleanup(func() { require.NoError(t, os.Chdir(originalWd)) })
|
||||
return cwd
|
||||
}
|
||||
|
||||
// runWithoutLFA opens a test page with local file access off and runs the
|
||||
// actions on it.
|
||||
func runWithoutLFA(t *testing.T, actions []*Action) error {
|
||||
t.Helper()
|
||||
_, err := runWithoutLFAData(t, actions)
|
||||
return err
|
||||
}
|
||||
|
||||
// runWithoutLFAData is runWithoutLFA that also returns the action data.
|
||||
func runWithoutLFAData(t *testing.T, actions []*Action) (ActionData, error) {
|
||||
t.Helper()
|
||||
|
||||
// the proxy of the shared browser dials with the empty execution id
|
||||
require.NoError(t, protocolstate.Init(&types.Options{}))
|
||||
opts := &types.Options{ExecutionId: t.Name(), AllowLocalFileAccess: false}
|
||||
require.NoError(t, protocolstate.Init(opts))
|
||||
t.Cleanup(func() { protocolstate.Close(opts.ExecutionId) })
|
||||
|
||||
instance, err := sharedBrowser.NewInstance()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = instance.Close() })
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`<html><body>Nuclei Test Page</body></html>`))
|
||||
}))
|
||||
t.Cleanup(ts.Close)
|
||||
|
||||
input := contextargs.NewWithInput(context.Background(), ts.URL)
|
||||
input.CookieJar, err = cookiejar.New(nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
actions = append([]*Action{
|
||||
{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
|
||||
{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
|
||||
}, actions...)
|
||||
data, page, err := instance.Run(input, actions, nil, &Options{Timeout: 20 * time.Second, Options: opts})
|
||||
if page != nil {
|
||||
page.Close()
|
||||
}
|
||||
return data, err
|
||||
}
|
||||
|
||||
func screenshotAction(args map[string]string) *Action {
|
||||
return &Action{ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: args}
|
||||
}
|
||||
|
||||
func requireNoFiles(t *testing.T, dir string) {
|
||||
t.Helper()
|
||||
var files []string
|
||||
require.NoError(t, filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err == nil && !d.IsDir() {
|
||||
files = append(files, path)
|
||||
}
|
||||
return err
|
||||
}))
|
||||
require.Empty(t, files, "template wrote into the working directory")
|
||||
}
|
||||
|
||||
func TestScreenshotWithoutLFAWritesOnlyToScreenshotDir(t *testing.T) {
|
||||
cwd := chdirTemp(t)
|
||||
name := "shot-" + strconv.FormatInt(time.Now().UnixNano(), 10)
|
||||
err := runWithoutLFA(t, []*Action{
|
||||
screenshotAction(map[string]string{"to": name + "/deep/dir/marker", "mkdir": "true"}),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
requireNoFiles(t, cwd)
|
||||
|
||||
matches, err := filepath.Glob(filepath.Join(os.TempDir(), "nuclei-screenshots-*", name, "deep", "dir", "marker.png"))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, matches, 1, "screenshot is not in the nuclei screenshot directory")
|
||||
}
|
||||
|
||||
func TestScreenshotWithoutLFANamedResultIsFullPath(t *testing.T) {
|
||||
chdirTemp(t)
|
||||
action := screenshotAction(map[string]string{})
|
||||
action.Name = "shot"
|
||||
data, err := runWithoutLFAData(t, []*Action{action})
|
||||
require.NoError(t, err)
|
||||
|
||||
path, ok := data["shot"].(string)
|
||||
require.True(t, ok)
|
||||
require.True(t, filepath.IsAbs(path), "named result %q does not locate the screenshot", path)
|
||||
require.FileExists(t, path)
|
||||
}
|
||||
|
||||
func TestScreenshotWithoutLFARejectsPaths(t *testing.T) {
|
||||
cases := map[string]func(cwd string) string{
|
||||
"absolute inside working directory": func(cwd string) string { return filepath.Join(cwd, "abs") },
|
||||
"parent": func(string) string { return "../outside" },
|
||||
"parent after folder": func(string) string { return "a/../../outside" },
|
||||
}
|
||||
for name, to := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cwd := filepath.Join(chdirTemp(t), "work")
|
||||
require.NoError(t, os.Mkdir(cwd, 0700))
|
||||
require.NoError(t, os.Chdir(cwd))
|
||||
|
||||
err := runWithoutLFA(t, []*Action{screenshotAction(map[string]string{"to": to(cwd), "mkdir": "true"})})
|
||||
require.ErrorIs(t, err, ErrLFAccessDenied)
|
||||
requireNoFiles(t, filepath.Dir(cwd))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScreenshotWithoutLFAIsLimitedPerPage(t *testing.T) {
|
||||
chdirTemp(t)
|
||||
// one more than the limit of 10 screenshots per page
|
||||
actions := make([]*Action, 0, 11)
|
||||
for range 11 {
|
||||
actions = append(actions, screenshotAction(nil))
|
||||
}
|
||||
require.ErrorContains(t, runWithoutLFA(t, actions), "screenshot limit", "more screenshots than the limit were written for one page")
|
||||
}
|
||||
|
||||
func TestScreenshotWithoutLFAIsLimitedInSize(t *testing.T) {
|
||||
chdirTemp(t)
|
||||
noise := `() => { document.body.style.margin='0'; const c=document.createElement('canvas'); c.width=2000; c.height=2000; document.body.appendChild(c); const x=c.getContext('2d'); const d=x.createImageData(2000,2000); for(let i=0;i<d.data.length;i++){d.data[i]=(Math.random()*256)|0; if(i%4==3)d.data[i]=255;} x.putImageData(d,0,0); return 'ok'; }`
|
||||
err := runWithoutLFA(t, []*Action{
|
||||
{ActionType: ActionTypeHolder{ActionType: ActionScript}, Data: map[string]string{"code": noise}},
|
||||
screenshotAction(map[string]string{"fullpage": "true"}),
|
||||
})
|
||||
require.ErrorContains(t, err, "over the limit", "a screenshot over the size limit was written")
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -272,98 +271,19 @@ func (r *requestGenerator) validateSelfContainedHost(host string) error {
|
||||
// selfContained templates do not need/use target data and all values i.e {{Hostname}} , {{BaseURL}} etc are already available
|
||||
// in template . makeSelfContainedRequest parses and creates variables map and then creates corresponding http request or raw request
|
||||
func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, input *contextargs.Context, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
|
||||
isRawRequest := r.request.isRaw()
|
||||
|
||||
optionValues := generators.BuildPayloadFromOptions(r.request.options.Options)
|
||||
values := generators.MergeMaps(optionValues)
|
||||
if r.request.options.HasTemplateCtx(input.MetaInput) {
|
||||
values = generators.MergeMaps(values, r.request.options.GetTemplateCtx(input.MetaInput).GetAll())
|
||||
}
|
||||
values = generators.MergeMaps(values, dynamicValues, payloads)
|
||||
// adds all variables from `variables` section in template
|
||||
scope := r.request.options.NewVariablesScope(optionValues)
|
||||
r.request.options.AddTemplateCtxToVariablesScope(input.MetaInput, scope)
|
||||
scope.AddData(dynamicValues, payloads, r.options.Constants)
|
||||
evaluation := r.request.options.Variables.EvaluateWithInteractshScope(scope, r.request.options.Interactsh)
|
||||
|
||||
variablesMap, interactURLs := evaluation.Values, evaluation.InteractURLs
|
||||
if len(interactURLs) > 0 {
|
||||
r.interactshURLs = append(r.interactshURLs, interactURLs...)
|
||||
if r.request.isRaw() {
|
||||
return r.makeSelfContainedRawRequest(ctx, input, data, payloads, dynamicValues)
|
||||
}
|
||||
|
||||
values = generators.MergeMaps(variablesMap, values)
|
||||
signerVars := GetDefaultSignerVars(r.request.Signature.Value)
|
||||
|
||||
// this will ensure that default signer variables are overwritten by other variables
|
||||
values = generators.MergeMaps(signerVars, values, r.options.Constants)
|
||||
|
||||
for payloadName, payloadValue := range payloads {
|
||||
renderedPayload, err := r.renderText(types.ToString(payloadValue), values)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "could not evaluate payload helper expressions")
|
||||
}
|
||||
|
||||
payloads[payloadName] = renderedPayload
|
||||
values, _, err := r.selfContainedValues(input, payloads, dynamicValues, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
values = generators.MergeMaps(values, payloads, r.options.Constants)
|
||||
|
||||
// priority of variables is as follows (from low to high) for self contained templates
|
||||
// default signer vars < variables < cli vars < dynamic values < payload < constants
|
||||
|
||||
// evaluate request
|
||||
data, err := r.renderText(data, values)
|
||||
data, err = r.renderText(data, values)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "could not evaluate helper expressions")
|
||||
}
|
||||
|
||||
// If the request is a raw request, get the URL from the request
|
||||
// header and use it to make the request.
|
||||
if isRawRequest {
|
||||
// Get the hostname from the URL section to build the request.
|
||||
reader := bufio.NewReader(strings.NewReader(data))
|
||||
read_line:
|
||||
s, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not read request: %w", err)
|
||||
}
|
||||
|
||||
// ignore all annotations
|
||||
if stringsutil.HasPrefixAny(s, "@") {
|
||||
goto read_line
|
||||
}
|
||||
|
||||
parts := strings.Split(s, " ")
|
||||
if len(parts) < 3 {
|
||||
return nil, fmt.Errorf("malformed request supplied")
|
||||
}
|
||||
|
||||
if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil && !r.request.SkipVariablesCheck {
|
||||
return nil, errkit.Newf("unresolved variables `%v` found in request", parts[1])
|
||||
}
|
||||
|
||||
parsed, err := urlutil.ParseURL(parts[1], true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not parse request URL: %w", err)
|
||||
}
|
||||
// self-contained requests choose their own destination URL, so validate
|
||||
// that host against the network policy.
|
||||
if err := r.validateSelfContainedHost(parsed.Hostname()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values = generators.MergeMaps(
|
||||
generators.MergeMaps(dynamicValues, protocolutils.GenerateVariables(parsed, false, nil)),
|
||||
values,
|
||||
)
|
||||
|
||||
// Evaluate (replace) variable with final values
|
||||
data, err = r.renderText(data, values)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "could not evaluate helper expressions")
|
||||
}
|
||||
|
||||
return r.generateRawRequest(ctx, data, parsed, values, payloads)
|
||||
}
|
||||
if err := expressions.ContainsUnresolvedVariables(data); err != nil && !r.request.SkipVariablesCheck {
|
||||
// early exit: if there are any unresolved variables in `path` after evaluation
|
||||
// then return early since this will definitely fail
|
||||
@@ -381,6 +301,227 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, input *
|
||||
return r.generateHttpRequest(ctx, urlx, values, payloads)
|
||||
}
|
||||
|
||||
// makeSelfContainedRawRequest makes a self-contained raw request. Its request
|
||||
// line names the URL, and that URL provides variables such as {{Hostname}} to
|
||||
// the rest of the request. Each part of the template text is rendered once,
|
||||
// so inserted values such as response data are sent unchanged:
|
||||
//
|
||||
// 1. Render the request line and check its host against the network policy.
|
||||
// 2. Evaluate again the template variables and payloads whose template text
|
||||
// names URL variables. The others keep their values, so a random value
|
||||
// is the same in the request line and in the rest of the request.
|
||||
// 3. Render the annotations and the rest of the request.
|
||||
func (r *requestGenerator) makeSelfContainedRawRequest(ctx context.Context, input *contextargs.Context, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
|
||||
annotations, requestLine, rest, err := splitRequestLine(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Payloads are rendered in place, so keep their template text for step 2.
|
||||
payloadText := generators.CopyMap(payloads)
|
||||
|
||||
values, authored, err := r.selfContainedValues(input, payloads, dynamicValues, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
requestLine, err = r.renderText(requestLine, values)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "could not evaluate helper expressions")
|
||||
}
|
||||
|
||||
// Parse the line that raw.ParseRawRequest will read, even if a value
|
||||
// added a line break.
|
||||
firstLine, _, _ := strings.Cut(requestLine, "\n")
|
||||
|
||||
parsed, err := r.requestLineURL(firstLine)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// URL variables only fill in names that have no other value.
|
||||
fixed := make(map[string]interface{})
|
||||
|
||||
for name, value := range protocolutils.GenerateVariables(parsed, false, nil) {
|
||||
if _, ok := values[name]; !ok {
|
||||
fixed[name] = value
|
||||
}
|
||||
}
|
||||
|
||||
stale := r.staleValues(authored, payloadText, fixed)
|
||||
if len(stale) == 0 {
|
||||
values = generators.MergeMaps(fixed, values)
|
||||
} else {
|
||||
// Keep every other value, and evaluate the stale template variables
|
||||
// and payloads again from their template text.
|
||||
for name, value := range values {
|
||||
if _, ok := stale[name]; !ok {
|
||||
fixed[name] = value
|
||||
}
|
||||
}
|
||||
|
||||
for name := range stale {
|
||||
if text, ok := payloadText[name]; ok {
|
||||
payloads[name] = text
|
||||
}
|
||||
}
|
||||
|
||||
values, _, err = r.selfContainedValues(input, payloads, dynamicValues, fixed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
annotations, err = r.renderText(annotations, values)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "could not evaluate helper expressions")
|
||||
}
|
||||
|
||||
rest, err = r.renderText(rest, values)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "could not evaluate helper expressions")
|
||||
}
|
||||
|
||||
data = annotations + requestLine + rest
|
||||
|
||||
// A value with a line break must not change which line is the request
|
||||
// line, or the request would go to a host that was not checked.
|
||||
if _, line, _, err := splitRequestLine(data); err != nil || line != firstLine {
|
||||
return nil, errkit.New("malformed request supplied: a value changed the request line")
|
||||
}
|
||||
|
||||
return r.generateRawRequest(ctx, data, parsed, values, payloads)
|
||||
}
|
||||
|
||||
// staleValues returns the names of the authored values whose template text
|
||||
// names one of urlValues, directly or through another stale value. It reads
|
||||
// template text only, so data inserted into a value cannot mark it stale.
|
||||
// Variables are checked in evaluation order, then payloads, which are
|
||||
// rendered after the variables.
|
||||
func (r *requestGenerator) staleValues(authored, payloadText, urlValues map[string]interface{}) map[string]struct{} {
|
||||
stale := make(map[string]struct{})
|
||||
names := generators.CopyMap(urlValues)
|
||||
|
||||
mark := func(name string, text interface{}) {
|
||||
if _, ok := authored[name]; !ok {
|
||||
return
|
||||
}
|
||||
|
||||
if expressions.ReferencesVariables(types.ToString(text), names) {
|
||||
stale[name] = struct{}{}
|
||||
names[name] = nil
|
||||
}
|
||||
}
|
||||
|
||||
r.request.options.Variables.ForEach(mark)
|
||||
|
||||
for name, text := range payloadText {
|
||||
mark(name, text)
|
||||
}
|
||||
|
||||
return stale
|
||||
}
|
||||
|
||||
// requestLineURL returns the URL named by a rendered raw request line after
|
||||
// checking its host against the network policy.
|
||||
func (r *requestGenerator) requestLineURL(line string) (*urlutil.URL, error) {
|
||||
parts := strings.Split(line, " ")
|
||||
if len(parts) < 3 {
|
||||
return nil, fmt.Errorf("malformed request supplied")
|
||||
}
|
||||
|
||||
if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil && !r.request.SkipVariablesCheck {
|
||||
return nil, errkit.Newf("unresolved variables `%v` found in request", parts[1])
|
||||
}
|
||||
|
||||
parsed, err := urlutil.ParseURL(parts[1], true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not parse request URL: %w", err)
|
||||
}
|
||||
// self-contained requests choose their own destination URL, so validate
|
||||
// that host against the network policy.
|
||||
if err := r.validateSelfContainedHost(parsed.Hostname()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
// selfContainedValues evaluates the template variables and renders the
|
||||
// payloads of a self-contained request, and returns the values to render the
|
||||
// request with. Payloads are rendered in place. Values in fixed are data:
|
||||
// they fill in names that have no other value, and a template variable or
|
||||
// payload with the same name keeps that value instead of being evaluated.
|
||||
// It also returns the values it produced from template text, that is the
|
||||
// evaluated template variables and the rendered payloads.
|
||||
func (r *requestGenerator) selfContainedValues(input *contextargs.Context, payloads, dynamicValues, fixed map[string]interface{}) (map[string]interface{}, map[string]interface{}, error) {
|
||||
optionValues := generators.BuildPayloadFromOptions(r.request.options.Options)
|
||||
values := generators.MergeMaps(fixed, optionValues)
|
||||
|
||||
if r.request.options.HasTemplateCtx(input.MetaInput) {
|
||||
values = generators.MergeMaps(values, r.request.options.GetTemplateCtx(input.MetaInput).GetAll())
|
||||
}
|
||||
values = generators.MergeMaps(values, dynamicValues, payloads)
|
||||
// adds all variables from `variables` section in template
|
||||
scope := r.request.options.NewVariablesScope(optionValues)
|
||||
r.request.options.AddTemplateCtxToVariablesScope(input.MetaInput, scope)
|
||||
scope.AddData(dynamicValues, payloads, r.options.Constants, fixed)
|
||||
evaluation := r.request.options.Variables.EvaluateWithInteractshScope(scope, r.request.options.Interactsh)
|
||||
|
||||
variablesMap, interactURLs := evaluation.Values, evaluation.InteractURLs
|
||||
if len(interactURLs) > 0 {
|
||||
r.interactshURLs = append(r.interactshURLs, interactURLs...)
|
||||
}
|
||||
|
||||
values = generators.MergeMaps(variablesMap, values)
|
||||
signerVars := GetDefaultSignerVars(r.request.Signature.Value)
|
||||
|
||||
// this will ensure that default signer variables are overwritten by other variables
|
||||
values = generators.MergeMaps(signerVars, values, r.options.Constants)
|
||||
|
||||
// The evaluation owns TemplateValues, so add the rendered payloads to it.
|
||||
authored := evaluation.TemplateValues
|
||||
|
||||
for payloadName, payloadValue := range payloads {
|
||||
if _, ok := fixed[payloadName]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
renderedPayload, err := r.renderText(types.ToString(payloadValue), values)
|
||||
if err != nil {
|
||||
return nil, nil, errkit.Wrap(err, "could not evaluate payload helper expressions")
|
||||
}
|
||||
|
||||
payloads[payloadName] = renderedPayload
|
||||
authored[payloadName] = renderedPayload
|
||||
}
|
||||
|
||||
// priority of variables is as follows (from low to high) for self contained templates
|
||||
// default signer vars < variables < cli vars < dynamic values < payload < constants
|
||||
return generators.MergeMaps(values, payloads, r.options.Constants), authored, nil
|
||||
}
|
||||
|
||||
// splitRequestLine splits raw request template text at its request line, the
|
||||
// first line that is not an annotation. It returns the annotation lines, the
|
||||
// request line without its line break, and the remaining text starting with
|
||||
// that line break. The three parts concatenate back to raw.
|
||||
func splitRequestLine(raw string) (annotations, requestLine, rest string, err error) {
|
||||
start := 0
|
||||
|
||||
for {
|
||||
line, _, found := strings.Cut(raw[start:], "\n")
|
||||
if !found {
|
||||
return "", "", "", errkit.New("could not read request: no request line found")
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(line, "@") {
|
||||
return raw[:start], line, raw[start+len(line):], nil
|
||||
}
|
||||
|
||||
start += len(line) + 1
|
||||
}
|
||||
}
|
||||
|
||||
// generateHttpRequest generates http request from request data from template and variables
|
||||
// finalVars = contains all variables including generator and protocol specific variables
|
||||
// generatorValues = contains variables used in fuzzing or other generator specific values
|
||||
|
||||
@@ -14,10 +14,14 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
)
|
||||
@@ -451,6 +455,211 @@ func TestMakeSelfContainedRequestDoesNotAllocateRuntimeInteractshMarkerFromPaylo
|
||||
require.Equal(t, "{{interactsh-url}}", string(body))
|
||||
}
|
||||
|
||||
func TestSelfContainedRawRequestSendsResponseDataLiterally(t *testing.T) {
|
||||
const envName = "NUCLEI_TEST_SELF_CONTAINED_ENV"
|
||||
|
||||
t.Setenv(envName, "ENV-VALUE")
|
||||
|
||||
// A fresh Options pointer, because option payloads are cached per pointer.
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.EnvironmentVariables = true
|
||||
testutils.Init(options)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
body string
|
||||
}{
|
||||
{name: "template variable", body: "{{secret}}"},
|
||||
{name: "helper expression", body: "{{to_upper(seed)}}"},
|
||||
{name: "environment variable", body: "{{" + envName + "}}"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
probes := make(chan http.Header, 1)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/source":
|
||||
_, _ = io.WriteString(w, tt.body)
|
||||
case "/probe":
|
||||
header := r.Header.Clone()
|
||||
header.Set("Host", r.Host)
|
||||
|
||||
probes <- header
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
request := &Request{
|
||||
ID: "testing-self-contained-raw-response-data",
|
||||
SelfContained: true,
|
||||
// Send the probe even though it holds a literal {{...}}.
|
||||
SkipVariablesCheck: true,
|
||||
Raw: []string{
|
||||
"GET " + server.URL + "/source HTTP/1.1\nHost: {{Hostname}}\n\n",
|
||||
"GET " + server.URL + "/probe HTTP/1.1\nHost: {{Hostname}}\n" +
|
||||
"Probe: [{{tpl}}]\nAuthored: [{{base64(seed)}}-{{" + envName + "}}]\n\n",
|
||||
},
|
||||
Operators: operators.Operators{
|
||||
Extractors: []*extractors.Extractor{{
|
||||
Name: "tpl",
|
||||
Part: "body",
|
||||
Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor},
|
||||
Regex: []string{".+"},
|
||||
Internal: true,
|
||||
}},
|
||||
},
|
||||
}
|
||||
templateVars := variables.Variable{
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(2),
|
||||
}
|
||||
templateVars.Set("seed", "hello")
|
||||
templateVars.Set("secret", "TOP-SECRET")
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: request.ID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
executerOpts.Variables = templateVars
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
err := request.ExecuteWithResults(
|
||||
contextargs.NewWithInput(context.Background(), ""),
|
||||
make(output.InternalEvent),
|
||||
make(output.InternalEvent),
|
||||
func(*output.InternalWrappedEvent) {},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
var probe http.Header
|
||||
select {
|
||||
case probe = <-probes:
|
||||
default:
|
||||
t.Fatal("probe request was not sent")
|
||||
}
|
||||
|
||||
require.Equal(t, "["+tt.body+"]", probe.Get("Probe"), "response data must be sent unchanged")
|
||||
// Template-authored text still renders, including variables derived
|
||||
// from the request URL.
|
||||
require.Equal(t, "[aGVsbG8=-ENV-VALUE]", probe.Get("Authored"))
|
||||
require.Equal(t, strings.TrimPrefix(server.URL, "http://"), probe.Get("Host"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// makeSelfContainedRawRequest compiles request as a self-contained raw request
|
||||
// with the given template variables and makes its first request.
|
||||
func makeSelfContainedRawRequest(t *testing.T, request *Request, vars [][2]string, dynamicValues map[string]interface{}) (*generatedRequest, error) {
|
||||
t.Helper()
|
||||
|
||||
options := testutils.DefaultOptions
|
||||
testutils.Init(options)
|
||||
|
||||
request.ID = "testing-self-contained-raw"
|
||||
request.SelfContained = true
|
||||
templateVars := variables.Variable{
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(len(vars)),
|
||||
}
|
||||
|
||||
for _, v := range vars {
|
||||
templateVars.Set(v[0], v[1])
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: request.ID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
executerOpts.Variables = templateVars
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
generator := request.newGenerator(false)
|
||||
inputData, payloads, ok := generator.nextValue()
|
||||
require.True(t, ok)
|
||||
|
||||
return generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), ""), inputData, payloads, dynamicValues)
|
||||
}
|
||||
|
||||
func TestMakeSelfContainedRawRequestResolvesURLVariables(t *testing.T) {
|
||||
request := &Request{
|
||||
Raw: []string{"GET http://198.51.100.7/?t={{token}}&d={{data_token}} HTTP/1.1\nHost: {{Hostname}}\n" +
|
||||
"X-Var: {{host_var}}\nX-Upper: {{upper_host}}\nX-Payload: {{user}}\n" +
|
||||
"X-Token: {{token}}\nX-Data-Token: {{data_token}}\nX-Data: {{data}}\n\n"},
|
||||
Payloads: map[string]interface{}{"user": []string{"admin@{{Hostname}}"}},
|
||||
// Send values that hold a literal {{...}} from data.
|
||||
SkipVariablesCheck: true,
|
||||
}
|
||||
got, err := makeSelfContainedRawRequest(t, request, [][2]string{
|
||||
{"token", "{{rand_base(8)}}"},
|
||||
{"data_token", "{{rand_base(8)}}-{{tpl}}"},
|
||||
{"data", "{{tpl}}"},
|
||||
{"host_var", "{{Hostname}}"},
|
||||
{"upper_host", "{{to_upper(host_var)}}"},
|
||||
}, map[string]interface{}{"tpl": "{{Hostname}}"})
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, "198.51.100.7", got.request.Header.Get("X-Var"))
|
||||
require.Equal(t, "198.51.100.7", got.request.Header.Get("X-Upper"))
|
||||
require.Equal(t, "[email protected]", got.request.Header.Get("X-Payload"))
|
||||
// Data that names a URL variable is sent unchanged.
|
||||
require.Equal(t, "{{Hostname}}", got.request.Header.Get("X-Data"))
|
||||
// Values whose template text names no URL variable are not evaluated
|
||||
// again, even when data inserted into them does.
|
||||
require.Equal(t, got.request.Params.Get("t"), got.request.Header.Get("X-Token"))
|
||||
require.Equal(t, got.request.Params.Get("d"), got.request.Header.Get("X-Data-Token"))
|
||||
require.True(t, strings.HasSuffix(got.request.Header.Get("X-Data-Token"), "-{{Hostname}}"))
|
||||
}
|
||||
|
||||
func TestMakeSelfContainedRawRequestRejectsMovedRequestLine(t *testing.T) {
|
||||
request := &Request{
|
||||
Raw: []string{"@timeout: {{wait}}\nGET http://198.51.100.7/ HTTP/1.1\nHost: {{Hostname}}\n\n"},
|
||||
}
|
||||
_, err := makeSelfContainedRawRequest(t, request, nil, map[string]interface{}{
|
||||
"wait": "5s\nGET http://192.0.2.1/secret HTTP/1.1",
|
||||
})
|
||||
require.ErrorContains(t, err, "changed the request line")
|
||||
}
|
||||
|
||||
func TestSplitRequestLine(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
annotations, requestLine, rest string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "no annotations",
|
||||
raw: "GET http://a/ HTTP/1.1\r\nHost: a\r\n\r\n",
|
||||
requestLine: "GET http://a/ HTTP/1.1\r",
|
||||
rest: "\nHost: a\r\n\r\n",
|
||||
},
|
||||
{
|
||||
name: "annotations",
|
||||
raw: "@tls-sni: {{Hostname}}\n@timeout: 5s\nGET http://a/ HTTP/1.1\nHost: a\n",
|
||||
annotations: "@tls-sni: {{Hostname}}\n@timeout: 5s\n",
|
||||
requestLine: "GET http://a/ HTTP/1.1",
|
||||
rest: "\nHost: a\n",
|
||||
},
|
||||
{name: "no line break", raw: "GET http://a/ HTTP/1.1", wantErr: true},
|
||||
{name: "annotations only", raw: "@timeout: 5s\n", wantErr: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
annotations, requestLine, rest, err := splitRequestLine(tt.raw)
|
||||
if tt.wantErr {
|
||||
require.Error(t, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tt.annotations, annotations)
|
||||
require.Equal(t, tt.requestLine, requestLine)
|
||||
require.Equal(t, tt.rest, rest)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRequestDoesNotReevaluateGeneratedPayloadMetadata(t *testing.T) {
|
||||
options := testutils.DefaultOptions
|
||||
|
||||
|
||||
@@ -402,7 +402,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
|
||||
|
||||
// If we have request level timeout, ignore http client timeouts
|
||||
for _, req := range request.Raw {
|
||||
if reTimeoutAnnotation.MatchString(req) {
|
||||
if hasTimeoutAnnotation(req) {
|
||||
connectionConfiguration.NoTimeout = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/authprovider/authx"
|
||||
"github.com/projectdiscovery/rawhttp/client"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
)
|
||||
|
||||
@@ -193,23 +192,20 @@ func readRawRequest(request string, unsafe bool) (*Request, error) {
|
||||
Headers: make(map[string]string),
|
||||
}
|
||||
|
||||
_, request = SplitAnnotations(request)
|
||||
|
||||
// store body if it is unsafe request
|
||||
if unsafe {
|
||||
rawRequest.UnsafeRawBytes = stripLeadingAnnotations(request)
|
||||
rawRequest.UnsafeRawBytes = []byte(request)
|
||||
}
|
||||
|
||||
// parse raw request
|
||||
source := strings.NewReader(request)
|
||||
reader := bufio.NewReader(source)
|
||||
read_line:
|
||||
s, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not read request: %w", err)
|
||||
}
|
||||
// ignore all annotations
|
||||
if stringsutil.HasPrefixAny(s, "@") {
|
||||
goto read_line
|
||||
}
|
||||
|
||||
parts := strings.Fields(s)
|
||||
if len(parts) > 0 {
|
||||
@@ -292,34 +288,18 @@ read_line:
|
||||
|
||||
}
|
||||
|
||||
func stripLeadingAnnotations(request string) []byte {
|
||||
reader := bufio.NewReader(strings.NewReader(request))
|
||||
buffer := bufferPool.Get().(*bytes.Buffer)
|
||||
buffer.Reset()
|
||||
defer func() {
|
||||
buffer.Reset()
|
||||
bufferPool.Put(buffer)
|
||||
}()
|
||||
|
||||
requestLineFound := false
|
||||
for {
|
||||
line, err := reader.ReadString('\n')
|
||||
if len(line) > 0 {
|
||||
if requestLineFound || !stringsutil.HasPrefixAny(line, "@") {
|
||||
requestLineFound = true
|
||||
_, _ = buffer.WriteString(line)
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return []byte(request)
|
||||
}
|
||||
// SplitAnnotations splits a raw request into its annotations and the request
|
||||
// that follows them. Annotations are the lines starting with "@" before the
|
||||
// request line. A line starting with "@" after the request line is part of the
|
||||
// request, so text rendered into the headers or the body never becomes an
|
||||
// annotation.
|
||||
func SplitAnnotations(request string) (annotations, rest string) {
|
||||
rest = request
|
||||
for strings.HasPrefix(rest, "@") {
|
||||
_, rest, _ = strings.Cut(rest, "\n")
|
||||
}
|
||||
|
||||
return append([]byte(nil), buffer.Bytes()...)
|
||||
return request[:len(request)-len(rest)], rest
|
||||
}
|
||||
|
||||
// TryFillCustomHeaders after the Host header
|
||||
|
||||
@@ -310,6 +310,51 @@ Connection: close`, parseURL(t, "http://scanme.sh"), true, false)
|
||||
require.NotContains(t, string(request.UnsafeRawBytes), "@Host:", "annotation lines must not be present in unsafe raw bytes")
|
||||
}
|
||||
|
||||
func TestSplitAnnotations(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
request string
|
||||
annotations string
|
||||
}{
|
||||
{name: "empty"},
|
||||
{name: "no annotations", request: "GET / HTTP/1.1\r\nHost: a\r\n\r\n"},
|
||||
{
|
||||
name: "leading annotations",
|
||||
request: "@Host: b\r\n@timeout: 5s\r\nGET / HTTP/1.1\r\nHost: a\r\n\r\n",
|
||||
annotations: "@Host: b\r\n@timeout: 5s\r\n",
|
||||
},
|
||||
{
|
||||
name: "annotation lines after the request line",
|
||||
request: "@once\nPOST / HTTP/1.1\n@Host: c\n\nhello\n@Host: d\n",
|
||||
annotations: "@once\n",
|
||||
},
|
||||
{
|
||||
name: "annotations without a request line",
|
||||
request: "@Host: b\n@once",
|
||||
annotations: "@Host: b\n@once",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
annotations, rest := SplitAnnotations(tt.request)
|
||||
require.Equal(t, tt.annotations, annotations)
|
||||
require.Equal(t, tt.request, annotations+rest)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseUnsafeRequestKeepsAnnotationLinesInBody(t *testing.T) {
|
||||
const (
|
||||
head = "POST /foo HTTP/1.1\nHost: {{Hostname}}\n\n"
|
||||
body = "hello\n@Host: decoy.example\n"
|
||||
)
|
||||
request, err := Parse("@Host: honey.scanme.sh\n"+head+body, parseURL(t, "http://scanme.sh"), true, false)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, head+body, string(request.UnsafeRawBytes))
|
||||
require.Equal(t, body, request.Data)
|
||||
}
|
||||
|
||||
func TestTryFillCustomHeaders(t *testing.T) {
|
||||
testValue := "GET /manager/html HTTP/1.1\r\nHost: Test\r\n"
|
||||
expected := "GET /test/manager/html HTTP/1.1\r\nHost: Test\r\ntest: test\r\n"
|
||||
|
||||
@@ -1675,8 +1675,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
||||
}
|
||||
|
||||
responseContentType := respChain.Response().Header.Get("Content-Type")
|
||||
isResponseTruncated := (request.MaxSize > 0 && respChain.Body().Len() >= request.MaxSize) ||
|
||||
(request.options.Options.ResponseReadSize > 0 && respChain.Body().Len() >= request.options.Options.ResponseReadSize)
|
||||
isResponseTruncated := respChain.Body().Len() >= maxBodylimit
|
||||
dumpResponse(event, request, fullResponseStr, formedURL, responseContentType, isResponseTruncated, input.MetaInput.Input)
|
||||
|
||||
callback(event)
|
||||
@@ -1989,7 +1988,8 @@ func (request *Request) needsFullResponse(generatedRequest *generatedRequest) bo
|
||||
}
|
||||
if request.options != nil && request.options.Options != nil {
|
||||
cliOptions := request.options.Options
|
||||
if cliOptions.Debug || cliOptions.DebugResponse || cliOptions.StoreResponse || cliOptions.HTTPStats {
|
||||
// -ms writes the response of failed requests too, and a failure never reaches the lazy fill
|
||||
if cliOptions.Debug || cliOptions.DebugResponse || cliOptions.StoreResponse || cliOptions.HTTPStats || cliOptions.MatcherStatus {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/projectdiscovery/fastdialer/fastdialer"
|
||||
commoninteractsh "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/raw"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
@@ -54,8 +55,9 @@ func parseFlowAnnotations(rawRequest string) (flowMark, bool) {
|
||||
var fm flowMark
|
||||
// parse request for known override annotations
|
||||
var hasFlowOverride bool
|
||||
annotations, _ := raw.SplitAnnotations(rawRequest)
|
||||
// @once
|
||||
if reOnceAnnotation.MatchString(rawRequest) {
|
||||
if reOnceAnnotation.MatchString(annotations) {
|
||||
fm = Once
|
||||
hasFlowOverride = true
|
||||
}
|
||||
@@ -63,6 +65,12 @@ func parseFlowAnnotations(rawRequest string) (flowMark, bool) {
|
||||
return fm, hasFlowOverride
|
||||
}
|
||||
|
||||
// hasTimeoutAnnotation reports whether rawRequest has a @timeout annotation.
|
||||
func hasTimeoutAnnotation(rawRequest string) bool {
|
||||
annotations, _ := raw.SplitAnnotations(rawRequest)
|
||||
return reTimeoutAnnotation.MatchString(annotations)
|
||||
}
|
||||
|
||||
type annotationOverrides struct {
|
||||
request *retryablehttp.Request
|
||||
cancelFunc context.CancelFunc
|
||||
@@ -72,10 +80,12 @@ type annotationOverrides struct {
|
||||
|
||||
// parseAnnotations and override requests settings
|
||||
func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Request) (overrides annotationOverrides, modified bool) {
|
||||
// parse request for known override annotations
|
||||
// rawRequest is rendered, so its headers and body can hold values taken
|
||||
// from responses. Only the annotation lines may override the request.
|
||||
annotations, _ := raw.SplitAnnotations(rawRequest)
|
||||
|
||||
// @Host:target
|
||||
if hosts := reHostAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 {
|
||||
if hosts := reHostAnnotation.FindStringSubmatch(annotations); len(hosts) > 0 {
|
||||
value := strings.TrimSpace(hosts[1])
|
||||
// handle scheme
|
||||
switch {
|
||||
@@ -101,7 +111,7 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req
|
||||
}
|
||||
|
||||
// @tls-sni:target
|
||||
if hosts := reSniAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 {
|
||||
if hosts := reSniAnnotation.FindStringSubmatch(annotations); len(hosts) > 0 {
|
||||
value := strings.TrimSpace(hosts[1])
|
||||
value = stringsutil.TrimPrefixAny(value, "http://", "https://")
|
||||
|
||||
@@ -140,7 +150,7 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req
|
||||
modified = true
|
||||
var ctx context.Context
|
||||
|
||||
if duration := reTimeoutAnnotation.FindStringSubmatch(rawRequest); len(duration) > 0 {
|
||||
if duration := reTimeoutAnnotation.FindStringSubmatch(annotations); len(duration) > 0 {
|
||||
value := strings.TrimSpace(duration[1])
|
||||
if parsedTimeout, err := time.ParseDuration(value); err == nil {
|
||||
// Cap at maximum allowed timeout to prevent DoS via extremely large timeout values
|
||||
|
||||
@@ -2,13 +2,25 @@ package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
commoninteractsh "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
@@ -191,3 +203,123 @@ func TestRequestParseAnnotationsTimeout(t *testing.T) {
|
||||
require.False(t, deadlined, "could not get set request deadline")
|
||||
})
|
||||
}
|
||||
|
||||
func TestRequestParseAnnotationsIgnoresLinesAfterRequestLine(t *testing.T) {
|
||||
req := &Request{connConfiguration: &httpclientpool.Configuration{}}
|
||||
rawRequest := "POST /probe HTTP/1.1\r\n" +
|
||||
"Host: example.com\r\n" +
|
||||
"\r\n" +
|
||||
"hello\n@Host: http://127.0.0.1:8797\n@tls-sni: decoy.example\n"
|
||||
|
||||
httpReq, err := retryablehttp.NewRequest(http.MethodPost, "https://example.com/probe", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
overrides, modified := req.parseAnnotations(rawRequest, httpReq)
|
||||
require.False(t, modified)
|
||||
require.Equal(t, "https://example.com/probe", overrides.request.String())
|
||||
require.Nil(t, overrides.request.TLS)
|
||||
}
|
||||
|
||||
// A response value reused in a raw request must stay request content even when
|
||||
// one of its lines looks like an annotation. Annotations are only the lines
|
||||
// before the request line.
|
||||
func TestRawRequestReusingResponseWithAnnotationLine(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
unsafe bool
|
||||
selfContained bool
|
||||
}{
|
||||
{name: "raw"},
|
||||
{name: "unsafe", unsafe: true},
|
||||
{name: "self-contained", selfContained: true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var decoyRequests atomic.Int32
|
||||
decoy := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
|
||||
decoyRequests.Add(1)
|
||||
}))
|
||||
defer decoy.Close()
|
||||
|
||||
responseBody := fmt.Sprintf("hello\n@Host: %s\n", decoy.URL)
|
||||
|
||||
var mu sync.Mutex
|
||||
var probes []string
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/source" {
|
||||
_, _ = io.WriteString(w, responseBody)
|
||||
return
|
||||
}
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
mu.Lock()
|
||||
probes = append(probes, r.Method+" "+r.URL.Path+" "+string(body))
|
||||
mu.Unlock()
|
||||
}))
|
||||
defer target.Close()
|
||||
|
||||
baseURL, host, input := "", "{{Hostname}}", target.URL
|
||||
if tt.selfContained {
|
||||
// A self-contained request names its own target.
|
||||
baseURL, host, input = target.URL, strings.TrimPrefix(target.URL, "http://"), ""
|
||||
}
|
||||
probeHeaders := "Host: " + host + "\nContent-Type: text/plain\n"
|
||||
if tt.unsafe {
|
||||
// An unsafe request is sent as written, so it states its own length.
|
||||
probeHeaders += "Content-Length: {{len(tpl)}}\n"
|
||||
}
|
||||
request := &Request{
|
||||
ID: "testing-http-response-annotation-line",
|
||||
Unsafe: tt.unsafe,
|
||||
SelfContained: tt.selfContained,
|
||||
Raw: []string{
|
||||
"GET " + baseURL + "/source HTTP/1.1\nHost: " + host + "\n\n",
|
||||
"POST " + baseURL + "/probe HTTP/1.1\n" + probeHeaders + "\n{{tpl}}",
|
||||
},
|
||||
Operators: operators.Operators{
|
||||
Extractors: []*extractors.Extractor{{
|
||||
Name: "tpl",
|
||||
Part: "body",
|
||||
Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor},
|
||||
Regex: []string{`(?s).+`},
|
||||
Internal: true,
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
options := testutils.DefaultOptions
|
||||
testutils.Init(options)
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: request.ID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}, Name: "test"},
|
||||
})
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
err := request.ExecuteWithResults(
|
||||
contextargs.NewWithInput(context.Background(), input),
|
||||
make(output.InternalEvent),
|
||||
make(output.InternalEvent),
|
||||
func(*output.InternalWrappedEvent) {},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Zero(t, decoyRequests.Load(), "a response line must not redirect the request")
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
require.Equal(t, []string{"POST /probe " + responseBody}, probes)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemplateAnnotationsAfterRequestLine(t *testing.T) {
|
||||
const request = "GET / HTTP/1.1\r\nHost: {{Hostname}}\r\n\r\n@timeout: 5s\r\n@once\r\n"
|
||||
|
||||
require.False(t, hasTimeoutAnnotation(request))
|
||||
require.True(t, hasTimeoutAnnotation("@timeout: 5s\r\n"+request))
|
||||
|
||||
_, hasFlowOverride := parseFlowAnnotations(request)
|
||||
require.False(t, hasFlowOverride)
|
||||
mark, hasFlowOverride := parseFlowAnnotations("@once\r\n" + request)
|
||||
require.True(t, hasFlowOverride)
|
||||
require.Equal(t, Once, mark)
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
@@ -1865,6 +1866,84 @@ func TestHTTPResponseReadSizeHonoursTemplateMaxSize(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// storedResponseWriter keeps the stored response dumps.
|
||||
type storedResponseWriter struct {
|
||||
*testutils.MockOutputWriter
|
||||
|
||||
responses []string
|
||||
}
|
||||
|
||||
func (w *storedResponseWriter) WriteStoreDebugData(_, _, _, data string) {
|
||||
if strings.Contains(data, "Dumped HTTP response") {
|
||||
w.responses = append(w.responses, data)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHTTPResponseTruncatedUsesEffectiveReadLimit checks that a stored response
|
||||
// is marked truncated exactly when its body reached the limit it was read with.
|
||||
func TestHTTPResponseTruncatedUsesEffectiveReadLimit(t *testing.T) {
|
||||
previous := MaxBodyRead
|
||||
|
||||
t.Cleanup(func() { MaxBodyRead = previous })
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
_, _ = w.Write([]byte(strings.Repeat("A", 2000)))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
defaultLimit int // replaces MaxBodyRead when set
|
||||
templateMaxSize int
|
||||
responseReadSize int
|
||||
wantTruncated bool
|
||||
}{
|
||||
{name: "default limit", defaultLimit: 1000, wantTruncated: true},
|
||||
{name: "default limit, negative response-size-read", defaultLimit: 1000, responseReadSize: -1, wantTruncated: true},
|
||||
{name: "body under default limit", wantTruncated: false},
|
||||
{name: "template max-size", templateMaxSize: 500, wantTruncated: true},
|
||||
{name: "response-size-read", responseReadSize: 800, wantTruncated: true},
|
||||
{name: "max-size below response-size-read", templateMaxSize: 500, responseReadSize: 3000, wantTruncated: true},
|
||||
{name: "limits above body length", templateMaxSize: 4000, responseReadSize: 3000, wantTruncated: false},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
MaxBodyRead = cmp.Or(tc.defaultLimit, previous)
|
||||
|
||||
opts := testutils.DefaultOptions.Copy()
|
||||
opts.StoreResponse = true
|
||||
opts.ResponseReadSize = tc.responseReadSize
|
||||
testutils.Init(opts)
|
||||
t.Cleanup(func() { testutils.Cleanup(opts) })
|
||||
|
||||
request := &Request{
|
||||
ID: "test-truncated",
|
||||
MaxSize: tc.templateMaxSize,
|
||||
Raw: []string{"GET / HTTP/1.1\nHost: {{Hostname}}\n\n"},
|
||||
}
|
||||
executerOptions := testutils.NewMockExecuterOptions(opts, &testutils.TemplateInfo{
|
||||
ID: request.ID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
writer := &storedResponseWriter{MockOutputWriter: testutils.NewMockOutputWriter(opts.OmitTemplate)}
|
||||
executerOptions.Output = writer
|
||||
require.NoError(t, request.Compile(executerOptions))
|
||||
|
||||
err := request.ExecuteWithResults(
|
||||
contextargs.NewWithInput(context.Background(), server.URL),
|
||||
make(output.InternalEvent),
|
||||
make(output.InternalEvent),
|
||||
func(*output.InternalWrappedEvent) {},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, writer.responses, 1)
|
||||
require.Equal(t, tc.wantTruncated, strings.Contains(writer.responses[0], "(Truncated)"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const httpToHTTPSPage = "<html><title>400 The plain HTTP request was sent to HTTPS port</title><center>The plain HTTP request was sent to HTTPS port</center></html>"
|
||||
|
||||
// servePlainOrTLS accepts HTTP and TLS on one port. A plaintext request is
|
||||
|
||||
@@ -248,6 +248,26 @@ func TestNeedsFullResponse(t *testing.T) {
|
||||
require.NoError(t, req.Compile(execOpts))
|
||||
require.True(t, req.needsFullResponse(nil))
|
||||
})
|
||||
|
||||
t.Run("matcher status option requires full response", func(t *testing.T) {
|
||||
msOpts := testutils.DefaultOptions.Copy()
|
||||
msOpts.MatcherStatus = true
|
||||
|
||||
req := &Request{
|
||||
ID: "matcher-status-flag",
|
||||
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
|
||||
Operators: operators.Operators{
|
||||
Matchers: []*matchers.Matcher{{
|
||||
Part: "body",
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
|
||||
Words: []string{"test"},
|
||||
}},
|
||||
},
|
||||
}
|
||||
execOpts := testutils.NewMockExecuterOptions(msOpts, &testutils.TemplateInfo{ID: "matcher-status-flag"})
|
||||
require.NoError(t, req.Compile(execOpts))
|
||||
require.True(t, req.needsFullResponse(nil))
|
||||
})
|
||||
}
|
||||
|
||||
func TestExecuteRequestDeferredResponse_NoMatch(t *testing.T) {
|
||||
@@ -295,6 +315,49 @@ func TestExecuteRequestDeferredResponse_NoMatch(t *testing.T) {
|
||||
require.Contains(t, types.ToString(capturedEvent.InternalEvent["all_headers"]), "X-Custom-Header")
|
||||
}
|
||||
|
||||
// With -ms a failed match is written out too, so its event must carry the
|
||||
// response even though no result is created.
|
||||
func TestExecuteRequestDeferredResponse_NoMatchWithMatcherStatus(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = fmt.Fprintf(w, "response body content")
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.MatcherStatus = true
|
||||
testutils.Init(options)
|
||||
|
||||
req := &Request{
|
||||
ID: "defer-no-match-ms",
|
||||
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
|
||||
Path: []string{"{{BaseURL}}"},
|
||||
Operators: operators.Operators{
|
||||
Matchers: []*matchers.Matcher{{
|
||||
Part: "body",
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
|
||||
Words: []string{"nonexistent-needle"},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
execOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: "defer-no-match-ms",
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}, Name: "test"},
|
||||
})
|
||||
require.NoError(t, req.Compile(execOpts))
|
||||
|
||||
var capturedEvent *output.InternalWrappedEvent
|
||||
ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
|
||||
err := req.ExecuteWithResults(ctxArgs, make(output.InternalEvent), make(output.InternalEvent), func(event *output.InternalWrappedEvent) {
|
||||
capturedEvent = event
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, capturedEvent)
|
||||
require.False(t, capturedEvent.HasResults())
|
||||
require.Contains(t, types.ToString(capturedEvent.InternalEvent["response"]), "HTTP/1.1 200")
|
||||
require.Contains(t, types.ToString(capturedEvent.InternalEvent["response"]), "response body content")
|
||||
}
|
||||
|
||||
func TestExecuteRequestDeferredResponse_WithMatch(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Server", "nuclei-test")
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/alecthomas/chroma/quick"
|
||||
"github.com/alecthomas/chroma/v2/quick"
|
||||
"github.com/ditashi/jsbeautifier-go/jsbeautifier"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/goja"
|
||||
|
||||
@@ -1,39 +1,57 @@
|
||||
package rdapclientpool
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
|
||||
"github.com/openrdap/rdap/bootstrap"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/rdap"
|
||||
)
|
||||
|
||||
var normalClient *rdap.Client
|
||||
// clients holds one client per execution, because each one dials with the
|
||||
// network policy of its execution.
|
||||
var clients = map[string]*rdap.Client{}
|
||||
var m sync.Mutex
|
||||
|
||||
// Init initializes the client pool implementation
|
||||
func Init(options *types.Options) error {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
_, err := Get(options, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
// Don't create clients if already created in the past.
|
||||
if normalClient != nil {
|
||||
return nil
|
||||
// newClient returns an rdap client whose requests, bootstrap lookups and
|
||||
// redirects all dial through the fastdialer of the execution, so every
|
||||
// address it connects to is checked against the network policy.
|
||||
func newClient(options *types.Options) *rdap.Client {
|
||||
executionID := options.ExecutionId
|
||||
httpClient := &http.Client{
|
||||
Transport: &http.Transport{
|
||||
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
dialers := protocolstate.GetDialersWithId(executionID)
|
||||
if dialers == nil || dialers.Fastdialer == nil {
|
||||
return nil, fmt.Errorf("dialers not initialized for %s", executionID)
|
||||
}
|
||||
return dialers.Fastdialer.Dial(ctx, network, address)
|
||||
},
|
||||
},
|
||||
Timeout: options.GetTimeouts().HttpTimeout,
|
||||
}
|
||||
client := &rdap.Client{
|
||||
HTTP: httpClient,
|
||||
Bootstrap: &bootstrap.Client{HTTP: httpClient},
|
||||
}
|
||||
|
||||
normalClient = &rdap.Client{}
|
||||
if options.Verbose || options.Debug || options.DebugRequests || options.DebugResponse {
|
||||
normalClient.Verbose = func(text string) {
|
||||
client.Verbose = func(text string) {
|
||||
gologger.Debug().Msgf("rdap: %s", text)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getNormalClient() *rdap.Client {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
return normalClient
|
||||
return client
|
||||
}
|
||||
|
||||
// Configuration contains the custom configuration options for a client - placeholder
|
||||
@@ -44,7 +62,25 @@ func (c *Configuration) Hash() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Get creates or gets a client for the protocol based on custom configuration
|
||||
// Get creates or gets the client of the execution
|
||||
func Get(options *types.Options, configuration *Configuration) (*rdap.Client, error) {
|
||||
return getNormalClient(), nil
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
if client, ok := clients[options.ExecutionId]; ok {
|
||||
return client, nil
|
||||
}
|
||||
client := newClient(options)
|
||||
clients[options.ExecutionId] = client
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// Close drops the client of an execution.
|
||||
func Close(executionID string) {
|
||||
m.Lock()
|
||||
client := clients[executionID]
|
||||
delete(clients, executionID)
|
||||
m.Unlock()
|
||||
if client != nil {
|
||||
client.HTTP.CloseIdleConnections()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package rdapclientpool
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/rdap"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newRdapServer(t *testing.T, hits *atomic.Int32) *httptest.Server {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.Header().Set("Content-Type", "application/rdap+json")
|
||||
_, _ = w.Write([]byte(`{"objectClassName":"domain","ldhName":"example.com"}`))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return server
|
||||
}
|
||||
|
||||
func rdapOptions(t *testing.T, executionID string, excludeTargets []string) *types.Options {
|
||||
t.Helper()
|
||||
options := types.DefaultOptions()
|
||||
options.ExecutionId = executionID
|
||||
options.ExcludeTargets = excludeTargets
|
||||
require.NoError(t, protocolstate.Init(options))
|
||||
t.Cleanup(func() { protocolstate.Close(options.ExecutionId) })
|
||||
require.NoError(t, Init(options))
|
||||
return options
|
||||
}
|
||||
|
||||
func domainRequest(t *testing.T, server string) *rdap.Request {
|
||||
t.Helper()
|
||||
serverURL, err := url.Parse(server)
|
||||
require.NoError(t, err)
|
||||
request := rdap.NewAutoRequest("example.com")
|
||||
request.Server = serverURL
|
||||
return request
|
||||
}
|
||||
|
||||
// Each execution gets a client that dials with its own network policy.
|
||||
func TestClientObeysExecutionNetworkPolicy(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
server := newRdapServer(t, &hits)
|
||||
|
||||
open := rdapOptions(t, t.Name()+"-open", nil)
|
||||
restricted := rdapOptions(t, t.Name()+"-restricted", []string{"127.0.0.1"})
|
||||
|
||||
openClient, err := Get(open, nil)
|
||||
require.NoError(t, err)
|
||||
_, err = openClient.Do(domainRequest(t, server.URL))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int32(1), hits.Load())
|
||||
|
||||
restrictedClient, err := Get(restricted, nil)
|
||||
require.NoError(t, err)
|
||||
_, err = restrictedClient.Do(domainRequest(t, server.URL))
|
||||
require.Error(t, err)
|
||||
require.Equal(t, int32(1), hits.Load(), "rdap request reached a denied address")
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package whois
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestWhoisServerRedirectObeysNetworkPolicy(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
internal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
hits.Add(1)
|
||||
w.Header().Set("Content-Type", "application/rdap+json")
|
||||
_, _ = w.Write([]byte(`{"objectClassName":"domain","ldhName":"example.com"}`))
|
||||
}))
|
||||
t.Cleanup(internal.Close)
|
||||
// the redirector listens on ::1 so that it is reached while 127.0.0.1 is denied
|
||||
var redirects atomic.Int32
|
||||
redirector := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
redirects.Add(1)
|
||||
http.Redirect(w, r, internal.URL+r.URL.Path, http.StatusFound)
|
||||
}))
|
||||
listener, err := net.Listen("tcp", "[::1]:0")
|
||||
require.NoError(t, err)
|
||||
redirector.Listener = listener
|
||||
redirector.Start()
|
||||
t.Cleanup(redirector.Close)
|
||||
_, port, err := net.SplitHostPort(listener.Addr().String())
|
||||
require.NoError(t, err)
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.SetExecutionID(t.Name())
|
||||
options.ExcludeTargets = []string{"127.0.0.1"}
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() { protocolinit.Close(options.ExecutionId) })
|
||||
require.True(t, protocolstate.IsHostAllowed(options.ExecutionId, "::1"))
|
||||
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "whois-redirect"})
|
||||
request := &Request{ID: "whois-redirect", Query: "example.com", Server: "http://[::1]:" + port + "/rdap/"}
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
input := contextargs.NewWithInput(context.Background(), "example.com")
|
||||
_ = request.ExecuteWithResults(input, nil, nil, func(*output.InternalWrappedEvent) {})
|
||||
require.NotZero(t, redirects.Load(), "the redirector was never reached")
|
||||
require.Zero(t, hits.Load(), "whois redirect reached a denied address")
|
||||
}
|
||||
@@ -71,9 +71,10 @@ func New(option *Options) (*Exporter, error) {
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 10,
|
||||
MaxIdleConnsPerHost: 10,
|
||||
DialContext: dialers.Fastdialer.Dial,
|
||||
DialTLSContext: dialers.Fastdialer.DialTLS,
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification},
|
||||
// No DialTLSContext: fastdialer DialTLS ignores TLSClientConfig and
|
||||
// skips verification; net/http does the handshake with TLSClientConfig.
|
||||
DialContext: dialers.Fastdialer.Dial,
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package es
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestExporterVerifiesTLS(t *testing.T) {
|
||||
var got atomic.Int32
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
got.Add(1)
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}))
|
||||
defer srv.Close()
|
||||
host, port, err := net.SplitHostPort(srv.Listener.Addr().String())
|
||||
require.NoError(t, err)
|
||||
p, err := strconv.Atoi(port)
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := types.DefaultOptions()
|
||||
opts.SetExecutionID("es-tls-verify")
|
||||
require.NoError(t, protocolstate.Init(opts))
|
||||
defer protocolstate.Close(opts.ExecutionId)
|
||||
|
||||
newExporter := func(skipVerify bool) *Exporter {
|
||||
e, err := New(&Options{IP: host, Port: p, SSL: true, SSLVerification: skipVerify, Username: "elastic", Password: "secret", IndexName: "nuclei", ExecutionId: opts.ExecutionId})
|
||||
require.NoError(t, err)
|
||||
return e
|
||||
}
|
||||
|
||||
err = newExporter(false).Export(&output.ResultEvent{})
|
||||
require.ErrorContains(t, err, "certificate")
|
||||
require.Zero(t, got.Load(), "the password went to an untrusted server")
|
||||
|
||||
require.NoError(t, newExporter(true).Export(&output.ResultEvent{}))
|
||||
require.EqualValues(t, 1, got.Load())
|
||||
}
|
||||
@@ -63,9 +63,10 @@ func New(option *Options) (*Exporter, error) {
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 10,
|
||||
MaxIdleConnsPerHost: 10,
|
||||
DialContext: dialers.Fastdialer.Dial,
|
||||
DialTLSContext: dialers.Fastdialer.DialTLS,
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification},
|
||||
// No DialTLSContext: fastdialer DialTLS ignores TLSClientConfig and
|
||||
// skips verification; net/http does the handshake with TLSClientConfig.
|
||||
DialContext: dialers.Fastdialer.Dial,
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package splunk
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestExporterVerifiesTLS(t *testing.T) {
|
||||
var got atomic.Int32
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
got.Add(1)
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}))
|
||||
defer srv.Close()
|
||||
host, port, err := net.SplitHostPort(srv.Listener.Addr().String())
|
||||
require.NoError(t, err)
|
||||
p, err := strconv.Atoi(port)
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := types.DefaultOptions()
|
||||
opts.SetExecutionID("splunk-tls-verify")
|
||||
require.NoError(t, protocolstate.Init(opts))
|
||||
defer protocolstate.Close(opts.ExecutionId)
|
||||
|
||||
newExporter := func(skipVerify bool) *Exporter {
|
||||
e, err := New(&Options{Host: host, Port: p, SSL: true, SSLVerification: skipVerify, Token: "secret", IndexName: "nuclei", ExecutionId: opts.ExecutionId})
|
||||
require.NoError(t, err)
|
||||
return e
|
||||
}
|
||||
|
||||
err = newExporter(false).Export(&output.ResultEvent{})
|
||||
require.ErrorContains(t, err, "certificate")
|
||||
require.Zero(t, got.Load(), "the token went to an untrusted server")
|
||||
|
||||
require.NoError(t, newExporter(true).Export(&output.ResultEvent{}))
|
||||
require.EqualValues(t, 1, got.Load())
|
||||
}
|
||||
@@ -247,6 +247,9 @@ type Options struct {
|
||||
LeaveDefaultPorts bool
|
||||
// AutomaticScan enables automatic tech based template execution
|
||||
AutomaticScan bool
|
||||
// AutomaticScanCoverage tunes automatic scan selection breadth:
|
||||
// "lean" (dependents only), "balanced" (default), or "thorough".
|
||||
AutomaticScanCoverage string
|
||||
// Silent suppresses any extra text and only writes found URLs on screen.
|
||||
Silent bool
|
||||
// Validate validates the templates passed to nuclei.
|
||||
@@ -705,6 +708,7 @@ func (options *Options) Copy() *Options {
|
||||
AutoCalibrate: options.AutoCalibrate,
|
||||
LeaveDefaultPorts: options.LeaveDefaultPorts,
|
||||
AutomaticScan: options.AutomaticScan,
|
||||
AutomaticScanCoverage: options.AutomaticScanCoverage,
|
||||
Silent: options.Silent,
|
||||
Validate: options.Validate,
|
||||
NoStrictSyntax: options.NoStrictSyntax,
|
||||
|
||||
Reference in new issue
Block a user