mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2026-11-07 17:43:04 -10:00
Compare commits
19
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
943ec5df0f | ||
|
|
abcdcf19a4 | ||
|
|
9dc004926d | ||
|
|
3d2f411e1d | ||
|
|
763e490f42 | ||
|
|
421ec70686 | ||
|
|
aac1427cdc | ||
|
|
ecdc9e4a35 | ||
|
|
cc88919631 | ||
|
|
d80cccd076 | ||
|
|
c0ad64ee5a | ||
|
|
c4552c6011 | ||
|
|
4331cd0f7a | ||
|
|
682225e1ad | ||
|
|
b0bfc46e01 | ||
|
|
4489c63f6d | ||
|
|
0993a669e9 | ||
|
|
c565fa9537 | ||
|
|
84e633d7c5 |
No files matched your search
@@ -530,6 +530,16 @@ Description of the template.
|
||||
|
||||
You can go in-depth here on what the template actually does.
|
||||
|
||||
Description, impact and remediation may hold {{placeholders}} filled for
|
||||
each finding: named extractors, payloads, variables and DSL helpers
|
||||
(network helpers excluded), and the result fields host, port, ip, url,
|
||||
scheme, path, matched-at, matcher-name and extractor-name, which win over
|
||||
values of the same name. Use join(", ", name) for an extractor with
|
||||
several values, and brackets for a hyphenated name inside a helper:
|
||||
to_upper([matcher-name]). Unknown placeholders stay as written, and so
|
||||
does a field whose expression fails. Global matcher templates match on
|
||||
other templates' responses, so only the result fields fill their text.
|
||||
|
||||
|
||||
|
||||
Examples:
|
||||
@@ -543,6 +553,10 @@ description: Bower is a package manager which stores package information in the
|
||||
description: Subversion ALM for the enterprise before 8.8.2 allows reflected XSS at multiple locations
|
||||
```
|
||||
|
||||
```yaml
|
||||
description: Redis {{version}} on {{host}}:{{port}} accepts the default password
|
||||
```
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
@@ -559,6 +573,8 @@ Impact of the template.
|
||||
|
||||
You can go in-depth here on impact of the template.
|
||||
|
||||
Supports the same placeholders as description.
|
||||
|
||||
|
||||
|
||||
Examples:
|
||||
@@ -665,6 +681,8 @@ Remediation steps for the template.
|
||||
|
||||
You can go in-depth here on how to mitigate the problem found by this template.
|
||||
|
||||
Supports the same placeholders as description.
|
||||
|
||||
|
||||
|
||||
Examples:
|
||||
|
||||
@@ -57,7 +57,8 @@
|
||||
# severity: low
|
||||
# # duplicate-issue-check (optional) flag to enable duplicate tracking issue check
|
||||
# duplicate-issue-check: false
|
||||
# # duplicate-issue-page-size (optional) controls how many issues to fetch per page when searching for duplicates
|
||||
# # duplicate-issue-page-size (optional) sets the number of issues to request per page when searching for duplicates
|
||||
# # (GitLab returns at most 100 issues per page)
|
||||
# duplicate-issue-page-size: 100
|
||||
# # duplicate-issue-max-pages (optional) limits how many pages to fetch when searching for duplicates (0 = no limit)
|
||||
# duplicate-issue-max-pages: 0
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/invopop/jsonschema"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/marker"
|
||||
)
|
||||
|
||||
type schemaMetadata struct {
|
||||
@@ -47,15 +51,28 @@ type Info struct {
|
||||
//
|
||||
// You can go in-depth here on what the template actually does.
|
||||
//
|
||||
// Description, impact and remediation may hold {{placeholders}} filled for
|
||||
// each finding: named extractors, payloads, variables and DSL helpers
|
||||
// (network helpers excluded), and the result fields host, port, ip, url,
|
||||
// scheme, path, matched-at, matcher-name and extractor-name, which win over
|
||||
// values of the same name. Use join(", ", name) for an extractor with
|
||||
// several values, and brackets for a hyphenated name inside a helper:
|
||||
// to_upper([matcher-name]). Unknown placeholders stay as written, and so
|
||||
// does a field whose expression fails. Global matcher templates match on
|
||||
// other templates' responses, so only the result fields fill their text.
|
||||
//
|
||||
// examples:
|
||||
// - value: "\"Bower is a package manager which stores package information in the bower.json file\""
|
||||
// - value: "\"Subversion ALM for the enterprise before 8.8.2 allows reflected XSS at multiple locations\""
|
||||
// - value: "\"Redis {{version}} on {{host}}:{{port}} accepts the default password\""
|
||||
Description string `json:"description,omitempty" yaml:"description,omitempty" jsonschema:"title=description of the template,description=In-depth explanation on what the template does,type=string,example=Bower is a package manager which stores package information in the bower.json file"`
|
||||
// description: |
|
||||
// Impact of the template.
|
||||
//
|
||||
// You can go in-depth here on impact of the template.
|
||||
//
|
||||
// Supports the same placeholders as description.
|
||||
//
|
||||
// examples:
|
||||
// - value: "\"Successful exploitation of this vulnerability could allow an attacker to execute arbitrary SQL queries, potentially leading to unauthorized access, data leakage, or data manipulation.\""
|
||||
// - value: "\"Successful exploitation of this vulnerability could allow an attacker to execute arbitrary script code in the context of the victim's browser, potentially leading to session hijacking, defacement, or theft of sensitive information.\""
|
||||
@@ -89,11 +106,57 @@ type Info struct {
|
||||
//
|
||||
// You can go in-depth here on how to mitigate the problem found by this template.
|
||||
//
|
||||
// Supports the same placeholders as description.
|
||||
//
|
||||
// examples:
|
||||
// - value: "\"Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties\""
|
||||
Remediation string `json:"remediation,omitempty" yaml:"remediation,omitempty" jsonschema:"title=remediation steps for the template,description=In-depth explanation on how to fix the issues found by the template,example=Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties,type=string"`
|
||||
}
|
||||
|
||||
// TextField is a free-text field of the info, labeled as reports show it.
|
||||
type TextField struct {
|
||||
Label string
|
||||
Value *string
|
||||
}
|
||||
|
||||
// PlaceholderFields returns the free-text fields that may hold {{placeholders}}
|
||||
// filled for each finding. Rendering, redaction and the sarif export all walk
|
||||
// this one list, so a field added here is handled by every one of them.
|
||||
func (i *Info) PlaceholderFields() []TextField {
|
||||
return []TextField{
|
||||
{Label: "Description", Value: &i.Description},
|
||||
{Label: "Impact", Value: &i.Impact},
|
||||
{Label: "Remediation", Value: &i.Remediation},
|
||||
}
|
||||
}
|
||||
|
||||
// HasPlaceholders reports whether the field holds a {{...}} or §...§ marker
|
||||
// the renderer could fill.
|
||||
func (f TextField) HasPlaceholders() bool {
|
||||
return strings.Contains(*f.Value, marker.ParenthesisOpen) || strings.Contains(*f.Value, marker.General)
|
||||
}
|
||||
|
||||
// HasPlaceholders reports whether any placeholder field holds a marker.
|
||||
func (i *Info) HasPlaceholders() bool {
|
||||
return slices.ContainsFunc(i.PlaceholderFields(), TextField.HasPlaceholders)
|
||||
}
|
||||
|
||||
// FilledFields returns the placeholder fields whose text differs from written,
|
||||
// the info as the template wrote it. A nil written means nothing was filled.
|
||||
func (i *Info) FilledFields(written *Info) []TextField {
|
||||
if written == nil {
|
||||
return nil
|
||||
}
|
||||
writtenFields := written.PlaceholderFields()
|
||||
var filled []TextField
|
||||
for n, field := range i.PlaceholderFields() {
|
||||
if *field.Value != *writtenFields[n].Value {
|
||||
filled = append(filled, field)
|
||||
}
|
||||
}
|
||||
return filled
|
||||
}
|
||||
|
||||
// JSONSchemaProperty returns the JSON schema property for the Info object.
|
||||
func (i Info) JSONSchemaExtend(base *jsonschema.Schema) {
|
||||
// since we are re-using a stringslice and rawStringSlice everywhere, we can extend/edit the schema here
|
||||
|
||||
@@ -135,3 +135,21 @@ func TestUnmarshal(t *testing.T) {
|
||||
info2 := assertUnmarshalledTemplateInfo(t, yamlPayload2)
|
||||
require.Equal(t, info1, info2)
|
||||
}
|
||||
|
||||
func TestInfoFilledFields(t *testing.T) {
|
||||
written := Info{Description: "{{host}} is exposed", Impact: "Data loss"}
|
||||
filled := written
|
||||
filled.Description = "example.com is exposed"
|
||||
|
||||
require.Nil(t, filled.FilledFields(nil))
|
||||
fields := filled.FilledFields(&written)
|
||||
require.Len(t, fields, 1)
|
||||
require.Equal(t, "Description", fields[0].Label)
|
||||
require.Equal(t, "example.com is exposed", *fields[0].Value)
|
||||
}
|
||||
|
||||
func TestInfoHasPlaceholders(t *testing.T) {
|
||||
require.False(t, (&Info{Description: "Redis is exposed"}).HasPlaceholders())
|
||||
require.True(t, (&Info{Impact: "{{host}} is exposed"}).HasPlaceholders())
|
||||
require.True(t, (&Info{Remediation: "rotate §password§"}).HasPlaceholders())
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package operators
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHasErrorMatchers(t *testing.T) {
|
||||
t.Run("error type", func(t *testing.T) {
|
||||
ops := &Operators{Matchers: []*matchers.Matcher{{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
|
||||
Errors: []string{"timeout"},
|
||||
}}}
|
||||
require.NoError(t, ops.Compile())
|
||||
require.True(t, ops.HasErrorMatchers())
|
||||
})
|
||||
|
||||
t.Run("dsl timeout var", func(t *testing.T) {
|
||||
ops := &Operators{Matchers: []*matchers.Matcher{{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
|
||||
DSL: []string{"timeout == true"},
|
||||
}}}
|
||||
require.NoError(t, ops.Compile())
|
||||
require.True(t, ops.HasErrorMatchers())
|
||||
})
|
||||
|
||||
t.Run("dsl error_type var", func(t *testing.T) {
|
||||
ops := &Operators{Matchers: []*matchers.Matcher{{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
|
||||
DSL: []string{`error_type == "timeout"`},
|
||||
}}}
|
||||
require.NoError(t, ops.Compile())
|
||||
require.True(t, ops.HasErrorMatchers())
|
||||
})
|
||||
|
||||
t.Run("dsl error var", func(t *testing.T) {
|
||||
ops := &Operators{Matchers: []*matchers.Matcher{{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
|
||||
DSL: []string{`contains(error, "refused")`},
|
||||
}}}
|
||||
require.NoError(t, ops.Compile())
|
||||
require.True(t, ops.HasErrorMatchers())
|
||||
})
|
||||
|
||||
t.Run("dsl without error vars", func(t *testing.T) {
|
||||
ops := &Operators{Matchers: []*matchers.Matcher{{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
|
||||
DSL: []string{"status_code == 200"},
|
||||
}}}
|
||||
require.NoError(t, ops.Compile())
|
||||
require.False(t, ops.HasErrorMatchers())
|
||||
})
|
||||
|
||||
t.Run("dsl timeout as identifier prefix does not match", func(t *testing.T) {
|
||||
ops := &Operators{Matchers: []*matchers.Matcher{{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
|
||||
DSL: []string{"timeout_ms > 100"},
|
||||
}}}
|
||||
require.NoError(t, ops.Compile())
|
||||
require.False(t, ops.HasErrorMatchers())
|
||||
})
|
||||
|
||||
t.Run("dsl quoted error fields do not match", func(t *testing.T) {
|
||||
for _, expression := range []string{
|
||||
`contains(body, "timeout")`,
|
||||
`contains(body, 'error')`,
|
||||
`contains(body, "escaped \"timeout\"")`,
|
||||
} {
|
||||
ops := &Operators{Matchers: []*matchers.Matcher{{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
|
||||
DSL: []string{expression},
|
||||
}}}
|
||||
require.NoError(t, ops.Compile())
|
||||
require.False(t, ops.HasErrorMatchers(), expression)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("dsl indexed error fields match", func(t *testing.T) {
|
||||
for _, expression := range []string{
|
||||
"timeout_2 == true",
|
||||
`error_type_10 == "connection"`,
|
||||
`contains(error_3, "refused")`,
|
||||
} {
|
||||
ops := &Operators{Matchers: []*matchers.Matcher{{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
|
||||
DSL: []string{expression},
|
||||
}}}
|
||||
require.NoError(t, ops.Compile())
|
||||
require.True(t, ops.HasErrorMatchers(), expression)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("word matcher only", func(t *testing.T) {
|
||||
ops := &Operators{Matchers: []*matchers.Matcher{{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
|
||||
Words: []string{"ok"},
|
||||
Part: "body",
|
||||
}}}
|
||||
require.NoError(t, ops.Compile())
|
||||
require.False(t, ops.HasErrorMatchers())
|
||||
})
|
||||
|
||||
t.Run("nil operators", func(t *testing.T) {
|
||||
var ops *Operators
|
||||
require.False(t, ops.HasErrorMatchers())
|
||||
})
|
||||
}
|
||||
@@ -38,7 +38,7 @@ func (matcher *Matcher) CompileMatchers() error {
|
||||
}
|
||||
|
||||
// By default, match on body if user hasn't provided any specific items
|
||||
if matcher.Part == "" && matcher.GetType() != DSLMatcher {
|
||||
if matcher.Part == "" && matcher.GetType() != DSLMatcher && matcher.GetType() != ErrorMatcher {
|
||||
matcher.Part = "body"
|
||||
}
|
||||
|
||||
@@ -90,17 +90,34 @@ func (matcher *Matcher) CompileMatchers() error {
|
||||
}
|
||||
|
||||
if matcher.CaseInsensitive {
|
||||
if matcher.GetType() != WordsMatcher {
|
||||
return fmt.Errorf("case-insensitive flag is supported only for 'word' matchers (not '%s')", matcher.Type)
|
||||
}
|
||||
for i := range matcher.Words {
|
||||
matcher.Words[i] = strings.ToLower(matcher.Words[i])
|
||||
switch matcher.GetType() {
|
||||
case WordsMatcher:
|
||||
for i := range matcher.Words {
|
||||
matcher.Words[i] = strings.ToLower(matcher.Words[i])
|
||||
}
|
||||
case ErrorMatcher:
|
||||
// Applied at MatchError time so reserved kinds (timeout/connection/any) stay intact.
|
||||
default:
|
||||
return fmt.Errorf("case-insensitive flag is supported only for 'word' and 'error' matchers (not '%s')", matcher.Type)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetType returns the condition type of the matcher
|
||||
// NeedsPart returns true when the matcher requires a response part corpus.
|
||||
func (matcher *Matcher) NeedsPart() bool {
|
||||
switch matcher.GetType() {
|
||||
case DSLMatcher, ErrorMatcher:
|
||||
return false
|
||||
case LLMMatcher:
|
||||
// an llm matcher with inputs compares whole responses instead of one part
|
||||
return len(matcher.Inputs) == 0
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// GetCondition returns the condition type of the matcher
|
||||
// todo: the field should be exposed natively
|
||||
func (matcher *Matcher) GetCondition() ConditionType {
|
||||
return matcher.condition
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
package matchers
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMatchError(t *testing.T) {
|
||||
t.Run("any error when errors empty", func(t *testing.T) {
|
||||
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}}
|
||||
require.NoError(t, m.CompileMatchers())
|
||||
ok, snippets := m.MatchError(map[string]interface{}{"error": "boom", "error_type": "unknown", "timeout": false})
|
||||
require.True(t, m.Result(ok))
|
||||
require.Equal(t, []string{"boom"}, snippets)
|
||||
ok, _ = m.MatchError(map[string]interface{}{})
|
||||
require.False(t, m.Result(ok))
|
||||
})
|
||||
|
||||
t.Run("empty error string does not match", func(t *testing.T) {
|
||||
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}, Errors: []string{"any"}}
|
||||
require.NoError(t, m.CompileMatchers())
|
||||
ok, _ := m.MatchError(map[string]interface{}{"error": "", "error_type": "unknown", "timeout": false})
|
||||
require.False(t, m.Result(ok))
|
||||
ok, _ = m.MatchError(map[string]interface{}{"error": nil})
|
||||
require.False(t, m.Result(ok))
|
||||
})
|
||||
|
||||
t.Run("timeout kind", func(t *testing.T) {
|
||||
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}, Errors: []string{"timeout"}}
|
||||
require.NoError(t, m.CompileMatchers())
|
||||
ok, _ := m.MatchError(map[string]interface{}{"error": "i/o timeout", "error_type": "timeout", "timeout": true})
|
||||
require.True(t, m.Result(ok))
|
||||
ok, _ = m.MatchError(map[string]interface{}{"error": "connection refused", "error_type": "connection", "timeout": false})
|
||||
require.False(t, m.Result(ok))
|
||||
})
|
||||
|
||||
t.Run("connection and connect kinds", func(t *testing.T) {
|
||||
for _, kind := range []string{"connection", "connect"} {
|
||||
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}, Errors: []string{kind}}
|
||||
require.NoError(t, m.CompileMatchers())
|
||||
ok, _ := m.MatchError(map[string]interface{}{"error": "connection refused", "error_type": "connection", "timeout": false})
|
||||
require.True(t, m.Result(ok), "kind %q", kind)
|
||||
ok, _ = m.MatchError(map[string]interface{}{"error": "i/o timeout", "error_type": "timeout", "timeout": true})
|
||||
require.False(t, m.Result(ok), "kind %q", kind)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("any and star kinds", func(t *testing.T) {
|
||||
for _, kind := range []string{"any", "*"} {
|
||||
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}, Errors: []string{kind}}
|
||||
require.NoError(t, m.CompileMatchers())
|
||||
ok, snippets := m.MatchError(map[string]interface{}{"error": "something went wrong", "error_type": "unknown", "timeout": false})
|
||||
require.True(t, m.Result(ok), "kind %q", kind)
|
||||
require.Equal(t, []string{"something went wrong"}, snippets)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("substring in error message", func(t *testing.T) {
|
||||
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}, Errors: []string{"connection refused"}}
|
||||
require.NoError(t, m.CompileMatchers())
|
||||
ok, snippets := m.MatchError(map[string]interface{}{"error": "dial tcp 127.0.0.1:1: connection refused", "error_type": "connection", "timeout": false})
|
||||
require.True(t, m.Result(ok))
|
||||
require.Equal(t, []string{"connection refused"}, snippets)
|
||||
|
||||
ok, _ = m.MatchError(map[string]interface{}{"error": "i/o timeout", "error_type": "timeout", "timeout": true})
|
||||
require.False(t, m.Result(ok))
|
||||
})
|
||||
|
||||
t.Run("case insensitive substring", func(t *testing.T) {
|
||||
m := &Matcher{
|
||||
Type: MatcherTypeHolder{MatcherType: ErrorMatcher},
|
||||
Errors: []string{"No Such Host"},
|
||||
CaseInsensitive: true,
|
||||
}
|
||||
require.NoError(t, m.CompileMatchers())
|
||||
ok, snippets := m.MatchError(map[string]interface{}{"error": "lookup example.invalid: no such host", "error_type": "connection", "timeout": false})
|
||||
require.True(t, m.Result(ok))
|
||||
require.Equal(t, []string{"No Such Host"}, snippets)
|
||||
})
|
||||
|
||||
t.Run("case sensitive substring misses", func(t *testing.T) {
|
||||
m := &Matcher{
|
||||
Type: MatcherTypeHolder{MatcherType: ErrorMatcher},
|
||||
Errors: []string{"No Such Host"},
|
||||
}
|
||||
require.NoError(t, m.CompileMatchers())
|
||||
ok, _ := m.MatchError(map[string]interface{}{"error": "lookup example.invalid: no such host", "error_type": "connection", "timeout": false})
|
||||
require.False(t, m.Result(ok))
|
||||
})
|
||||
|
||||
t.Run("or condition across kind and substring", func(t *testing.T) {
|
||||
m := &Matcher{
|
||||
Type: MatcherTypeHolder{MatcherType: ErrorMatcher},
|
||||
Errors: []string{"timeout", "no such host"},
|
||||
Condition: "or",
|
||||
}
|
||||
require.NoError(t, m.CompileMatchers())
|
||||
ok, _ := m.MatchError(map[string]interface{}{"error": "lookup x: no such host", "error_type": "connection", "timeout": false})
|
||||
require.True(t, m.Result(ok))
|
||||
ok, _ = m.MatchError(map[string]interface{}{"error": "i/o timeout", "error_type": "timeout", "timeout": true})
|
||||
require.True(t, m.Result(ok))
|
||||
ok, _ = m.MatchError(map[string]interface{}{"error": "broken pipe", "error_type": "connection", "timeout": false})
|
||||
require.False(t, m.Result(ok))
|
||||
})
|
||||
|
||||
t.Run("and condition requires all", func(t *testing.T) {
|
||||
m := &Matcher{
|
||||
Type: MatcherTypeHolder{MatcherType: ErrorMatcher},
|
||||
Errors: []string{"dial tcp", "connection refused"},
|
||||
Condition: "and",
|
||||
}
|
||||
require.NoError(t, m.CompileMatchers())
|
||||
ok, snippets := m.MatchError(map[string]interface{}{"error": "dial tcp 127.0.0.1:1: connection refused", "error_type": "connection", "timeout": false})
|
||||
require.True(t, m.Result(ok))
|
||||
require.Equal(t, []string{"dial tcp", "connection refused"}, snippets)
|
||||
|
||||
ok, _ = m.MatchError(map[string]interface{}{"error": "dial tcp: i/o timeout", "error_type": "timeout", "timeout": true})
|
||||
require.False(t, m.Result(ok))
|
||||
})
|
||||
|
||||
t.Run("match-all returns every matching snippet", func(t *testing.T) {
|
||||
m := &Matcher{
|
||||
Type: MatcherTypeHolder{MatcherType: ErrorMatcher},
|
||||
Errors: []string{"dial tcp", "connection refused", "missing"},
|
||||
Condition: "or",
|
||||
MatchAll: true,
|
||||
}
|
||||
require.NoError(t, m.CompileMatchers())
|
||||
ok, snippets := m.MatchError(map[string]interface{}{"error": "dial tcp 127.0.0.1:1: connection refused", "error_type": "connection", "timeout": false})
|
||||
require.True(t, m.Result(ok))
|
||||
require.Equal(t, []string{"dial tcp", "connection refused"}, snippets)
|
||||
})
|
||||
|
||||
t.Run("negative", func(t *testing.T) {
|
||||
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}, Errors: []string{"timeout"}, Negative: true}
|
||||
require.NoError(t, m.CompileMatchers())
|
||||
ok, snippets := m.MatchError(map[string]interface{}{"error": "i/o timeout", "error_type": "timeout", "timeout": true})
|
||||
matched, out := m.ResultWithMatchedSnippet(ok, snippets)
|
||||
require.False(t, matched)
|
||||
require.Empty(t, out)
|
||||
ok, _ = m.MatchError(map[string]interface{}{})
|
||||
require.True(t, m.Result(ok))
|
||||
})
|
||||
}
|
||||
|
||||
func TestHasErrorMatchersViaOperatorsPackage(t *testing.T) {
|
||||
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}, Errors: []string{"timeout"}}
|
||||
require.NoError(t, m.CompileMatchers())
|
||||
require.Equal(t, ErrorMatcher, m.GetType())
|
||||
require.False(t, m.NeedsPart())
|
||||
}
|
||||
@@ -13,7 +13,7 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
fuzzMatcherTypes = []MatcherType{WordsMatcher, RegexMatcher, BinaryMatcher, StatusMatcher, SizeMatcher, DSLMatcher, XPathMatcher}
|
||||
fuzzMatcherTypes = []MatcherType{WordsMatcher, RegexMatcher, BinaryMatcher, StatusMatcher, SizeMatcher, DSLMatcher, XPathMatcher, ErrorMatcher}
|
||||
fuzzConditions = []string{"", "and", "or"}
|
||||
fuzzEncodings = []string{"", "hex"}
|
||||
fuzzParts = []string{"", "body", "raw", "all_headers", "header", "response"}
|
||||
@@ -193,10 +193,18 @@ func (candidate *fuzzMatcherCandidate) build() (*Matcher, bool) {
|
||||
matcher.Status = append([]int(nil), candidate.status...)
|
||||
case SizeMatcher:
|
||||
matcher.Size = append([]int(nil), candidate.size...)
|
||||
case ErrorMatcher:
|
||||
matcher.Part = ""
|
||||
matcher.Encoding = ""
|
||||
matcher.CaseInsensitive = false
|
||||
matcher.Errors = append([]string(nil), candidate.values...)
|
||||
default:
|
||||
matcher.Words = append([]string(nil), candidate.values...)
|
||||
}
|
||||
|
||||
if matcher.GetType() == ErrorMatcher {
|
||||
return matcher, true
|
||||
}
|
||||
if matcher.GetType() == StatusMatcher || matcher.GetType() == SizeMatcher {
|
||||
return matcher, len(matcher.Status) > 0 || len(matcher.Size) > 0
|
||||
}
|
||||
|
||||
@@ -261,6 +261,93 @@ func (matcher *Matcher) MatchDSLWithOptions(data map[string]interface{}, options
|
||||
return false
|
||||
}
|
||||
|
||||
// MatchError matches request execution errors exposed on the event map.
|
||||
//
|
||||
// Errors values may be:
|
||||
// - kinds: timeout, connection, connect, any, * (classify via error_type / timeout)
|
||||
// - any other string: substring match against the error message
|
||||
//
|
||||
// An empty Errors list matches any request error. Condition (and/or), MatchAll,
|
||||
// and CaseInsensitive apply the same way as word matchers.
|
||||
func (matcher *Matcher) MatchError(data map[string]interface{}) (bool, []string) {
|
||||
errVal, ok := data["error"]
|
||||
if !ok || errVal == nil {
|
||||
return false, []string{}
|
||||
}
|
||||
errStr, _ := errVal.(string)
|
||||
if errStr == "" {
|
||||
return false, []string{}
|
||||
}
|
||||
|
||||
if len(matcher.Errors) == 0 {
|
||||
return true, []string{errStr}
|
||||
}
|
||||
|
||||
corpus := errStr
|
||||
if matcher.CaseInsensitive {
|
||||
corpus = strings.ToLower(corpus)
|
||||
}
|
||||
|
||||
errType, _ := data["error_type"].(string)
|
||||
isTimeout, _ := data["timeout"].(bool)
|
||||
|
||||
var matched []string
|
||||
for i, item := range matcher.Errors {
|
||||
item = strings.TrimSpace(item)
|
||||
if item == "" {
|
||||
if matcher.condition == ANDCondition {
|
||||
return false, []string{}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
okMatch, snippet := matcher.matchErrorItem(item, errStr, corpus, errType, isTimeout)
|
||||
if !okMatch {
|
||||
if matcher.condition == ANDCondition {
|
||||
return false, []string{}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if matcher.condition == ORCondition && !matcher.MatchAll {
|
||||
return true, []string{snippet}
|
||||
}
|
||||
matched = append(matched, snippet)
|
||||
|
||||
if len(matcher.Errors)-1 == i && !matcher.MatchAll {
|
||||
return true, matched
|
||||
}
|
||||
}
|
||||
if len(matched) > 0 && matcher.MatchAll {
|
||||
return true, matched
|
||||
}
|
||||
return false, []string{}
|
||||
}
|
||||
|
||||
func (matcher *Matcher) matchErrorItem(item, errStr, corpus, errType string, isTimeout bool) (bool, string) {
|
||||
switch strings.ToLower(item) {
|
||||
case "timeout":
|
||||
if isTimeout || errType == "timeout" {
|
||||
return true, item
|
||||
}
|
||||
case "connection", "connect":
|
||||
if errType == "connection" {
|
||||
return true, item
|
||||
}
|
||||
case "any", "*":
|
||||
return true, errStr
|
||||
default:
|
||||
needle := item
|
||||
if matcher.CaseInsensitive {
|
||||
needle = strings.ToLower(item)
|
||||
}
|
||||
if strings.Contains(corpus, needle) {
|
||||
return true, item
|
||||
}
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
// MatchXPath matches on a generic map result
|
||||
func (matcher *Matcher) MatchXPath(corpus string) bool {
|
||||
if strings.HasPrefix(corpus, "<?xml") {
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
type Matcher struct {
|
||||
// description: |
|
||||
// Type is the type of the matcher.
|
||||
Type MatcherTypeHolder `yaml:"type" json:"type" jsonschema:"title=type of matcher,description=Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl"`
|
||||
Type MatcherTypeHolder `yaml:"type" json:"type" jsonschema:"title=type of matcher,description=Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl,enum=xpath,enum=error"`
|
||||
// description: |
|
||||
// Condition is the optional condition between two matcher variables. By default,
|
||||
// the condition is assumed to be OR.
|
||||
@@ -105,6 +105,20 @@ type Matcher struct {
|
||||
// []string{"//a[@target=\"_blank\"]"}
|
||||
XPath []string `yaml:"xpath,omitempty" json:"xpath,omitempty" jsonschema:"title=xpath queries to match in response,description=xpath are the XPath queries that will be evaluated against the response part of nuclei matching rules"`
|
||||
// description: |
|
||||
// Errors are patterns matched against a failed request (no successful response).
|
||||
//
|
||||
// Reserved kinds (matched via error classification): timeout, connection, connect, any, *.
|
||||
// Any other value is treated as a substring of the error message.
|
||||
// When empty, any request error matches.
|
||||
// examples:
|
||||
// - name: Match request timeouts by kind
|
||||
// value: >
|
||||
// []string{"timeout"}
|
||||
// - name: Match a substring in the error message
|
||||
// value: >
|
||||
// []string{"connection refused"}
|
||||
Errors []string `yaml:"errors,omitempty" json:"errors,omitempty" jsonschema:"title=error patterns to match,description=Error kinds (timeout, connection, connect, any, *) or substrings of the error message"`
|
||||
// description: |
|
||||
// Prompt is the natural-language question for an llm matcher. The model
|
||||
// judges the selected response part and returns a verdict; the matcher
|
||||
// fires when the verdict equals Expect with at least MinConfidence.
|
||||
|
||||
@@ -27,6 +27,8 @@ const (
|
||||
DSLMatcher
|
||||
// name:xpath
|
||||
XPathMatcher
|
||||
// name:error
|
||||
ErrorMatcher
|
||||
// name:llm
|
||||
LLMMatcher
|
||||
limit
|
||||
@@ -41,6 +43,7 @@ var MatcherTypes = map[MatcherType]string{
|
||||
BinaryMatcher: "binary",
|
||||
DSLMatcher: "dsl",
|
||||
XPathMatcher: "xpath",
|
||||
ErrorMatcher: "error",
|
||||
LLMMatcher: "llm",
|
||||
}
|
||||
|
||||
|
||||
@@ -57,6 +57,8 @@ func (matcher *Matcher) Validate() error {
|
||||
case XPathMatcher:
|
||||
requiredField, valueCount = "xpath", len(matcher.XPath)
|
||||
expectedFields = append(commonExpectedFields, "XPath", "Part")
|
||||
case ErrorMatcher:
|
||||
expectedFields = append(commonExpectedFields, "Errors", "CaseInsensitive")
|
||||
case LLMMatcher:
|
||||
requiredField, valueCount = "prompt", len(matcher.Prompt)
|
||||
expectedFields = append(commonExpectedFields, "Prompt", "Expect", "Options", "MinConfidence", "MaxInputTokens", "AllowSole", "Part", "Inputs")
|
||||
|
||||
@@ -34,6 +34,10 @@ func TestValidateRequiresValues(t *testing.T) {
|
||||
for _, matcherType := range GetSupportedMatcherTypes() {
|
||||
t.Run(matcherType.String(), func(t *testing.T) {
|
||||
m := &Matcher{matcherType: matcherType}
|
||||
if matcherType == ErrorMatcher {
|
||||
require.NoError(t, m.Validate(), "an empty error matcher matches any request error")
|
||||
return
|
||||
}
|
||||
require.ErrorContains(t, m.Validate(), "requires at least one")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -81,6 +81,93 @@ func (operators *Operators) HasDSL() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// HasErrorMatchers reports whether operators include an error matcher or a DSL
|
||||
// expression that references request-error fields (timeout / error / error_type).
|
||||
// When true, protocols should still run matchers on failed requests and should
|
||||
// not skip the template solely because the host is marked unresponsive.
|
||||
func (operators *Operators) HasErrorMatchers() bool {
|
||||
if operators == nil {
|
||||
return false
|
||||
}
|
||||
for _, matcher := range operators.Matchers {
|
||||
if matcher.GetType() == matchers.ErrorMatcher {
|
||||
return true
|
||||
}
|
||||
if matcher.GetType() == matchers.DSLMatcher {
|
||||
for _, expr := range matcher.DSL {
|
||||
if dslReferencesRequestError(expr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func dslReferencesRequestError(expr string) bool {
|
||||
for i := 0; i < len(expr); {
|
||||
if expr[i] == '\'' || expr[i] == '"' || expr[i] == '`' {
|
||||
quote := expr[i]
|
||||
i++
|
||||
for i < len(expr) {
|
||||
if expr[i] == '\\' && quote != '`' {
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
if expr[i] == quote {
|
||||
i++
|
||||
break
|
||||
}
|
||||
i++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !isIdentByte(expr[i]) || (expr[i] >= '0' && expr[i] <= '9') {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
start := i
|
||||
for i < len(expr) && isIdentByte(expr[i]) {
|
||||
i++
|
||||
}
|
||||
if isRequestErrorIdentifier(strings.ToLower(expr[start:i])) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isRequestErrorIdentifier(identifier string) bool {
|
||||
for _, field := range []string{"timeout", "error_type", "error"} {
|
||||
if identifier == field {
|
||||
return true
|
||||
}
|
||||
prefix := field + "_"
|
||||
if !strings.HasPrefix(identifier, prefix) {
|
||||
continue
|
||||
}
|
||||
suffix := identifier[len(prefix):]
|
||||
if suffix == "" {
|
||||
continue
|
||||
}
|
||||
indexed := true
|
||||
for i := 0; i < len(suffix); i++ {
|
||||
if suffix[i] < '0' || suffix[i] > '9' {
|
||||
indexed = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if indexed {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isIdentByte(b byte) bool {
|
||||
return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_'
|
||||
}
|
||||
|
||||
// GetMatchersCondition returns the condition for the matchers
|
||||
func (operators *Operators) GetMatchersCondition() matchers.ConditionType {
|
||||
return operators.matchersCondition
|
||||
|
||||
+11
-1
@@ -237,6 +237,11 @@ type ResultEvent struct {
|
||||
FileToIndexPosition map[string]int `json:"-"`
|
||||
TemplateVerifier string `json:"-"`
|
||||
Error string `json:"error,omitempty"`
|
||||
// TemplateInfo is the info as the template wrote it, kept only when
|
||||
// placeholders changed the text. The changed fields can carry response data,
|
||||
// so they are redacted like the raw request and response. It is bookkeeping,
|
||||
// so no exporter writes it; mongo serializes with bson tags, not json ones.
|
||||
TemplateInfo *model.Info `json:"-" bson:"-"`
|
||||
}
|
||||
|
||||
type IssueTrackerMetadata struct {
|
||||
@@ -324,7 +329,9 @@ func (w *StandardWriter) ResultCount() int {
|
||||
|
||||
// Write writes the event to file and/or screen.
|
||||
func (w *StandardWriter) Write(event *ResultEvent) error {
|
||||
if event.Error != "" && !w.matcherStatus {
|
||||
// failure events (skipped or errored requests) are only written with -ms;
|
||||
// a matched result can carry the request error, e.g. from an error matcher
|
||||
if event.Error != "" && !event.MatcherStatus && !w.matcherStatus {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -364,6 +371,9 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
|
||||
event.Response = redactWithPatterns(event.Response, patterns)
|
||||
event.CURLCommand = redactWithPatterns(event.CURLCommand, patterns)
|
||||
event.Matched = redactWithPatterns(event.Matched, patterns)
|
||||
for _, field := range event.Info.FilledFields(event.TemplateInfo) {
|
||||
*field.Value = redactWithPatterns(*field.Value, patterns)
|
||||
}
|
||||
|
||||
event.Timestamp = time.Now()
|
||||
|
||||
|
||||
@@ -6,8 +6,11 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
)
|
||||
|
||||
func TestStandardWriterRequest(t *testing.T) {
|
||||
@@ -51,6 +54,51 @@ func TestStandardWriterRequest(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestStandardWriterWriteErroredEvents(t *testing.T) {
|
||||
outputWriter := &testWriteCloser{}
|
||||
w, err := NewStandardWriter(&types.Options{})
|
||||
require.NoError(t, err)
|
||||
w.outputFile = outputWriter
|
||||
|
||||
// a failure event (skipped or errored request) is only written with -ms
|
||||
require.NoError(t, w.Write(&ResultEvent{TemplateID: "failed", Error: "connection refused"}))
|
||||
require.Empty(t, outputWriter.String())
|
||||
|
||||
// a matched result can carry the request error, e.g. from an error matcher
|
||||
require.NoError(t, w.Write(&ResultEvent{TemplateID: "matched", MatcherStatus: true, Error: "connection refused"}))
|
||||
require.Contains(t, outputWriter.String(), "matched")
|
||||
}
|
||||
|
||||
// Only the fields filled from a response are redacted; text the template wrote
|
||||
// stays as it is, even next to a filled field.
|
||||
func TestStandardWriterRedactsFilledInfo(t *testing.T) {
|
||||
w, err := NewStandardWriter(&types.Options{Redact: goflags.StringSlice{"password"}})
|
||||
require.NoError(t, err)
|
||||
w.DisableStdout = true
|
||||
|
||||
written := model.Info{Description: "login with password: {{password}}", Remediation: "Rotate the password: it is public"}
|
||||
filled := &ResultEvent{TemplateInfo: &written, Info: model.Info{Description: "login with password: admin", Remediation: written.Remediation}}
|
||||
static := &ResultEvent{Info: model.Info{Description: "send the password: header"}}
|
||||
require.NoError(t, w.Write(filled))
|
||||
require.NoError(t, w.Write(static))
|
||||
|
||||
require.Equal(t, "login with password: ***", filled.Info.Description)
|
||||
require.Equal(t, "Rotate the password: it is public", filled.Info.Remediation)
|
||||
require.Equal(t, "send the password: header", static.Info.Description)
|
||||
}
|
||||
|
||||
// The mongo exporter serializes results with bson, which ignores json tags.
|
||||
func TestResultEventKeepsTemplateInfoOutOfBSON(t *testing.T) {
|
||||
written := model.Info{Description: "{{host}} is exposed"}
|
||||
data, err := bson.Marshal(&ResultEvent{TemplateInfo: &written, Info: model.Info{Description: "example.com is exposed"}})
|
||||
require.NoError(t, err)
|
||||
|
||||
var doc bson.M
|
||||
require.NoError(t, bson.Unmarshal(data, &doc))
|
||||
require.NotContains(t, doc, "templateinfo")
|
||||
require.Contains(t, doc, "info")
|
||||
}
|
||||
|
||||
type testWriteCloser struct {
|
||||
strings.Builder
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
contextutil "github.com/projectdiscovery/utils/context"
|
||||
@@ -349,6 +350,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
||||
if request.options.Interactsh != nil {
|
||||
request.options.Interactsh.MakePlaceholders(interactshURLs, data)
|
||||
}
|
||||
if err != nil {
|
||||
requesterr.Annotate(data, err, 0)
|
||||
}
|
||||
|
||||
// todo #1: interactsh async callback should be eliminated as it lead to ton of code duplication
|
||||
// todo #2: various structs InternalWrappedEvent, InternalEvent should be unwrapped and merged into minimal callbacks and a unique struct (eg. event?)
|
||||
|
||||
@@ -18,6 +18,8 @@ func (request *Request) TmplClusterKey() uint64 {
|
||||
}
|
||||
|
||||
// IsClusterable returns true if the request is eligible to be clustered.
|
||||
// Error matchers are excluded: the cluster executer only shares responses, so
|
||||
// a failed request would never reach them.
|
||||
func (request *Request) IsClusterable() bool {
|
||||
return len(request.Resolvers) <= 0 && !request.Trace && request.ID == ""
|
||||
return len(request.Resolvers) <= 0 && !request.Trace && request.ID == "" && !request.HasErrorMatchers()
|
||||
}
|
||||
@@ -21,8 +21,7 @@ import (
|
||||
// Match matches a generic data response against a given matcher
|
||||
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
|
||||
item, ok := request.getMatchPart(matcher.Part, data)
|
||||
// an llm matcher with inputs compares whole responses instead of one part
|
||||
if !ok && matcher.Type.MatcherType != matchers.DSLMatcher && len(matcher.Inputs) == 0 {
|
||||
if !ok && matcher.NeedsPart() {
|
||||
return false, []string{}
|
||||
}
|
||||
|
||||
@@ -45,6 +44,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
|
||||
return matcher.Result(matcher.MatchDSLWithOptions(data, request.options.GetOptions())), []string{}
|
||||
case matchers.XPathMatcher:
|
||||
return matcher.Result(matcher.MatchXPath(types.ToString(item))), []string{}
|
||||
case matchers.ErrorMatcher:
|
||||
return matcher.ResultWithMatchedSnippet(matcher.MatchError(data))
|
||||
case matchers.LLMMatcher:
|
||||
return protocols.MatchLLM(data, matcher, types.ToString(item), request.options)
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/retryabledns"
|
||||
iputil "github.com/projectdiscovery/utils/ip"
|
||||
@@ -186,6 +187,23 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad
|
||||
request.options.Progress.IncrementRequests()
|
||||
}
|
||||
if response == nil {
|
||||
if err != nil && request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers() {
|
||||
outputEvent := output.InternalEvent{
|
||||
"host": domain,
|
||||
"matched": question,
|
||||
"request": requestString,
|
||||
"duration": duration.Seconds(),
|
||||
"template-id": request.options.TemplateID,
|
||||
"template-info": request.options.TemplateInfo,
|
||||
"template-path": request.options.TemplatePath,
|
||||
"type": request.Type().String(),
|
||||
}
|
||||
maps.Copy(outputEvent, previous)
|
||||
maps.Copy(outputEvent, vars)
|
||||
requesterr.Annotate(outputEvent, err, duration)
|
||||
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
|
||||
callback(event)
|
||||
}
|
||||
return errors.Wrap(err, "could not send dns request")
|
||||
}
|
||||
|
||||
|
||||
@@ -15,8 +15,7 @@ import (
|
||||
// Match matches a generic data response again a given matcher
|
||||
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
|
||||
itemStr, ok := request.getMatchPart(matcher.Part, data)
|
||||
// an llm matcher with inputs compares whole responses instead of one part
|
||||
if !ok && matcher.Type.MatcherType != matchers.DSLMatcher && len(matcher.Inputs) == 0 {
|
||||
if !ok && matcher.NeedsPart() {
|
||||
return false, []string{}
|
||||
}
|
||||
|
||||
@@ -33,6 +32,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
|
||||
return matcher.Result(matcher.MatchDSLWithOptions(data, request.options.GetOptions())), []string{}
|
||||
case matchers.XPathMatcher:
|
||||
return matcher.Result(matcher.MatchXPath(itemStr)), []string{}
|
||||
case matchers.ErrorMatcher:
|
||||
return matcher.ResultWithMatchedSnippet(matcher.MatchError(data))
|
||||
case matchers.LLMMatcher:
|
||||
return protocols.MatchLLM(data, matcher, itemStr, request.options)
|
||||
}
|
||||
|
||||
@@ -17,8 +17,7 @@ import (
|
||||
// Match matches a generic data response again a given matcher
|
||||
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
|
||||
itemStr, ok := request.getMatchPart(matcher.Part, data)
|
||||
// an llm matcher with inputs compares whole responses instead of one part
|
||||
if !ok && matcher.Type.MatcherType != matchers.DSLMatcher && len(matcher.Inputs) == 0 {
|
||||
if !ok && matcher.NeedsPart() {
|
||||
return false, []string{}
|
||||
}
|
||||
|
||||
@@ -41,6 +40,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
|
||||
return matcher.Result(matcher.MatchDSLWithOptions(data, request.options.GetOptions())), []string{}
|
||||
case matchers.XPathMatcher:
|
||||
return matcher.Result(matcher.MatchXPath(itemStr)), []string{}
|
||||
case matchers.ErrorMatcher:
|
||||
return matcher.ResultWithMatchedSnippet(matcher.MatchError(data))
|
||||
case matchers.LLMMatcher:
|
||||
return protocols.MatchLLM(data, matcher, itemStr, request.options)
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
|
||||
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
@@ -188,6 +189,32 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
|
||||
if err != nil {
|
||||
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
|
||||
request.options.Progress.IncrementFailedRequestsBy(1)
|
||||
hasErrorMatchers := request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers()
|
||||
if hasErrorMatchers || len(interactshURLs) > 0 {
|
||||
outputEvent := request.responseToDSLMap("", "", "", "", input.MetaInput.Input, input.MetaInput.Input, "")
|
||||
outputEvent["duration"] = runDuration.Seconds()
|
||||
maps.Copy(outputEvent, payloads)
|
||||
requesterr.Annotate(outputEvent, err, runDuration)
|
||||
var event *output.InternalWrappedEvent
|
||||
if hasErrorMatchers {
|
||||
event = eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
|
||||
} else {
|
||||
event = &output.InternalWrappedEvent{InternalEvent: outputEvent}
|
||||
}
|
||||
if len(interactshURLs) > 0 && request.options.Interactsh != nil {
|
||||
event.UsesInteractsh = true
|
||||
request.options.RegisterInteractshRequest(interactshURLs, &interactsh.RequestData{
|
||||
MakeResultFunc: request.MakeResultEvent,
|
||||
Event: event,
|
||||
Operators: request.CompiledOperators,
|
||||
MatchFunc: request.Match,
|
||||
ExtractFunc: request.Extract,
|
||||
})
|
||||
}
|
||||
if hasErrorMatchers {
|
||||
callback(event)
|
||||
}
|
||||
}
|
||||
return errors.Wrap(err, errCouldNotGetHtmlElement)
|
||||
}
|
||||
page = createdPage
|
||||
|
||||
@@ -16,7 +16,9 @@ func (request *Request) TmplClusterKey() uint64 {
|
||||
}
|
||||
|
||||
// IsClusterable returns true if the request is eligible to be clustered.
|
||||
// Error matchers are excluded: the cluster executer only shares responses, so
|
||||
// a failed request would never reach them.
|
||||
func (request *Request) IsClusterable() bool {
|
||||
//nolint
|
||||
return !(len(request.Payloads) > 0 || len(request.Fuzzing) > 0 || len(request.Raw) > 0 || len(request.Body) > 0 || request.Unsafe || request.NeedsRequestCondition() || request.Name != "")
|
||||
return !(len(request.Payloads) > 0 || len(request.Fuzzing) > 0 || len(request.Raw) > 0 || len(request.Body) > 0 || request.Unsafe || request.NeedsRequestCondition() || request.Name != "" || request.HasErrorMatchers())
|
||||
}
|
||||
@@ -26,6 +26,12 @@ func TestIsClusterable(t *testing.T) {
|
||||
require.True(t, req.IsClusterable())
|
||||
})
|
||||
|
||||
t.Run("error matchers prevent clustering", func(t *testing.T) {
|
||||
req := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}
|
||||
req.Matchers = []*matchers.Matcher{{Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher}}}
|
||||
require.False(t, req.IsClusterable())
|
||||
})
|
||||
|
||||
t.Run("payloads prevent clustering", func(t *testing.T) {
|
||||
req := &Request{
|
||||
Path: []string{"{{BaseURL}}"},
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
)
|
||||
|
||||
@@ -21,8 +22,7 @@ import (
|
||||
// TODO: Try to consolidate this in protocols.MakeDefaultMatchFunc to avoid any inconsistencies
|
||||
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
|
||||
item, ok := request.getMatchPart(matcher.Part, data)
|
||||
// an llm matcher with inputs compares whole responses instead of one part
|
||||
if !ok && matcher.Type.MatcherType != matchers.DSLMatcher && len(matcher.Inputs) == 0 {
|
||||
if !ok && matcher.NeedsPart() {
|
||||
return false, []string{}
|
||||
}
|
||||
|
||||
@@ -49,6 +49,12 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
|
||||
return matcher.Result(matcher.MatchDSLWithOptions(data, request.options.GetOptions())), []string{}
|
||||
case matchers.XPathMatcher:
|
||||
return matcher.Result(matcher.MatchXPath(item)), []string{}
|
||||
case matchers.ErrorMatcher:
|
||||
if !requesterr.IsMarked(data) {
|
||||
// no request error: a response header named "error" is not one
|
||||
return matcher.ResultWithMatchedSnippet(false, []string{})
|
||||
}
|
||||
return matcher.ResultWithMatchedSnippet(matcher.MatchError(data))
|
||||
case matchers.LLMMatcher:
|
||||
return protocols.MatchLLM(data, matcher, item, request.options)
|
||||
}
|
||||
@@ -136,6 +142,7 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw
|
||||
request.setHashOrDefault(data, "all_headers", headers)
|
||||
request.setHashOrDefault(data, "header", headers)
|
||||
data["duration"] = duration.Seconds()
|
||||
data["timeout"] = false
|
||||
data["template-id"] = request.options.TemplateID
|
||||
data["template-info"] = request.options.TemplateInfo
|
||||
data["template-path"] = request.options.TemplatePath
|
||||
|
||||
@@ -41,7 +41,7 @@ func TestResponseToDSLMap(t *testing.T) {
|
||||
matched := "http://example.com/test/?test=1"
|
||||
|
||||
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
|
||||
require.Len(t, event, 15, "could not get correct number of items in dsl map")
|
||||
require.Len(t, event, 16, "could not get correct number of items in dsl map")
|
||||
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
|
||||
require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
|
||||
}
|
||||
@@ -67,11 +67,13 @@ func TestHTTPOperatorMatch(t *testing.T) {
|
||||
resp := &http.Response{}
|
||||
resp.Header = make(http.Header)
|
||||
resp.Header.Set("Test", "Test-Response")
|
||||
resp.Header.Set("Error", "unavailable")
|
||||
resp.Header.Set("Error-Type", "timeout")
|
||||
host := "http://example.com/test/"
|
||||
matched := "http://example.com/test/?test=1"
|
||||
|
||||
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
|
||||
require.Len(t, event, 15, "could not get correct number of items in dsl map")
|
||||
require.Len(t, event, 18, "could not get correct number of items in dsl map")
|
||||
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
|
||||
require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
|
||||
|
||||
@@ -118,6 +120,37 @@ func TestHTTPOperatorMatch(t *testing.T) {
|
||||
require.Equal(t, []string{}, matched)
|
||||
})
|
||||
|
||||
t.Run("response error header is not a request error", func(t *testing.T) {
|
||||
errorMatcher := &matchers.Matcher{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
|
||||
Errors: []string{"any"},
|
||||
}
|
||||
require.NoError(t, errorMatcher.CompileMatchers())
|
||||
|
||||
isMatched, matched := request.Match(event, errorMatcher)
|
||||
require.False(t, isMatched)
|
||||
require.Empty(t, matched)
|
||||
|
||||
negativeMatcher := &matchers.Matcher{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
|
||||
Errors: []string{"any"},
|
||||
Negative: true,
|
||||
}
|
||||
require.NoError(t, negativeMatcher.CompileMatchers())
|
||||
isMatched, _ = request.Match(event, negativeMatcher)
|
||||
require.True(t, isMatched, "negative error matcher must match a response without request error")
|
||||
|
||||
headerMatcher := &matchers.Matcher{
|
||||
Part: "error",
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
|
||||
Words: []string{"unavailable"},
|
||||
}
|
||||
require.NoError(t, headerMatcher.CompileMatchers())
|
||||
isMatched, matched = request.Match(event, headerMatcher)
|
||||
require.True(t, isMatched)
|
||||
require.Equal(t, []string{"unavailable"}, matched)
|
||||
})
|
||||
|
||||
t.Run("caseInsensitive", func(t *testing.T) {
|
||||
matcher := &matchers.Matcher{
|
||||
Part: "body",
|
||||
@@ -159,7 +192,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
|
||||
matched := "http://example.com/test/?test=1"
|
||||
|
||||
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
|
||||
require.Len(t, event, 15, "could not get correct number of items in dsl map")
|
||||
require.Len(t, event, 16, "could not get correct number of items in dsl map")
|
||||
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
|
||||
require.Equal(t, "Test-Response", event["test_header"], "could not get correct resp for header")
|
||||
|
||||
@@ -286,7 +319,7 @@ func TestHTTPMakeResult(t *testing.T) {
|
||||
matched := "http://example.com/test/?test=1"
|
||||
|
||||
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
|
||||
require.Len(t, event, 15, "could not get correct number of items in dsl map")
|
||||
require.Len(t, event, 16, "could not get correct number of items in dsl map")
|
||||
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
|
||||
require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
|
||||
|
||||
@@ -304,6 +337,53 @@ func TestHTTPMakeResult(t *testing.T) {
|
||||
require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
|
||||
}
|
||||
|
||||
func TestHTTPMakeResultRendersInfo(t *testing.T) {
|
||||
options := testutils.DefaultOptions
|
||||
|
||||
testutils.Init(options)
|
||||
templateID := "testing-http-info"
|
||||
request := &Request{
|
||||
ID: templateID,
|
||||
Path: []string{"{{BaseURL}}"},
|
||||
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
|
||||
Operators: operators.Operators{
|
||||
Matchers: []*matchers.Matcher{{
|
||||
Part: "body",
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
|
||||
Words: []string{"version"},
|
||||
}},
|
||||
Extractors: []*extractors.Extractor{{
|
||||
Name: "version",
|
||||
Part: "body",
|
||||
Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor},
|
||||
Regex: []string{`version: ([0-9.]+)`},
|
||||
RegexGroup: 1,
|
||||
Internal: true,
|
||||
}},
|
||||
},
|
||||
}
|
||||
description := "Version {{version}} on {{host}}:{{port}} accepts password {{replace_regex(password, '^(.{2}).*$', '$1***')}}"
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: templateID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.High}, Name: "test", Description: description},
|
||||
})
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
resp := &http.Response{Header: make(http.Header)}
|
||||
// The payload values of the request that matched travel in the event.
|
||||
payloads := map[string]interface{}{"password": "s3cr3t"}
|
||||
event := request.responseToDSLMap(resp, "https://example.com:8443/", "https://example.com:8443/", "", "", "server version: 7.2.4", "", time.Second, payloads)
|
||||
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
|
||||
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false)
|
||||
require.True(t, ok)
|
||||
finalEvent.OperatorsResult = result
|
||||
finalEvent.Results = request.MakeResultEvent(finalEvent)
|
||||
|
||||
require.Len(t, finalEvent.Results, 1)
|
||||
require.Equal(t, "Version 7.2.4 on example.com:8443 accepts password s3***", finalEvent.Results[0].Info.Description)
|
||||
require.Equal(t, description, executerOpts.TemplateInfo.Description, "the template info must stay as written")
|
||||
}
|
||||
|
||||
const exampleRawRequest = `GET / HTTP/1.1
|
||||
Host: example.com
|
||||
Upgrade-Insecure-Requests: 1
|
||||
|
||||
@@ -44,6 +44,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httputils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
|
||||
@@ -1408,10 +1409,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
||||
request.options.Output.Request(request.options.TemplatePath, formedURL, request.Type().String(), err)
|
||||
request.options.Progress.IncrementErrorsBy(1)
|
||||
|
||||
// In case of interactsh markers and request times out, still send
|
||||
// a callback event so in case we receive an interaction, correlation is possible.
|
||||
// Also, to log failed use-cases.
|
||||
outputEvent := request.responseToDSLMap(&http.Response{}, input.MetaInput.Input, formedURL, convUtil.String(dumpedRequest.data), "", "", "", 0, generatedRequest.meta)
|
||||
duration := time.Since(timeStart)
|
||||
outputEvent := request.responseToDSLMap(&http.Response{}, input.MetaInput.Input, formedURL, convUtil.String(dumpedRequest.data), "", "", "", duration, generatedRequest.meta)
|
||||
requesterr.Annotate(outputEvent, err, duration)
|
||||
if i := strings.LastIndex(hostname, ":"); i != -1 {
|
||||
hostname = hostname[:i]
|
||||
}
|
||||
@@ -1421,14 +1421,35 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
||||
// try getting cname
|
||||
request.addCNameIfAvailable(hostname, outputEvent)
|
||||
}
|
||||
|
||||
if len(generatedRequest.interactshURLs) > 0 {
|
||||
// according to logic we only need to trigger a callback if interactsh was used
|
||||
// and request failed in hope that later on oast interaction will be received
|
||||
event := &output.InternalWrappedEvent{}
|
||||
if request.CompiledOperators != nil && request.CompiledOperators.HasDSL() {
|
||||
event.InternalEvent = outputEvent
|
||||
if request.options.Interactsh != nil {
|
||||
request.options.Interactsh.MakePlaceholders(generatedRequest.interactshURLs, outputEvent)
|
||||
}
|
||||
requestEvent := maps.Clone(outputEvent)
|
||||
if previousEvent != nil {
|
||||
outputEvent = maps.Clone(previousEvent)
|
||||
maps.Copy(outputEvent, requestEvent)
|
||||
}
|
||||
if request.NeedsRequestCondition() {
|
||||
for key, value := range requestEvent {
|
||||
outputEvent[fmt.Sprintf("%s_%d", key, requestCount)] = value
|
||||
}
|
||||
}
|
||||
// mark after the indexed copy so the provenance token gets no "_N" twin
|
||||
requesterr.Mark(outputEvent)
|
||||
|
||||
hasErrorMatchers := request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers()
|
||||
if hasErrorMatchers || len(generatedRequest.interactshURLs) > 0 {
|
||||
var event *output.InternalWrappedEvent
|
||||
if hasErrorMatchers {
|
||||
event = eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
|
||||
} else {
|
||||
// Interactsh-only failure path: keep a lightweight event for later OAST correlation.
|
||||
event = &output.InternalWrappedEvent{}
|
||||
if request.CompiledOperators != nil && request.CompiledOperators.HasDSL() {
|
||||
event.InternalEvent = outputEvent
|
||||
}
|
||||
}
|
||||
requesterr.Unmark(outputEvent)
|
||||
callback(event)
|
||||
}
|
||||
return err
|
||||
@@ -1941,8 +1962,13 @@ func (request *Request) recordHostResultAndCancelIfUnresponsive(input *contextar
|
||||
}
|
||||
}
|
||||
|
||||
// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history
|
||||
// isUnresponsiveAddress checks if the host is marked unresponsive based on
|
||||
// execution history. Templates with error/timeout matchers intentionally bypass
|
||||
// this skip so they can still observe and match request failures.
|
||||
func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool {
|
||||
if request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers() {
|
||||
return false
|
||||
}
|
||||
if request.options.HostErrorsCache != nil {
|
||||
return request.options.HostErrorsCache.Check(request.options.ProtocolType.String(), input)
|
||||
}
|
||||
|
||||
@@ -2120,3 +2120,201 @@ func TestHTTPToHTTPSUnrelated400IsNotRetried(t *testing.T) {
|
||||
require.True(t, matched, "an ordinary 400 is still a response the matcher can see")
|
||||
require.Equal(t, int32(0), tlsHits.Load())
|
||||
}
|
||||
|
||||
func TestHTTPErrorMatcherOnConnectionFailure(t *testing.T) {
|
||||
options := testutils.DefaultOptions
|
||||
testutils.Init(options)
|
||||
|
||||
templateID := "http-error-matcher"
|
||||
request := &Request{
|
||||
ID: templateID,
|
||||
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
|
||||
Path: []string{"{{BaseURL}}/"},
|
||||
Operators: operators.Operators{
|
||||
Matchers: []*matchers.Matcher{{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
|
||||
Errors: []string{"connection"},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: templateID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
err := request.Compile(executerOpts)
|
||||
require.NoError(t, err)
|
||||
|
||||
var matched bool
|
||||
var event *output.InternalWrappedEvent
|
||||
ctxArgs := contextargs.NewWithInput(context.Background(), "http://127.0.0.1:1")
|
||||
previous := output.InternalEvent{
|
||||
"status_code_1": 200,
|
||||
"error": "stale",
|
||||
"error_type": "timeout",
|
||||
"timeout": true,
|
||||
}
|
||||
err = request.ExecuteWithResults(ctxArgs, nil, previous, func(e *output.InternalWrappedEvent) {
|
||||
event = e
|
||||
if e.OperatorsResult != nil && e.OperatorsResult.Matched {
|
||||
matched = true
|
||||
}
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.NotNil(t, event)
|
||||
require.True(t, matched, "connection error should match type: error matcher")
|
||||
require.Equal(t, 200, event.InternalEvent["status_code_1"])
|
||||
require.NotEqual(t, "stale", event.InternalEvent["error"])
|
||||
require.Equal(t, "connection", event.InternalEvent["error_type"])
|
||||
require.Equal(t, false, event.InternalEvent["timeout"])
|
||||
}
|
||||
|
||||
func TestHTTPErrorMatcherSubstring(t *testing.T) {
|
||||
options := testutils.DefaultOptions
|
||||
testutils.Init(options)
|
||||
|
||||
templateID := "http-error-substring"
|
||||
request := &Request{
|
||||
ID: templateID,
|
||||
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
|
||||
Path: []string{"{{BaseURL}}/"},
|
||||
Operators: operators.Operators{
|
||||
Matchers: []*matchers.Matcher{{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
|
||||
Errors: []string{"refused"},
|
||||
}},
|
||||
},
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: templateID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
var matched bool
|
||||
ctxArgs := contextargs.NewWithInput(context.Background(), "http://127.0.0.1:1")
|
||||
err := request.ExecuteWithResults(ctxArgs, nil, nil, func(e *output.InternalWrappedEvent) {
|
||||
if e.OperatorsResult != nil && e.OperatorsResult.Matched {
|
||||
matched = true
|
||||
}
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.True(t, matched)
|
||||
}
|
||||
|
||||
func TestHTTPErrorMatcherDSL(t *testing.T) {
|
||||
options := testutils.DefaultOptions
|
||||
testutils.Init(options)
|
||||
|
||||
templateID := "http-error-dsl"
|
||||
request := &Request{
|
||||
ID: templateID,
|
||||
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
|
||||
Path: []string{"{{BaseURL}}/"},
|
||||
Operators: operators.Operators{
|
||||
Matchers: []*matchers.Matcher{{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
|
||||
DSL: []string{`error_type == "connection"`},
|
||||
}},
|
||||
},
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: templateID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
var matched bool
|
||||
ctxArgs := contextargs.NewWithInput(context.Background(), "http://127.0.0.1:1")
|
||||
err := request.ExecuteWithResults(ctxArgs, nil, nil, func(e *output.InternalWrappedEvent) {
|
||||
if e.OperatorsResult != nil && e.OperatorsResult.Matched {
|
||||
matched = true
|
||||
}
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.True(t, matched, "DSL error_type matcher should fire on connection failure")
|
||||
}
|
||||
|
||||
func TestHTTPErrorMatcherBypassesHostErrorsCache(t *testing.T) {
|
||||
options := testutils.DefaultOptions
|
||||
testutils.Init(options)
|
||||
|
||||
templateID := "http-error-host-cache"
|
||||
request := &Request{
|
||||
ID: templateID,
|
||||
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
|
||||
Path: []string{"{{BaseURL}}/"},
|
||||
Operators: operators.Operators{
|
||||
Matchers: []*matchers.Matcher{{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
|
||||
Errors: []string{"connection"},
|
||||
}},
|
||||
},
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: templateID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
// Host is marked unresponsive; without error matchers the request would be skipped.
|
||||
executerOpts.HostErrorsCache = &spyHostErrorsCache{skip: true}
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
var matched bool
|
||||
var callbacks int
|
||||
ctxArgs := contextargs.NewWithInput(context.Background(), "http://127.0.0.1:1")
|
||||
err := request.ExecuteWithResults(ctxArgs, nil, nil, func(e *output.InternalWrappedEvent) {
|
||||
callbacks++
|
||||
if e.OperatorsResult != nil && e.OperatorsResult.Matched {
|
||||
matched = true
|
||||
}
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.Equal(t, 1, callbacks, "error matcher should bypass host-errors skip")
|
||||
require.True(t, matched)
|
||||
}
|
||||
|
||||
func TestHTTPErrorMatcherTimeout(t *testing.T) {
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.Timeout = 1
|
||||
testutils.Init(options)
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(3 * time.Second)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
templateID := "http-error-timeout"
|
||||
request := &Request{
|
||||
ID: templateID,
|
||||
Raw: []string{`@timeout: 1s
|
||||
GET / HTTP/1.1
|
||||
Host: {{Hostname}}
|
||||
`},
|
||||
Operators: operators.Operators{
|
||||
Matchers: []*matchers.Matcher{{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
|
||||
Errors: []string{"timeout"},
|
||||
}},
|
||||
},
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: templateID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
var matched bool
|
||||
var event *output.InternalWrappedEvent
|
||||
ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
|
||||
err := request.ExecuteWithResults(ctxArgs, nil, nil, func(e *output.InternalWrappedEvent) {
|
||||
event = e
|
||||
if e.OperatorsResult != nil && e.OperatorsResult.Matched {
|
||||
matched = true
|
||||
}
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.NotNil(t, event)
|
||||
require.True(t, matched, "timeout error should match type: error matcher")
|
||||
require.Equal(t, true, event.InternalEvent["timeout"])
|
||||
require.Equal(t, "timeout", event.InternalEvent["error_type"])
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package protocols
|
||||
|
||||
import (
|
||||
"maps"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
)
|
||||
|
||||
// renderInfo fills the {{placeholders}} in the free text of each result's info
|
||||
// with the values of the event that produced it, so a finding can say what was
|
||||
// found and where. Name, severity, tags and classification stay as written:
|
||||
// they identify the template for filtering, grouping and issue trackers.
|
||||
//
|
||||
// Each result holds its own copy of the info, so the template is never changed.
|
||||
// When filling changes the text, the result keeps the info as written in
|
||||
// TemplateInfo.
|
||||
func renderInfo(results []*output.ResultEvent, data map[string]interface{}) []*output.ResultEvent {
|
||||
// A global matcher matches on another template's event, with that template's
|
||||
// payloads, variables and extracted values, so only the result fields fill
|
||||
// its info.
|
||||
if isGlobal, _ := data["global-matchers"].(bool); isGlobal {
|
||||
data = nil
|
||||
}
|
||||
for _, result := range results {
|
||||
if !result.Info.HasPlaceholders() {
|
||||
continue
|
||||
}
|
||||
written := result.Info
|
||||
values := infoValues(result, data)
|
||||
for _, field := range result.Info.PlaceholderFields() {
|
||||
if field.HasPlaceholders() {
|
||||
*field.Value = renderInfoText(result.TemplateID, *field.Value, values)
|
||||
}
|
||||
}
|
||||
if len(result.Info.FilledFields(&written)) > 0 {
|
||||
result.TemplateInfo = &written
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
// infoValues returns the event data with the result's own fields on top, under
|
||||
// their names in the JSON output. Those win over the event's keys: in an http
|
||||
// event host is the raw input URL, while the output reports the hostname.
|
||||
func infoValues(result *output.ResultEvent, data map[string]interface{}) map[string]interface{} {
|
||||
values := make(map[string]interface{}, len(data)+9)
|
||||
maps.Copy(values, data)
|
||||
for key, value := range map[string]string{
|
||||
"host": result.Host,
|
||||
"port": result.Port,
|
||||
"ip": result.IP,
|
||||
"url": result.URL,
|
||||
"scheme": result.Scheme,
|
||||
"path": result.Path,
|
||||
"matched-at": result.Matched,
|
||||
"matcher-name": result.MatcherName,
|
||||
"extractor-name": result.ExtractorName,
|
||||
} {
|
||||
if value != "" {
|
||||
values[key] = value
|
||||
}
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
// renderInfoText renders the text through the shared renderer, as requests and
|
||||
// matchers are, so nested placeholders and helpers behave the same way. Scan
|
||||
// options are left out, so network helpers stay denied while results are built.
|
||||
// If an expression fails, the text stays as written, as a word matcher's does:
|
||||
// a typo in the template must not cost the finding.
|
||||
func renderInfoText(templateID, text string, values map[string]interface{}) string {
|
||||
rendered, err := render.Render(render.Input{Text: text, Values: values})
|
||||
if err != nil {
|
||||
gologger.Verbose().Msgf("[%s] Could not render template info: %s\n", templateID, err)
|
||||
return text
|
||||
}
|
||||
return rendered.Text
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package protocols
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRenderInfoLeavesStaticInfoAlone(t *testing.T) {
|
||||
result := &output.ResultEvent{Host: "example.com", Info: model.Info{Description: "Redis is exposed"}}
|
||||
|
||||
renderInfo([]*output.ResultEvent{result}, map[string]interface{}{"host": "example.com"})
|
||||
|
||||
require.Equal(t, "Redis is exposed", result.Info.Description)
|
||||
require.Nil(t, result.TemplateInfo)
|
||||
}
|
||||
|
||||
// Braces that fill nothing, like an SSTI payload in the description, leave the
|
||||
// text static, so it is neither redacted nor moved off the sarif rule.
|
||||
func TestRenderInfoLeavesUnfilledBracesStatic(t *testing.T) {
|
||||
result := &output.ResultEvent{Host: "example.com", Info: model.Info{Description: "Send {{7*7}} or {{config}} in the name parameter"}}
|
||||
|
||||
renderInfo([]*output.ResultEvent{result}, map[string]interface{}{})
|
||||
|
||||
require.Equal(t, "Send {{7*7}} or {{config}} in the name parameter", result.Info.Description)
|
||||
require.Nil(t, result.TemplateInfo)
|
||||
}
|
||||
|
||||
func TestRenderInfoFillsResultAndEventValues(t *testing.T) {
|
||||
result := &output.ResultEvent{
|
||||
Host: "example.com",
|
||||
Port: "6379",
|
||||
MatcherName: "auth",
|
||||
Info: model.Info{
|
||||
Name: "Redis {{version}}",
|
||||
Description: "{{matcher-name}} on {{host}}:{{port}}, version {{version}}",
|
||||
Impact: "{{to_upper(password)}}",
|
||||
Remediation: "{{credentials}} stays",
|
||||
},
|
||||
}
|
||||
// The event's host is the raw input, while the output reports the hostname.
|
||||
data := map[string]interface{}{"host": "redis://example.com:6379", "version": "7.2.4", "password": "admin"}
|
||||
|
||||
renderInfo([]*output.ResultEvent{result}, data)
|
||||
|
||||
require.Equal(t, "auth on example.com:6379, version 7.2.4", result.Info.Description)
|
||||
require.Equal(t, "ADMIN", result.Info.Impact)
|
||||
require.Equal(t, "{{credentials}} stays", result.Info.Remediation)
|
||||
require.Equal(t, "Redis {{version}}", result.Info.Name)
|
||||
require.Equal(t, "redis://example.com:6379", data["host"])
|
||||
require.NotNil(t, result.TemplateInfo)
|
||||
require.Equal(t, "{{matcher-name}} on {{host}}:{{port}}, version {{version}}", result.TemplateInfo.Description)
|
||||
require.Equal(t, "{{to_upper(password)}}", result.TemplateInfo.Impact)
|
||||
}
|
||||
|
||||
// Info renders like requests do, so the usual nested placeholder idiom and an
|
||||
// expression spread over lines work too.
|
||||
func TestRenderInfoFillsNestedAndMultilinePlaceholders(t *testing.T) {
|
||||
result := &output.ResultEvent{Info: model.Info{
|
||||
Description: "Authorization: Basic {{base64('{{username}}:{{password}}')}}",
|
||||
Remediation: "Upgrade from {{join(\", \",\n versions)}}",
|
||||
}}
|
||||
|
||||
renderInfo([]*output.ResultEvent{result}, map[string]interface{}{
|
||||
"username": "admin", "password": "secret", "versions": []string{"7.2.4", "6.0.9"},
|
||||
})
|
||||
|
||||
require.Equal(t, "Authorization: Basic YWRtaW46c2VjcmV0", result.Info.Description)
|
||||
require.Equal(t, "Upgrade from 7.2.4, 6.0.9", result.Info.Remediation)
|
||||
}
|
||||
|
||||
// A hyphenated name reads as subtraction inside a helper unless it is bracketed,
|
||||
// and the general §name§ marker fills like a {{name}} one.
|
||||
func TestRenderInfoFillsBracketedNamesAndGeneralMarkers(t *testing.T) {
|
||||
result := &output.ResultEvent{MatcherName: "auth", Info: model.Info{
|
||||
Description: "{{to_upper([matcher-name])}} matched",
|
||||
Impact: "default login §username§ accepted",
|
||||
}}
|
||||
|
||||
renderInfo([]*output.ResultEvent{result}, map[string]interface{}{"username": "admin"})
|
||||
|
||||
require.Equal(t, "AUTH matched", result.Info.Description)
|
||||
require.Equal(t, "default login admin accepted", result.Info.Impact)
|
||||
}
|
||||
|
||||
// A failing expression leaves its field as written; the other fields render.
|
||||
func TestRenderInfoKeepsAFieldWithAFailingExpression(t *testing.T) {
|
||||
result := &output.ResultEvent{Host: "example.com", Info: model.Info{
|
||||
Description: "{{host}} {{to_upper(missing)}}",
|
||||
Impact: "{{host}} is exposed",
|
||||
}}
|
||||
|
||||
renderInfo([]*output.ResultEvent{result}, map[string]interface{}{})
|
||||
|
||||
require.Equal(t, "{{host}} {{to_upper(missing)}}", result.Info.Description)
|
||||
require.Equal(t, "example.com is exposed", result.Info.Impact)
|
||||
require.Len(t, result.Info.FilledFields(result.TemplateInfo), 1)
|
||||
}
|
||||
|
||||
// A global matcher's event belongs to the template whose response it matched,
|
||||
// so only the result fields fill its info.
|
||||
func TestRenderInfoFillsGlobalMatcherInfoFromResultFieldsOnly(t *testing.T) {
|
||||
result := &output.ResultEvent{Host: "example.com", Info: model.Info{Description: "Leaked {{token}} on {{host}}"}}
|
||||
|
||||
renderInfo([]*output.ResultEvent{result}, map[string]interface{}{"global-matchers": true, "token": "origin-template-payload"})
|
||||
|
||||
require.Equal(t, "Leaked {{token}} on example.com", result.Info.Description)
|
||||
}
|
||||
|
||||
// Results are built without scan options, so a network helper never runs.
|
||||
func TestRenderInfoDeniesNetworkHelpers(t *testing.T) {
|
||||
result := &output.ResultEvent{Host: "example.com", Info: model.Info{Description: "{{host}} resolves to {{resolve(host)}}"}}
|
||||
|
||||
renderInfo([]*output.ResultEvent{result}, map[string]interface{}{})
|
||||
|
||||
require.Equal(t, "{{host}} resolves to {{resolve(host)}}", result.Info.Description)
|
||||
require.Nil(t, result.TemplateInfo)
|
||||
}
|
||||
@@ -32,6 +32,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
@@ -721,6 +722,10 @@ func (request *Request) executeRequestWithPayloads(
|
||||
if genErr != nil {
|
||||
return genErr
|
||||
}
|
||||
if err != nil {
|
||||
// Make classified failure fields available to later protocol steps.
|
||||
requesterr.Annotate(data, err, 0)
|
||||
}
|
||||
|
||||
// add and get values from templatectx
|
||||
request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), data)
|
||||
@@ -737,7 +742,13 @@ func (request *Request) executeRequestWithPayloads(
|
||||
}
|
||||
|
||||
if _, ok := data["error"]; ok {
|
||||
event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(data, payloadValues), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
|
||||
eventData := generators.MergeMaps(data, payloadValues)
|
||||
if err != nil {
|
||||
// Template context is merged above and may contain stale namespaced
|
||||
// values, so ensure the emitted event retains the current failure.
|
||||
requesterr.Annotate(eventData, err, 0)
|
||||
}
|
||||
event := eventcreator.CreateEventWithAdditionalOptions(request, eventData, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
|
||||
wrappedEvent.OperatorsResult.PayloadValues = payload
|
||||
})
|
||||
callback(event)
|
||||
|
||||
@@ -287,3 +287,39 @@ func TestPreConditionRequestsAreCounted(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteFailureAddsClassifiedFieldsToTemplateContext(t *testing.T) {
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() {
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
executorOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "javascript-classified-failure"})
|
||||
executorOptions.JsCompiler = templates.GetJsCompiler()
|
||||
executorOptions.Verified = true
|
||||
executorOptions.Flow = "flow()"
|
||||
|
||||
request := &javascript.Request{
|
||||
ID: "js",
|
||||
Code: `throw new Error("classified failure")`,
|
||||
}
|
||||
require.NoError(t, request.Compile(executorOptions))
|
||||
|
||||
target := contextargs.NewWithInput(context.Background(), "127.0.0.1:1")
|
||||
var event *output.InternalWrappedEvent
|
||||
err := request.ExecuteWithResults(target, nil, nil, func(got *output.InternalWrappedEvent) {
|
||||
event = got
|
||||
})
|
||||
|
||||
require.ErrorContains(t, err, "classified failure")
|
||||
require.NotNil(t, event)
|
||||
require.Contains(t, event.InternalEvent["error"], "classified failure")
|
||||
require.Equal(t, "unknown", event.InternalEvent["error_type"])
|
||||
require.Equal(t, false, event.InternalEvent["timeout"])
|
||||
|
||||
templateCtx := executorOptions.GetTemplateCtx(target.MetaInput).GetAll()
|
||||
require.Contains(t, templateCtx["js_error"], "classified failure")
|
||||
require.Equal(t, "unknown", templateCtx["js_error_type"])
|
||||
require.Equal(t, false, templateCtx["js_timeout"])
|
||||
}
|
||||
@@ -16,8 +16,7 @@ import (
|
||||
// Match matches a generic data response again a given matcher
|
||||
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
|
||||
itemStr, ok := request.getMatchPart(matcher.Part, data)
|
||||
// an llm matcher with inputs compares whole responses instead of one part
|
||||
if !ok && matcher.Type.MatcherType != matchers.DSLMatcher && len(matcher.Inputs) == 0 {
|
||||
if !ok && matcher.NeedsPart() {
|
||||
return false, []string{}
|
||||
}
|
||||
|
||||
@@ -34,6 +33,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
|
||||
return matcher.Result(matcher.MatchDSLWithOptions(data, request.options.GetOptions())), []string{}
|
||||
case matchers.XPathMatcher:
|
||||
return matcher.Result(matcher.MatchXPath(itemStr)), []string{}
|
||||
case matchers.ErrorMatcher:
|
||||
return matcher.ResultWithMatchedSnippet(matcher.MatchError(data))
|
||||
case matchers.LLMMatcher:
|
||||
return protocols.MatchLLM(data, matcher, itemStr, request.options)
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
|
||||
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
mapsutil "github.com/projectdiscovery/utils/maps"
|
||||
@@ -360,6 +361,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
|
||||
if err != nil {
|
||||
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
|
||||
request.options.Progress.IncrementFailedRequestsBy(1)
|
||||
request.emitErrorEvent(callback, err, address, actualAddress, payloads, previous, hostname, interactshURLs)
|
||||
return errors.Wrap(err, "could not connect to server")
|
||||
}
|
||||
defer func() {
|
||||
@@ -418,12 +420,14 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
|
||||
if _, err := conn.Write(dataInBytes); err != nil {
|
||||
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
|
||||
request.options.Progress.IncrementFailedRequestsBy(1)
|
||||
request.emitErrorEvent(callback, err, address, actualAddress, payloads, previous, hostname, interactshURLs)
|
||||
return errors.Wrap(err, "could not write request to server")
|
||||
}
|
||||
|
||||
if input.Read > 0 {
|
||||
buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), request.options.Options.GetTimeouts().TcpReadTimeout)
|
||||
if err != nil {
|
||||
request.emitErrorEvent(callback, err, address, actualAddress, payloads, previous, hostname, interactshURLs)
|
||||
return errkit.Wrap(err, "could not read response from connection")
|
||||
}
|
||||
stepDurations = append(stepDurations, time.Since(timeStart))
|
||||
@@ -502,6 +506,9 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
|
||||
if request.options.Interactsh != nil {
|
||||
request.options.Interactsh.MakePlaceholders(interactshURLs, outputEvent)
|
||||
}
|
||||
if readErr != nil {
|
||||
requesterr.Annotate(outputEvent, readErr, 0)
|
||||
}
|
||||
|
||||
var event *output.InternalWrappedEvent
|
||||
if len(interactshURLs) == 0 {
|
||||
@@ -619,6 +626,47 @@ func ConnReadNWithTimeout(conn net.Conn, n int64, timeout time.Duration) ([]byte
|
||||
return b[:count], nil
|
||||
}
|
||||
|
||||
// emitErrorEvent creates a matcher event for a failed network I/O operation when
|
||||
// the template has error/timeout matchers.
|
||||
func (request *Request) emitErrorEvent(callback protocols.OutputEventCallback, err error, address, actualAddress string, payloads map[string]interface{}, previous output.InternalEvent, hostname string, interactshURLs []string) {
|
||||
hasErrorMatchers := request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers()
|
||||
if !hasErrorMatchers && len(interactshURLs) == 0 {
|
||||
return
|
||||
}
|
||||
outputEvent := request.responseToDSLMap("", "", "", address, actualAddress)
|
||||
maps.Copy(outputEvent, previous)
|
||||
maps.Copy(outputEvent, payloads)
|
||||
if hostname != "" {
|
||||
outputEvent["ip"] = request.dialer.GetDialedIP(hostname)
|
||||
}
|
||||
if request.options.Interactsh != nil {
|
||||
request.options.Interactsh.MakePlaceholders(interactshURLs, outputEvent)
|
||||
}
|
||||
requesterr.Annotate(outputEvent, err, 0)
|
||||
|
||||
var event *output.InternalWrappedEvent
|
||||
if hasErrorMatchers {
|
||||
event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
|
||||
wrappedEvent.OperatorsResult.PayloadValues = payloads
|
||||
})
|
||||
} else {
|
||||
event = &output.InternalWrappedEvent{InternalEvent: outputEvent}
|
||||
}
|
||||
if len(interactshURLs) > 0 && request.options.Interactsh != nil {
|
||||
event.UsesInteractsh = true
|
||||
request.options.RegisterInteractshRequest(interactshURLs, &interactsh.RequestData{
|
||||
MakeResultFunc: request.MakeResultEvent,
|
||||
Event: event,
|
||||
Operators: request.CompiledOperators,
|
||||
MatchFunc: request.Match,
|
||||
ExtractFunc: request.Extract,
|
||||
})
|
||||
}
|
||||
if hasErrorMatchers {
|
||||
callback(event)
|
||||
}
|
||||
}
|
||||
|
||||
// markHostError checks if the error is a unreponsive host error and marks it
|
||||
func (request *Request) markHostError(input *contextargs.Context, err error) {
|
||||
if request.options.HostErrorsCache != nil {
|
||||
@@ -626,8 +674,12 @@ func (request *Request) markHostError(input *contextargs.Context, err error) {
|
||||
}
|
||||
}
|
||||
|
||||
// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history
|
||||
// isUnresponsiveAddress checks if the host is marked unresponsive based on
|
||||
// execution history. Templates with error/timeout matchers bypass this skip.
|
||||
func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool {
|
||||
if request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers() {
|
||||
return false
|
||||
}
|
||||
if request.options.HostErrorsCache != nil {
|
||||
return request.options.HostErrorsCache.Check(request.options.ProtocolType.String(), input)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
@@ -276,6 +277,51 @@ func TestNetworkCompileDefersInteractshMarkerHelpersToRuntime(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitErrorEventPreservesHistoryAndAnnotations(t *testing.T) {
|
||||
options := testutils.DefaultOptions
|
||||
testutils.Init(options)
|
||||
|
||||
request := &Request{
|
||||
ID: "network-error-history",
|
||||
Address: []string{"{{Hostname}}"},
|
||||
Operators: operators.Operators{
|
||||
Matchers: []*matchers.Matcher{{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
|
||||
Errors: []string{"connection"},
|
||||
}},
|
||||
},
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: request.ID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
previous := output.InternalEvent{
|
||||
"banner_1": "hello",
|
||||
"error": "stale",
|
||||
"error_type": "timeout",
|
||||
"timeout": true,
|
||||
}
|
||||
payloads := map[string]interface{}{
|
||||
"error": "payload",
|
||||
"error_type": "unknown",
|
||||
"timeout": true,
|
||||
}
|
||||
var event *output.InternalWrappedEvent
|
||||
request.emitErrorEvent(func(got *output.InternalWrappedEvent) {
|
||||
event = got
|
||||
}, errors.New("dial tcp 127.0.0.1:1: connection refused"), "127.0.0.1:1", "127.0.0.1:1", payloads, previous, "", nil)
|
||||
|
||||
require.NotNil(t, event)
|
||||
require.Equal(t, "hello", event.InternalEvent["banner_1"])
|
||||
require.Equal(t, "connection", event.InternalEvent["error_type"])
|
||||
require.Equal(t, false, event.InternalEvent["timeout"])
|
||||
require.Contains(t, event.InternalEvent["error"], "connection refused")
|
||||
require.True(t, event.OperatorsResult.Matched)
|
||||
require.Equal(t, payloads, event.OperatorsResult.PayloadValues)
|
||||
}
|
||||
|
||||
var exampleBody = `<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
|
||||
@@ -14,14 +14,14 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
)
|
||||
|
||||
// Match matches a generic data response again a given matcher
|
||||
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
|
||||
item, ok := getMatchPart(matcher.Part, data)
|
||||
// an llm matcher with inputs compares whole responses instead of one part
|
||||
if !ok && matcher.Type.MatcherType != matchers.DSLMatcher && len(matcher.Inputs) == 0 {
|
||||
if !ok && matcher.NeedsPart() {
|
||||
return false, []string{}
|
||||
}
|
||||
|
||||
@@ -44,6 +44,12 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
|
||||
return matcher.Result(matcher.MatchDSLWithOptions(data, request.options.GetOptions())), []string{}
|
||||
case matchers.XPathMatcher:
|
||||
return matcher.Result(matcher.MatchXPath(item)), []string{}
|
||||
case matchers.ErrorMatcher:
|
||||
if !requesterr.IsMarked(data) {
|
||||
// no request error: a response header named "error" is not one
|
||||
return matcher.ResultWithMatchedSnippet(false, []string{})
|
||||
}
|
||||
return matcher.ResultWithMatchedSnippet(matcher.MatchError(data))
|
||||
case matchers.LLMMatcher:
|
||||
return protocols.MatchLLM(data, matcher, item, request.options)
|
||||
}
|
||||
|
||||
@@ -7,13 +7,13 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
)
|
||||
|
||||
func TestResponseToDSLMap(t *testing.T) {
|
||||
@@ -109,6 +109,19 @@ func TestHTTPOperatorMatch(t *testing.T) {
|
||||
require.False(t, isMatched, "could match invalid response matcher")
|
||||
require.Equal(t, []string{}, matched)
|
||||
})
|
||||
|
||||
t.Run("error header is not request failure", func(t *testing.T) {
|
||||
event["error"] = "server-controlled"
|
||||
matcher := &matchers.Matcher{
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
|
||||
Errors: []string{"any"},
|
||||
}
|
||||
require.NoError(t, matcher.CompileMatchers())
|
||||
|
||||
isMatched, matched := request.Match(event, matcher)
|
||||
require.False(t, isMatched)
|
||||
require.Empty(t, matched)
|
||||
})
|
||||
}
|
||||
|
||||
func TestHTTPOperatorExtract(t *testing.T) {
|
||||
|
||||
@@ -9,54 +9,95 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
noMinor = regexp.MustCompile(`^HTTP/([0-9]) `)
|
||||
responseStatusRegex = regexp.MustCompile(`(?m)^HTTP/[0-9](?:\.[0-9])?\s+[0-9]{3}`)
|
||||
)
|
||||
// noMinor matches a leading status-line protocol with no minor version, such
|
||||
// as "HTTP/2 ". The go parser only accepts HTTP/X.Y.
|
||||
var noMinor = regexp.MustCompile(`^HTTP/[0-9] `)
|
||||
|
||||
// readResponseFromString reads a raw http response from a string.
|
||||
// errNotRequest marks data that does not start with an HTTP request, so the
|
||||
// caller parses it as a bare response instead.
|
||||
var errNotRequest = errors.New("not an http request")
|
||||
|
||||
// readResponseFromString reads one raw HTTP response. Data may be the response
|
||||
// itself or a request followed by that response. A status line is recognized
|
||||
// only where a response starts, never by searching the body for "HTTP/".
|
||||
func readResponseFromString(data string) (*http.Response, error) {
|
||||
// Check if "data" contains RFC compatible Request followed by a response
|
||||
br := bufio.NewReader(strings.NewReader(data))
|
||||
if req, err := http.ReadRequest(br); err == nil {
|
||||
if req.Body != nil {
|
||||
_, _ = io.Copy(io.Discard, req.Body)
|
||||
_ = req.Body.Close()
|
||||
resp, err := readRequestResponse(data)
|
||||
if err == nil || !errors.Is(err, errNotRequest) {
|
||||
return resp, err
|
||||
}
|
||||
return readResponse(data, nil)
|
||||
}
|
||||
|
||||
// readRequestResponse reads a raw HTTP request from the start of data and the
|
||||
// response that follows it.
|
||||
func readRequestResponse(data string) (*http.Response, error) {
|
||||
sr := strings.NewReader(data)
|
||||
br := bufio.NewReader(sr)
|
||||
// unread bytes are either still in sr or buffered in br
|
||||
unread := func() string { return data[len(data)-sr.Len()-br.Buffered():] }
|
||||
|
||||
req, err := http.ReadRequest(br)
|
||||
if err != nil {
|
||||
return nil, errNotRequest
|
||||
}
|
||||
afterHeaders := unread()
|
||||
// the response follows the request body, so it has to be read first.
|
||||
// a dump that keeps Content-Length but drops the body is tried at the headers.
|
||||
if _, err := io.Copy(io.Discard, req.Body); err == nil {
|
||||
if resp, err := readResponse(unread(), req); err == nil {
|
||||
return resp, nil
|
||||
}
|
||||
remaining, _ := io.ReadAll(br)
|
||||
if len(remaining) > 0 {
|
||||
remainingStr := addMinorVersionToHTTP(string(remaining))
|
||||
if resp, err := http.ReadResponse(bufio.NewReader(strings.NewReader(remainingStr)), req); err == nil {
|
||||
return resp, nil
|
||||
}
|
||||
return readResponse(afterHeaders, req)
|
||||
}
|
||||
|
||||
// maxInterimResponses bounds how many interim responses are skipped before the
|
||||
// final one. Each skip copies the remainder of data.
|
||||
const maxInterimResponses = 5
|
||||
|
||||
// readResponse reads a raw HTTP response whose status line starts data, after
|
||||
// any leading blank lines. Interim responses such as 100 Continue are skipped,
|
||||
// as the HTTP client does. The last interim response is returned when no final
|
||||
// response follows it.
|
||||
func readResponse(data string, req *http.Request) (*http.Response, error) {
|
||||
var lastInterim *http.Response
|
||||
for range maxInterimResponses + 1 {
|
||||
data = strings.TrimLeft(data, "\r\n")
|
||||
data = addMinorVersionToHTTP(data)
|
||||
sr := strings.NewReader(data)
|
||||
br := bufio.NewReader(sr)
|
||||
resp, err := http.ReadResponse(br, req)
|
||||
if err != nil {
|
||||
if lastInterim != nil {
|
||||
return lastInterim, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// otherwise tries to patch known cases such as http minor version
|
||||
var final string
|
||||
if strings.HasPrefix(data, "HTTP/") {
|
||||
final = addMinorVersionToHTTP(data)
|
||||
} else {
|
||||
loc := responseStatusRegex.FindStringIndex(data)
|
||||
if loc == nil {
|
||||
return nil, errors.New("malformed raw http response")
|
||||
if !isInterimResponse(resp) {
|
||||
return resp, nil
|
||||
}
|
||||
final = addMinorVersionToHTTP(data[loc[0]:])
|
||||
if lastInterim != nil && lastInterim.Body != nil {
|
||||
_ = lastInterim.Body.Close()
|
||||
}
|
||||
lastInterim = resp
|
||||
data = data[len(data)-sr.Len()-br.Buffered():]
|
||||
}
|
||||
return http.ReadResponse(bufio.NewReader(strings.NewReader(final)), nil)
|
||||
return lastInterim, nil
|
||||
}
|
||||
|
||||
// addMinorVersionToHTTP tries to add a minor version to http status header
|
||||
// fixing the compatibility issue with go standard library.
|
||||
// isInterimResponse reports whether resp is an interim response, such as
|
||||
// 100 Continue, that comes before the final response. 101 Switching Protocols
|
||||
// is the response.
|
||||
func isInterimResponse(resp *http.Response) bool {
|
||||
return resp != nil && resp.StatusCode >= 100 && resp.StatusCode <= 199 && resp.StatusCode != http.StatusSwitchingProtocols
|
||||
}
|
||||
|
||||
// addMinorVersionToHTTP inserts ".0" into a leading protocol such as HTTP/2.
|
||||
// Only those bytes change, so the reason phrase, headers, and body stay intact.
|
||||
func addMinorVersionToHTTP(data string) string {
|
||||
matches := noMinor.FindAllStringSubmatch(data, 1)
|
||||
if len(matches) == 0 {
|
||||
if !noMinor.MatchString(data) {
|
||||
return data
|
||||
}
|
||||
if len(matches[0]) < 2 {
|
||||
return data
|
||||
}
|
||||
replacedVersion := strings.Replace(matches[0][0], matches[0][1], matches[0][1]+".0", 1)
|
||||
data = strings.Replace(data, matches[0][0], replacedVersion, 1)
|
||||
return data
|
||||
const protoLen = len("HTTP/X")
|
||||
return data[:protoLen] + ".0" + data[protoLen:]
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/http/httputil"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -200,31 +201,101 @@ Server: Google Frontend
|
||||
})
|
||||
}
|
||||
|
||||
func TestReadResponseFromStringPreservesHTTPVersionText(t *testing.T) {
|
||||
const body = "Supports HTTP/2 and HTTP/3."
|
||||
|
||||
func TestReadResponse_HTTPInBodyCases(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data string
|
||||
expectedBody string
|
||||
statusCode int
|
||||
name string
|
||||
data string
|
||||
wantProto string
|
||||
wantStatus string
|
||||
wantVia string
|
||||
wantMethod string
|
||||
}{
|
||||
{
|
||||
name: "http2 response without minor version and HTTP/ in body (#7924)",
|
||||
data: "GET / HTTP/1.1\r\nHost: example.com\r\n\r\nHTTP/2 200 OK\r\nContent-Length: 27\r\n\r\nSupports HTTP/2 and HTTP/3.",
|
||||
expectedBody: "Supports HTTP/2 and HTTP/3.",
|
||||
statusCode: 200,
|
||||
name: "body-with-content-length",
|
||||
data: "HTTP/1.1 200 OK\r\nContent-Length: 27\r\n\r\n" + body,
|
||||
wantProto: "HTTP/1.1",
|
||||
wantStatus: "200 OK",
|
||||
},
|
||||
{
|
||||
name: "request with body and HTTP/ in response body (#7924)",
|
||||
data: "POST /search HTTP/1.1\r\nHost: example.com\r\nContent-Length: 3\r\n\r\nq=1HTTP/1.1 200 OK\r\nContent-Length: 27\r\n\r\nSupports HTTP/2 and HTTP/3.",
|
||||
expectedBody: "Supports HTTP/2 and HTTP/3.",
|
||||
statusCode: 200,
|
||||
name: "body-without-content-length",
|
||||
data: "HTTP/1.1 200 OK\r\n\r\n" + body,
|
||||
wantProto: "HTTP/1.1",
|
||||
wantStatus: "200 OK",
|
||||
},
|
||||
{
|
||||
name: "control http 1.1 with HTTP/ in response body (#7924)",
|
||||
data: "GET / HTTP/1.1\r\nHost: example.com\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 27\r\n\r\nSupports HTTP/2 and HTTP/3.",
|
||||
expectedBody: "Supports HTTP/2 and HTTP/3.",
|
||||
statusCode: 200,
|
||||
name: "body-with-chunked-encoding",
|
||||
data: "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1b\r\n" + body + "\r\n0\r\n\r\n",
|
||||
wantProto: "HTTP/1.1",
|
||||
wantStatus: "200 OK",
|
||||
},
|
||||
{
|
||||
name: "header-value",
|
||||
data: "HTTP/1.1 200 OK\r\nVia: HTTP/2 edge\r\nContent-Length: 27\r\n\r\n" + body,
|
||||
wantProto: "HTTP/1.1",
|
||||
wantStatus: "200 OK",
|
||||
wantVia: "HTTP/2 edge",
|
||||
},
|
||||
{
|
||||
name: "reason-phrase",
|
||||
data: "HTTP/1.1 200 Use HTTP/2 instead\r\nContent-Length: 27\r\n\r\n" + body,
|
||||
wantProto: "HTTP/1.1",
|
||||
wantStatus: "200 Use HTTP/2 instead",
|
||||
},
|
||||
{
|
||||
name: "status-line-without-minor-version",
|
||||
data: "HTTP/2 200 OK\r\nVia: HTTP/2 edge\r\nContent-Length: 27\r\n\r\n" + body,
|
||||
wantProto: "HTTP/2.0",
|
||||
wantStatus: "200 OK",
|
||||
wantVia: "HTTP/2 edge",
|
||||
},
|
||||
{
|
||||
name: "http3-status-line-without-minor-version",
|
||||
data: "HTTP/3 200 OK\r\nContent-Length: 27\r\n\r\n" + body,
|
||||
wantProto: "HTTP/3.0",
|
||||
wantStatus: "200 OK",
|
||||
},
|
||||
{
|
||||
name: "request-then-status-line-without-minor-version",
|
||||
data: "GET / HTTP/1.1\r\nHost: example.com\r\n\r\nHTTP/2 200 OK\r\nContent-Length: 27\r\n\r\n" + body,
|
||||
wantProto: "HTTP/2.0",
|
||||
wantStatus: "200 OK",
|
||||
wantMethod: "GET",
|
||||
},
|
||||
{
|
||||
name: "blank-line-before-response",
|
||||
data: "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 27\r\n\r\n" + body,
|
||||
wantProto: "HTTP/1.1",
|
||||
wantStatus: "200 OK",
|
||||
wantMethod: "GET",
|
||||
},
|
||||
{
|
||||
name: "request-with-body-then-response",
|
||||
data: "POST /search HTTP/1.1\r\nHost: example.com\r\nContent-Length: 3\r\n\r\nq=1HTTP/1.1 200 OK\r\nContent-Length: 27\r\n\r\n" + body,
|
||||
wantProto: "HTTP/1.1",
|
||||
wantStatus: "200 OK",
|
||||
wantMethod: "POST",
|
||||
},
|
||||
{
|
||||
name: "request-without-saved-body-then-response",
|
||||
data: "POST /search HTTP/1.1\r\nHost: example.com\r\nContent-Length: 3\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 27\r\n\r\n" + body,
|
||||
wantProto: "HTTP/1.1",
|
||||
wantStatus: "200 OK",
|
||||
wantMethod: "POST",
|
||||
},
|
||||
{
|
||||
name: "request-then-interim-response",
|
||||
data: "POST /search HTTP/1.1\r\nHost: example.com\r\nExpect: 100-continue\r\nContent-Length: 3\r\n\r\nq=1HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 27\r\n\r\n" + body,
|
||||
wantProto: "HTTP/1.1",
|
||||
wantStatus: "200 OK",
|
||||
wantMethod: "POST",
|
||||
},
|
||||
{
|
||||
name: "interim-response-without-minor-version",
|
||||
data: "HTTP/2 103 Early Hints\r\nLink: </style.css>; rel=preload\r\n\r\nHTTP/2 200 OK\r\nContent-Length: 27\r\n\r\n" + body,
|
||||
wantProto: "HTTP/2.0",
|
||||
wantStatus: "200 OK",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -232,11 +303,102 @@ func TestReadResponse_HTTPInBodyCases(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
resp, err := readResponseFromString(tt.data)
|
||||
require.NoError(t, err, "could not read response from string")
|
||||
require.Equal(t, tt.statusCode, resp.StatusCode)
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tt.expectedBody, string(body))
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err, "could not read response body")
|
||||
require.Equal(t, body, string(respBody), "could not get correct parsed body")
|
||||
require.Equal(t, tt.wantProto, resp.Proto, "could not get correct protocol")
|
||||
require.Equal(t, tt.wantStatus, resp.Status, "could not get correct status")
|
||||
require.Equal(t, tt.wantVia, resp.Header.Get("Via"), "could not get correct headers")
|
||||
|
||||
var gotMethod string
|
||||
if resp.Request != nil {
|
||||
gotMethod = resp.Request.Method
|
||||
}
|
||||
require.Equal(t, tt.wantMethod, gotMethod, "could not get correct request")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadResponseFromStringInterimResponses(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data string
|
||||
wantStatus string
|
||||
wantMethod string
|
||||
}{
|
||||
{
|
||||
name: "interim-response-only",
|
||||
data: "HTTP/1.1 100 Continue\r\n\r\n",
|
||||
wantStatus: "100 Continue",
|
||||
},
|
||||
{
|
||||
name: "request-then-interim-response-only",
|
||||
data: "POST /search HTTP/1.1\r\nHost: example.com\r\nContent-Length: 3\r\n\r\nq=1HTTP/1.1 100 Continue\r\n\r\n",
|
||||
wantStatus: "100 Continue",
|
||||
wantMethod: "POST",
|
||||
},
|
||||
{
|
||||
name: "request-then-interim-then-final-response",
|
||||
data: "POST /search HTTP/1.1\r\nHost: example.com\r\nExpect: 100-continue\r\nContent-Length: 3\r\n\r\nq=1HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok",
|
||||
wantStatus: "200 OK",
|
||||
wantMethod: "POST",
|
||||
},
|
||||
{
|
||||
name: "request-then-interim-then-final-response-with-status-line-in-body",
|
||||
data: "GET / HTTP/1.1\r\nHost: example.com\r\n\r\nHTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 200 OK\r\n\r\nsee HTTP/1.1 404 fake\r\n\r\nx",
|
||||
wantStatus: "200 OK",
|
||||
wantMethod: "GET",
|
||||
},
|
||||
{
|
||||
name: "request-then-interim-then-blank-line-then-final-response-with-status-line-in-body",
|
||||
data: "GET / HTTP/1.1\r\nHost: example.com\r\n\r\nHTTP/1.1 100 Continue\r\n\r\n\r\nHTTP/1.1 200 OK\r\n\r\nsee HTTP/1.1 404 fake\r\n\r\nx",
|
||||
wantStatus: "200 OK",
|
||||
wantMethod: "GET",
|
||||
},
|
||||
{
|
||||
name: "too-many-interim-responses",
|
||||
data: strings.Repeat("HTTP/2 103 Early Hints\r\n\r\n", 6) + "HTTP/2 200 OK\r\n\r\n",
|
||||
wantStatus: "103 Early Hints",
|
||||
},
|
||||
{
|
||||
name: "request-then-interim-then-blank-line-then-final-response",
|
||||
data: "POST /search HTTP/1.1\r\nHost: example.com\r\nExpect: 100-continue\r\nContent-Length: 3\r\n\r\nq=1HTTP/1.1 100 Continue\r\n\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok",
|
||||
wantStatus: "200 OK",
|
||||
wantMethod: "POST",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
resp, err := readResponseFromString(tt.data)
|
||||
require.NoError(t, err, "could not read response from string")
|
||||
require.Equal(t, tt.wantStatus, resp.Status, "could not get correct status")
|
||||
|
||||
var gotMethod string
|
||||
if resp.Request != nil {
|
||||
gotMethod = resp.Request.Method
|
||||
}
|
||||
require.Equal(t, tt.wantMethod, gotMethod, "could not get correct request")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadResponseFromStringStopsAtFirstResponse(t *testing.T) {
|
||||
data := "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n\r\n" +
|
||||
"HTTP/1.1 301 Moved Permanently\r\nLocation: /next\r\nContent-Length: 0\r\n\r\n" +
|
||||
"GET /next HTTP/1.1\r\nHost: example.com\r\n\r\n" +
|
||||
"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"
|
||||
resp, err := readResponseFromString(data)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "301 Moved Permanently", resp.Status)
|
||||
require.Equal(t, "GET", resp.Request.Method)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, body)
|
||||
}
|
||||
|
||||
func TestReadResponseFromStringDoesNotSearchForHTTP(t *testing.T) {
|
||||
_, err := readResponseFromString("prefix HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
|
||||
require.Error(t, err)
|
||||
}
|
||||
@@ -573,10 +573,12 @@ func MakeDefaultResultEvent(request Request, wrapped *output.InternalWrappedEven
|
||||
data := request.MakeResultEventItem(wrapped)
|
||||
results = append(results, data)
|
||||
}
|
||||
// Every protocol builds its results here, so attaching the llm audit at this
|
||||
// one point keeps the verdict on findings from all of them rather than only
|
||||
// the protocol that remembers to ask. It is a no-op when no llm operator ran.
|
||||
return AttachLLMAudits(results, wrapped.InternalEvent)
|
||||
// Every protocol builds its results here, so attaching the llm audit and
|
||||
// rendering the info at this one point covers findings from all of them
|
||||
// rather than only the protocol that remembers to ask. Each is a no-op when
|
||||
// it has nothing to do: no llm operator ran, or the info has no placeholders.
|
||||
results = AttachLLMAudits(results, wrapped.InternalEvent)
|
||||
return renderInfo(results, wrapped.InternalEvent)
|
||||
}
|
||||
|
||||
// MakeDefaultExtractFunc performs extracting operation for an extractor on model and returns true or false.
|
||||
@@ -625,7 +627,7 @@ func MakeDefaultMatchFuncWithOptions(data map[string]interface{}, matcher *match
|
||||
}
|
||||
|
||||
partItem, ok := data[part]
|
||||
if !ok && matcher.Type.MatcherType != matchers.DSLMatcher {
|
||||
if !ok && matcher.NeedsPart() {
|
||||
return false, nil
|
||||
}
|
||||
item := types.ToString(partItem)
|
||||
@@ -644,6 +646,8 @@ func MakeDefaultMatchFuncWithOptions(data map[string]interface{}, matcher *match
|
||||
return matcher.Result(matcher.MatchDSLWithOptions(data, options)), nil
|
||||
case matchers.XPathMatcher:
|
||||
return matcher.Result(matcher.MatchXPath(item)), []string{}
|
||||
case matchers.ErrorMatcher:
|
||||
return matcher.ResultWithMatchedSnippet(matcher.MatchError(data))
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
|
||||
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
@@ -108,9 +109,12 @@ func (request *Request) TmplClusterKey() uint64 {
|
||||
return xxhash.Sum64String(inp)
|
||||
}
|
||||
|
||||
// IsClusterable returns true if the request is eligible to be clustered.
|
||||
// Error matchers are excluded: the cluster executer only shares responses, so
|
||||
// a failed request would never reach them.
|
||||
func (request *Request) IsClusterable() bool {
|
||||
// nolint
|
||||
return !(len(request.CipherSuites) > 0 || request.MinVersion != "" || request.MaxVersion != "")
|
||||
return !(len(request.CipherSuites) > 0 || request.MinVersion != "" || request.MaxVersion != "" || request.HasErrorMatchers())
|
||||
}
|
||||
|
||||
// Compile compiles the request generators preparing any requests possible.
|
||||
@@ -272,6 +276,23 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
||||
if err != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
if request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers() {
|
||||
data := make(map[string]interface{})
|
||||
maps.Copy(data, payloadValues)
|
||||
data["type"] = request.Type().String()
|
||||
data["host"] = input.MetaInput.Input
|
||||
data["matched"] = addressToDial
|
||||
data["duration"] = duration.Seconds()
|
||||
data["template-path"] = requestOptions.TemplatePath
|
||||
data["template-id"] = requestOptions.TemplateID
|
||||
data["template-info"] = requestOptions.TemplateInfo
|
||||
if input.MetaInput.CustomIP != "" {
|
||||
data["ip"] = hostIp
|
||||
}
|
||||
requesterr.Annotate(data, err, duration)
|
||||
event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse)
|
||||
callback(event)
|
||||
}
|
||||
return errkit.Wrap(err, "could not connect to server")
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
package requesterr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Kind classifies a request execution error for matcher / DSL use.
|
||||
type Kind string
|
||||
|
||||
const (
|
||||
KindTimeout Kind = "timeout"
|
||||
KindConnection Kind = "connection"
|
||||
KindUnknown Kind = "unknown"
|
||||
)
|
||||
|
||||
const provenanceKey = "\x00nuclei-request-error"
|
||||
|
||||
var provenanceToken = &struct{ internal byte }{}
|
||||
|
||||
// Mark adds internal provenance that the error fields came from request execution.
|
||||
// The token uses pointer identity so response-controlled fields cannot forge it.
|
||||
func Mark(event map[string]interface{}) {
|
||||
if event != nil {
|
||||
event[provenanceKey] = provenanceToken
|
||||
}
|
||||
}
|
||||
|
||||
// IsMarked reports whether an event has request-error provenance.
|
||||
func IsMarked(event map[string]interface{}) bool {
|
||||
if event == nil {
|
||||
return false
|
||||
}
|
||||
return event[provenanceKey] == provenanceToken
|
||||
}
|
||||
|
||||
// Unmark removes internal request-error provenance before an event is emitted.
|
||||
func Unmark(event map[string]interface{}) {
|
||||
delete(event, provenanceKey)
|
||||
}
|
||||
|
||||
// Classify returns the error kind and whether it is a timeout.
|
||||
func Classify(err error) (Kind, bool) {
|
||||
if err == nil {
|
||||
return "", false
|
||||
}
|
||||
if isTimeoutErr(err) {
|
||||
return KindTimeout, true
|
||||
}
|
||||
if isConnectionErr(err) {
|
||||
return KindConnection, false
|
||||
}
|
||||
return KindUnknown, false
|
||||
}
|
||||
|
||||
// Annotate adds error, error_type, timeout, and optional duration fields to an event map.
|
||||
func Annotate(event map[string]interface{}, err error, duration time.Duration) {
|
||||
if event == nil || err == nil {
|
||||
return
|
||||
}
|
||||
kind, isTimeout := Classify(err)
|
||||
event["error"] = err.Error()
|
||||
event["error_type"] = string(kind)
|
||||
event["timeout"] = isTimeout
|
||||
if duration > 0 {
|
||||
event["duration"] = duration.Seconds()
|
||||
}
|
||||
}
|
||||
|
||||
func isTimeoutErr(err error) bool {
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, os.ErrDeadlineExceeded) {
|
||||
return true
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) && netErr.Timeout() {
|
||||
return true
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
return strings.Contains(msg, "timeout") ||
|
||||
strings.Contains(msg, "deadline exceeded") ||
|
||||
strings.Contains(msg, "i/o timeout") ||
|
||||
strings.Contains(msg, "client.timeout")
|
||||
}
|
||||
|
||||
func isConnectionErr(err error) bool {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return true
|
||||
}
|
||||
if errors.Is(err, syscall.ECONNREFUSED) ||
|
||||
errors.Is(err, syscall.ECONNRESET) ||
|
||||
errors.Is(err, syscall.EPIPE) ||
|
||||
errors.Is(err, syscall.ENETUNREACH) ||
|
||||
errors.Is(err, syscall.EHOSTUNREACH) {
|
||||
return true
|
||||
}
|
||||
var opErr *net.OpError
|
||||
if errors.As(err, &opErr) {
|
||||
return true
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
return strings.Contains(msg, "connection refused") ||
|
||||
strings.Contains(msg, "actively refused") ||
|
||||
strings.Contains(msg, "connection reset") ||
|
||||
strings.Contains(msg, "no such host") ||
|
||||
strings.Contains(msg, "network is unreachable") ||
|
||||
strings.Contains(msg, "broken pipe")
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package requesterr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestClassifyNil(t *testing.T) {
|
||||
kind, isTimeout := Classify(nil)
|
||||
if kind != "" || isTimeout {
|
||||
t.Fatalf("expected empty kind for nil, got kind=%s isTimeout=%v", kind, isTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyTimeout(t *testing.T) {
|
||||
cases := []error{
|
||||
context.DeadlineExceeded,
|
||||
os.ErrDeadlineExceeded,
|
||||
errors.New("Client.Timeout exceeded while awaiting headers"),
|
||||
errors.New("i/o timeout"),
|
||||
errors.New("context deadline exceeded"),
|
||||
}
|
||||
for _, err := range cases {
|
||||
kind, isTimeout := Classify(err)
|
||||
if kind != KindTimeout || !isTimeout {
|
||||
t.Fatalf("expected timeout for %v, got kind=%s isTimeout=%v", err, kind, isTimeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyConnection(t *testing.T) {
|
||||
cases := []error{
|
||||
syscall.ECONNREFUSED,
|
||||
syscall.ECONNRESET,
|
||||
io.EOF,
|
||||
&net.OpError{Op: "dial", Err: errors.New("connect: connection refused")},
|
||||
errors.New("connection refused"),
|
||||
errors.New("connectex: No connection could be made because the target machine actively refused it"),
|
||||
errors.New("no such host"),
|
||||
errors.New("network is unreachable"),
|
||||
}
|
||||
for _, err := range cases {
|
||||
kind, isTimeout := Classify(err)
|
||||
if kind != KindConnection || isTimeout {
|
||||
t.Fatalf("expected connection for %v, got kind=%s isTimeout=%v", err, kind, isTimeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyUnknown(t *testing.T) {
|
||||
kind, isTimeout := Classify(errors.New("something else entirely"))
|
||||
if kind != KindUnknown || isTimeout {
|
||||
t.Fatalf("expected unknown, got kind=%s isTimeout=%v", kind, isTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnnotate(t *testing.T) {
|
||||
event := map[string]interface{}{}
|
||||
Annotate(event, context.DeadlineExceeded, 1500*time.Millisecond)
|
||||
if event["error"] == nil || event["error_type"] != string(KindTimeout) || event["timeout"] != true {
|
||||
t.Fatalf("unexpected annotate result: %#v", event)
|
||||
}
|
||||
if event["duration"].(float64) != 1.5 {
|
||||
t.Fatalf("unexpected duration: %v", event["duration"])
|
||||
}
|
||||
|
||||
Annotate(nil, context.DeadlineExceeded, time.Second) // no panic
|
||||
Annotate(map[string]interface{}{}, nil, time.Second) // no fields
|
||||
}
|
||||
|
||||
func TestProvenanceMarker(t *testing.T) {
|
||||
event := map[string]interface{}{
|
||||
"error": "server controlled",
|
||||
"error_type": "timeout",
|
||||
}
|
||||
if IsMarked(event) {
|
||||
t.Fatal("public error fields must not establish request-error provenance")
|
||||
}
|
||||
|
||||
Mark(event)
|
||||
if !IsMarked(event) {
|
||||
t.Fatal("expected internally marked event")
|
||||
}
|
||||
Unmark(event)
|
||||
if IsMarked(event) {
|
||||
t.Fatal("expected provenance to be removed")
|
||||
}
|
||||
}
|
||||
@@ -32,6 +32,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
|
||||
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
@@ -285,6 +286,19 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
|
||||
if err != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
if request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers() {
|
||||
data := make(map[string]interface{})
|
||||
data["type"] = request.Type().String()
|
||||
data["host"] = input
|
||||
data["matched"] = addressToDial
|
||||
data["duration"] = handshakeDuration.Seconds()
|
||||
data["ip"] = request.dialer.GetDialedIP(hostname)
|
||||
maps.Copy(data, previous)
|
||||
maps.Copy(data, payloadValues)
|
||||
requesterr.Annotate(data, err, handshakeDuration)
|
||||
event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse)
|
||||
callback(event)
|
||||
}
|
||||
|
||||
return errors.Wrap(err, "could not connect to server")
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/whois/rdapclientpool"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
@@ -132,6 +133,15 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
||||
res, err := request.client.Do(rdapReq)
|
||||
duration := time.Since(timeStart)
|
||||
if err != nil {
|
||||
if request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers() {
|
||||
data := make(map[string]interface{})
|
||||
data["type"] = request.Type().String()
|
||||
data["host"] = query
|
||||
data["duration"] = duration.Seconds()
|
||||
requesterr.Annotate(data, err, duration)
|
||||
event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse)
|
||||
callback(event)
|
||||
}
|
||||
return errors.Wrap(err, "could not make whois request")
|
||||
}
|
||||
gologger.Verbose().Msgf("Sent WHOIS request to %s", query)
|
||||
|
||||
@@ -20,6 +20,11 @@ const (
|
||||
fallbackTemplateRuleID = "nuclei-template"
|
||||
)
|
||||
|
||||
// maxInfoValueLength keeps filled info whole for every template in
|
||||
// nuclei-templates (the longest field is under 3000 characters) while a
|
||||
// placeholder such as {{body}} cannot pour a whole response into the report.
|
||||
const maxInfoValueLength = 4096
|
||||
|
||||
// Exporter is an exporter for nuclei sarif output format.
|
||||
type Exporter struct {
|
||||
sarif *sarif.Report
|
||||
@@ -130,6 +135,7 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
|
||||
}
|
||||
|
||||
// rule contain details of template
|
||||
templateEvent := asWritten(event)
|
||||
rule := sarif.ReportingDescriptor{
|
||||
Id: ruleID,
|
||||
Name: ruleName,
|
||||
@@ -138,11 +144,11 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
|
||||
Text: ruleName,
|
||||
},
|
||||
FullDescription: &sarif.MultiformatMessageString{
|
||||
Text: buildRuleDescription(event, ruleName),
|
||||
Text: buildRuleDescription(templateEvent, ruleName),
|
||||
},
|
||||
Help: &sarif.MultiformatMessageString{
|
||||
Text: buildRuleHelpText(event),
|
||||
Markdown: buildRuleHelpMarkdown(event, ruleName),
|
||||
Text: buildRuleHelpText(templateEvent),
|
||||
Markdown: buildRuleHelpMarkdown(templateEvent, ruleName),
|
||||
},
|
||||
Properties: ghMeta,
|
||||
}
|
||||
@@ -203,6 +209,18 @@ func (exporter *Exporter) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// asWritten returns the event with its info as the template wrote it. Every
|
||||
// finding of a template shares one rule, so the rule describes the template;
|
||||
// text filled for one finding goes on that finding's result message instead.
|
||||
func asWritten(event *output.ResultEvent) *output.ResultEvent {
|
||||
if event.TemplateInfo == nil {
|
||||
return event
|
||||
}
|
||||
written := *event
|
||||
written.Info = *event.TemplateInfo
|
||||
return &written
|
||||
}
|
||||
|
||||
func buildRuleDescription(event *output.ResultEvent, ruleName string) string {
|
||||
description := normalizeValue(event.Info.Description)
|
||||
if description == "" {
|
||||
@@ -275,6 +293,13 @@ func buildRuleHelpMarkdown(event *output.ResultEvent, ruleName string) string {
|
||||
|
||||
func buildResultMessage(event *output.ResultEvent, resultHeader, ruleID string) *sarif.Message {
|
||||
detailLines := []string{}
|
||||
// Filled info is all the report has of this finding's text, so it gets a
|
||||
// longer limit than the other details.
|
||||
for _, field := range event.Info.FilledFields(event.TemplateInfo) {
|
||||
if value := normalizeAndTruncate(*field.Value, maxInfoValueLength); value != "" {
|
||||
detailLines = append(detailLines, field.Label+": "+value)
|
||||
}
|
||||
}
|
||||
target := getResultTarget(event)
|
||||
addDetail(&detailLines, "Target", target)
|
||||
if matched := normalizeAndTruncate(event.Matched, maxDetailValueLength); matched != "" && matched != target {
|
||||
|
||||
@@ -431,6 +431,78 @@ func TestExport_StableRuleIndices(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The rule describes the template whatever finding arrives first, and each
|
||||
// finding carries the text filled for it, while text the template wrote stays
|
||||
// on the rule.
|
||||
func TestExport_FilledInfoStaysOnTheFinding(t *testing.T) {
|
||||
out := filepath.Join(t.TempDir(), "report.sarif")
|
||||
exp, err := New(&Options{File: out})
|
||||
require.NoError(t, err)
|
||||
|
||||
written := makeEvent().Info
|
||||
written.Description = "Service {{version}} found on {{host}}:6379"
|
||||
first := makeEvent()
|
||||
first.TemplateInfo = &written
|
||||
first.Info.Description = "Service 6.0.9 found on 10.0.0.2:6379"
|
||||
second := makeEvent()
|
||||
second.TemplateInfo = &written
|
||||
second.Info.Description = "Service 7.2.4 found on 10.0.0.1:6379"
|
||||
require.NoError(t, exp.Export(first))
|
||||
require.NoError(t, exp.Export(second))
|
||||
require.NoError(t, exp.Close())
|
||||
|
||||
bin, err := os.ReadFile(out)
|
||||
require.NoError(t, err)
|
||||
var log upstream.SarifLog
|
||||
require.NoError(t, json.Unmarshal(bin, &log))
|
||||
run := log.Runs[0]
|
||||
|
||||
require.Len(t, run.Tool.Driver.Rules, 1)
|
||||
rule := run.Tool.Driver.Rules[0]
|
||||
require.Contains(t, rule.FullDescription.Text, "Service {{version}} found on {{host}}:6379")
|
||||
require.NotContains(t, rule.FullDescription.Text, "6.0.9")
|
||||
require.Contains(t, rule.Help.Text, "Impact: Severe impact")
|
||||
require.Len(t, run.Result, 2)
|
||||
require.Contains(t, run.Result[0].Message.Text, "Description: Service 6.0.9 found on 10.0.0.2:6379")
|
||||
require.Contains(t, run.Result[1].Message.Text, "Description: Service 7.2.4 found on 10.0.0.1:6379")
|
||||
require.NotContains(t, run.Result[1].Message.Text, "Impact:")
|
||||
}
|
||||
|
||||
func TestBuildResultMessageKeepsFilledInfoWhole(t *testing.T) {
|
||||
event := makeEvent()
|
||||
written := event.Info
|
||||
event.TemplateInfo = &written
|
||||
event.Info.Description = strings.Repeat("x", 2*maxDetailValueLength)
|
||||
|
||||
message := buildResultMessage(event, "header", "test-template")
|
||||
|
||||
require.Contains(t, message.Text, "Description: "+event.Info.Description)
|
||||
}
|
||||
|
||||
// A placeholder such as {{body}} cannot pour a whole response into the report.
|
||||
func TestBuildResultMessageCapsFilledInfo(t *testing.T) {
|
||||
event := makeEvent()
|
||||
written := event.Info
|
||||
event.TemplateInfo = &written
|
||||
event.Info.Description = strings.Repeat("x", 2*maxInfoValueLength)
|
||||
|
||||
message := buildResultMessage(event, "header", "test-template")
|
||||
|
||||
require.Contains(t, message.Text, "Description: "+strings.Repeat("x", maxInfoValueLength-3)+"...")
|
||||
require.NotContains(t, message.Text, strings.Repeat("x", maxInfoValueLength))
|
||||
}
|
||||
|
||||
// Static info describes the template, so it stays on the rule as before.
|
||||
func TestExport_StaticInfoStaysOnTheRule(t *testing.T) {
|
||||
exp, err := New(&Options{File: filepath.Join(t.TempDir(), "report.sarif")})
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, exp.Export(makeEvent()))
|
||||
|
||||
require.Contains(t, exp.rules[0].FullDescription.Text, "A test vulnerability description.")
|
||||
require.NotContains(t, buildResultMessage(makeEvent(), "header", "test-template").Text, "Description:")
|
||||
}
|
||||
|
||||
func TestExport_CVSSScoreOverridesSeverity(t *testing.T) {
|
||||
out := filepath.Join(t.TempDir(), "report.sarif")
|
||||
exp, err := New(&Options{File: out})
|
||||
|
||||
@@ -156,6 +156,7 @@ func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// findIssueByTitle searches every linked issue page for an exact title match.
|
||||
func (i *Integration) findIssueByTitle(title string) (*gitea.Issue, error) {
|
||||
// Fetch issues in pages to ensure older issues are also checked for duplicates.
|
||||
pageSize := i.options.DuplicateIssuePageSize
|
||||
@@ -177,7 +178,7 @@ func (i *Integration) findIssueByTitle(title string) (*gitea.Issue, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
issueList, _, err := i.client.ListRepoIssues(i.options.ProjectOwner, i.options.ProjectName, opts)
|
||||
issueList, response, err := i.client.ListRepoIssues(i.options.ProjectOwner, i.options.ProjectName, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -188,12 +189,14 @@ func (i *Integration) findIssueByTitle(title string) (*gitea.Issue, error) {
|
||||
}
|
||||
}
|
||||
|
||||
if len(issueList) < opts.PageSize {
|
||||
// Last page reached.
|
||||
if response == nil || response.NextPage == 0 {
|
||||
// Gitea's Link header indicates there is no next page.
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
opts.Page++
|
||||
if response.NextPage <= opts.Page {
|
||||
return nil, fmt.Errorf("gitea returned a non-advancing issue page: current page %d, next page %d", opts.Page, response.NextPage)
|
||||
}
|
||||
opts.Page = response.NextPage
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
package gitea
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const testIssueTitle = "Test finding (template-id) found on example.com"
|
||||
|
||||
type issueRequest struct {
|
||||
Page string
|
||||
Limit string
|
||||
State string
|
||||
Keyword string
|
||||
}
|
||||
|
||||
type fakeGitea struct {
|
||||
server *httptest.Server
|
||||
requests []issueRequest
|
||||
comments []int64
|
||||
created int
|
||||
issueHandler func(http.ResponseWriter, *http.Request, issueRequest)
|
||||
}
|
||||
|
||||
// newFakeGitea starts a local Gitea API stub and returns its tracker integration.
|
||||
func newFakeGitea(t *testing.T, issueHandler func(http.ResponseWriter, *http.Request, issueRequest)) (*Integration, *fakeGitea) {
|
||||
t.Helper()
|
||||
|
||||
fake := &fakeGitea{issueHandler: issueHandler}
|
||||
fake.server = httptest.NewServer(http.HandlerFunc(fake.serveHTTP))
|
||||
t.Cleanup(fake.server.Close)
|
||||
|
||||
integration, err := New(&Options{
|
||||
BaseURL: fake.server.URL,
|
||||
Token: "test-token",
|
||||
ProjectOwner: "owner",
|
||||
ProjectName: "repo",
|
||||
DuplicateIssueCheck: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return integration, fake
|
||||
}
|
||||
|
||||
// serveHTTP records tracker requests and serves the configured fake API responses.
|
||||
func (f *fakeGitea) serveHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch {
|
||||
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/version"):
|
||||
_, _ = w.Write([]byte(`{"version":"1.21.0"}`))
|
||||
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/labels"):
|
||||
_, _ = w.Write([]byte("[]"))
|
||||
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/issues"):
|
||||
query := r.URL.Query()
|
||||
request := issueRequest{
|
||||
Page: query.Get("page"),
|
||||
Limit: query.Get("limit"),
|
||||
State: query.Get("state"),
|
||||
Keyword: query.Get("q"),
|
||||
}
|
||||
f.requests = append(f.requests, request)
|
||||
f.issueHandler(w, r, request)
|
||||
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/comments"):
|
||||
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
|
||||
issueNumber, _ := strconv.ParseInt(parts[len(parts)-2], 10, 64)
|
||||
f.comments = append(f.comments, issueNumber)
|
||||
_, _ = w.Write([]byte(`{"id":1}`))
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/issues"):
|
||||
f.created++
|
||||
_, _ = w.Write([]byte(`{"number":99,"title":"created","url":"https://gitea.test/issues/99"}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
// writeIssuePage writes issue objects as a JSON API response.
|
||||
func writeIssuePage(w http.ResponseWriter, issues ...map[string]any) {
|
||||
_ = json.NewEncoder(w).Encode(issues)
|
||||
}
|
||||
|
||||
// TestCreateIssueFollowsLinkAfterCappedPageAndComments verifies duplicate detection continues after a capped page.
|
||||
func TestCreateIssueFollowsLinkAfterCappedPageAndComments(t *testing.T) {
|
||||
var server *httptest.Server
|
||||
integration, fake := newFakeGitea(t, func(w http.ResponseWriter, r *http.Request, request issueRequest) {
|
||||
if request.Page == "1" {
|
||||
// Gitea may return only 50 items despite a requested limit of 100.
|
||||
w.Header().Set("Link", fmt.Sprintf("<%s%s?limit=100&page=2&state=all>; rel=\"next\"", server.URL, r.URL.Path))
|
||||
issues := make([]map[string]any, 50)
|
||||
for index := range issues {
|
||||
issues[index] = map[string]any{"number": index + 1, "title": "unrelated issue"}
|
||||
}
|
||||
writeIssuePage(w, issues...)
|
||||
return
|
||||
}
|
||||
writeIssuePage(w, map[string]any{"number": 42, "title": testIssueTitle, "url": "https://gitea.test/issues/42"})
|
||||
})
|
||||
server = fake.server
|
||||
|
||||
response, err := integration.CreateIssue(testGiteaEvent())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "42", response.IssueID)
|
||||
require.Equal(t, "https://gitea.test/issues/42", response.IssueURL)
|
||||
require.Equal(t, []int64{42}, fake.comments)
|
||||
require.Zero(t, fake.created)
|
||||
require.Equal(t, []string{"1", "2"}, []string{fake.requests[0].Page, fake.requests[1].Page})
|
||||
for _, request := range fake.requests {
|
||||
require.Equal(t, "100", request.Limit)
|
||||
require.Equal(t, "all", request.State)
|
||||
require.Empty(t, request.Keyword)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFindIssueByTitleRespectsMaxPages verifies the configured page limit stops further requests.
|
||||
func TestFindIssueByTitleRespectsMaxPages(t *testing.T) {
|
||||
var server *httptest.Server
|
||||
integration, fake := newFakeGitea(t, func(w http.ResponseWriter, r *http.Request, request issueRequest) {
|
||||
w.Header().Set("Link", fmt.Sprintf("<%s%s?page=2>; rel=\"next\"", server.URL, r.URL.Path))
|
||||
writeIssuePage(w, map[string]any{"number": 1, "title": "unrelated"})
|
||||
})
|
||||
server = fake.server
|
||||
integration.options.DuplicateIssueMaxPages = 1
|
||||
|
||||
issue, err := integration.findIssueByTitle(testIssueTitle)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, issue)
|
||||
require.Len(t, fake.requests, 1)
|
||||
require.Equal(t, "1", fake.requests[0].Page)
|
||||
}
|
||||
|
||||
// TestFindIssueByTitleStopsWhenLinkHeaderHasNoNextPage verifies pagination ends without a next link.
|
||||
func TestFindIssueByTitleStopsWhenLinkHeaderHasNoNextPage(t *testing.T) {
|
||||
integration, fake := newFakeGitea(t, func(w http.ResponseWriter, _ *http.Request, _ issueRequest) {
|
||||
writeIssuePage(w, map[string]any{"number": 1, "title": "unrelated"})
|
||||
})
|
||||
|
||||
issue, err := integration.findIssueByTitle(testIssueTitle)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, issue)
|
||||
require.Len(t, fake.requests, 1)
|
||||
}
|
||||
|
||||
// TestFindIssueByTitleRejectsNonAdvancingPage verifies malformed links return an error instead of looping.
|
||||
func TestFindIssueByTitleRejectsNonAdvancingPage(t *testing.T) {
|
||||
var server *httptest.Server
|
||||
integration, fake := newFakeGitea(t, func(w http.ResponseWriter, r *http.Request, _ issueRequest) {
|
||||
w.Header().Set("Link", fmt.Sprintf("<%s%s?page=1>; rel=\"next\"", server.URL, r.URL.Path))
|
||||
writeIssuePage(w, map[string]any{"number": 1, "title": "unrelated"})
|
||||
})
|
||||
server = fake.server
|
||||
|
||||
issue, err := integration.findIssueByTitle(testIssueTitle)
|
||||
require.ErrorContains(t, err, "non-advancing issue page")
|
||||
require.Nil(t, issue)
|
||||
require.Len(t, fake.requests, 1)
|
||||
}
|
||||
|
||||
// testGiteaEvent builds a small result event for tracker tests.
|
||||
func testGiteaEvent() *output.ResultEvent {
|
||||
return &output.ResultEvent{
|
||||
TemplateID: "template-id",
|
||||
Host: "example.com",
|
||||
Info: model.Info{Name: "Test finding"},
|
||||
}
|
||||
}
|
||||
@@ -41,7 +41,8 @@ type Options struct {
|
||||
DenyList *filters.Filter `yaml:"deny-list"`
|
||||
// DuplicateIssueCheck is a bool to enable duplicate tracking issue check and update the newest
|
||||
DuplicateIssueCheck bool `yaml:"duplicate-issue-check" default:"false"`
|
||||
// DuplicateIssuePageSize controls how many issues are fetched per page when searching for duplicates.
|
||||
// DuplicateIssuePageSize is the number of issues to request per page when searching for duplicates.
|
||||
// GitLab returns at most 100 issues per page.
|
||||
// If unset or <=0, a default of 100 is used.
|
||||
DuplicateIssuePageSize int `yaml:"duplicate-issue-page-size" default:"100"`
|
||||
// DuplicateIssueMaxPages limits how many pages are fetched when searching for duplicates.
|
||||
@@ -161,23 +162,23 @@ func (i *Integration) findIssueByTitle(title string) (*gitlab.Issue, error) {
|
||||
if pageSize <= 0 {
|
||||
pageSize = 100
|
||||
}
|
||||
maxPages := i.options.DuplicateIssueMaxPages
|
||||
maxPages := int64(i.options.DuplicateIssueMaxPages)
|
||||
|
||||
searchIn := "title"
|
||||
searchState := "all"
|
||||
page := 1
|
||||
page := int64(1)
|
||||
|
||||
for {
|
||||
if maxPages > 0 && page > maxPages {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
issues, _, err := i.client.Issues.ListProjectIssues(i.options.ProjectName, &gitlab.ListProjectIssuesOptions{
|
||||
issues, resp, err := i.client.Issues.ListProjectIssues(i.options.ProjectName, &gitlab.ListProjectIssuesOptions{
|
||||
In: &searchIn,
|
||||
State: &searchState,
|
||||
Search: &title,
|
||||
ListOptions: gitlab.ListOptions{
|
||||
Page: int64(page),
|
||||
Page: page,
|
||||
PerPage: int64(pageSize),
|
||||
},
|
||||
})
|
||||
@@ -191,11 +192,14 @@ func (i *Integration) findIssueByTitle(title string) (*gitlab.Issue, error) {
|
||||
}
|
||||
}
|
||||
|
||||
if len(issues) < pageSize {
|
||||
// GitLab caps per_page at 100, so a short page is not necessarily
|
||||
// the last one. Follow X-Next-Page instead, and stop when it is
|
||||
// empty or does not move forward.
|
||||
if resp.NextPage <= page {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
page++
|
||||
page = resp.NextPage
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
package gitlab
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type fakeIssue struct {
|
||||
ID int64 `json:"id"`
|
||||
IID int64 `json:"iid"`
|
||||
Title string `json:"title"`
|
||||
}
|
||||
|
||||
// fakeGitLab serves the GitLab API endpoints that findIssueByTitle uses. Like
|
||||
// GitLab, it caps per_page at maxPerPage and names the next page in the
|
||||
// X-Next-Page header, which is empty on the last page. Every issue counts as
|
||||
// a search hit, which leaves the exact title match to the tracker.
|
||||
type fakeGitLab struct {
|
||||
issues []fakeIssue
|
||||
maxPerPage int
|
||||
|
||||
mu sync.Mutex
|
||||
pages []int
|
||||
}
|
||||
|
||||
func newFakeGitLab(issueCount, maxPerPage int) *fakeGitLab {
|
||||
f := &fakeGitLab{maxPerPage: maxPerPage}
|
||||
for n := 1; n <= issueCount; n++ {
|
||||
f.issues = append(f.issues, fakeIssue{ID: int64(n), IID: int64(n), Title: fmt.Sprintf("issue %d", n)})
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fakeGitLab) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/api/v4/user":
|
||||
_, _ = w.Write([]byte(`{"id":1}`))
|
||||
case "/api/v4/projects/1234/issues":
|
||||
f.listIssues(w, r)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeGitLab) listIssues(w http.ResponseWriter, r *http.Request) {
|
||||
page, _ := strconv.Atoi(r.URL.Query().Get("page"))
|
||||
perPage, _ := strconv.Atoi(r.URL.Query().Get("per_page"))
|
||||
if perPage <= 0 || perPage > f.maxPerPage {
|
||||
perPage = f.maxPerPage
|
||||
}
|
||||
|
||||
f.mu.Lock()
|
||||
f.pages = append(f.pages, page)
|
||||
f.mu.Unlock()
|
||||
|
||||
nextPage := ""
|
||||
if page*perPage < len(f.issues) {
|
||||
nextPage = strconv.Itoa(page + 1)
|
||||
}
|
||||
w.Header().Set("X-Next-Page", nextPage)
|
||||
|
||||
start := min((page-1)*perPage, len(f.issues))
|
||||
end := min(start+perPage, len(f.issues))
|
||||
_ = json.NewEncoder(w).Encode(f.issues[start:end])
|
||||
}
|
||||
|
||||
func (f *fakeGitLab) requestedPages() []int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return slices.Clone(f.pages)
|
||||
}
|
||||
|
||||
func TestFindIssueByTitle(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
title string
|
||||
maxPages int
|
||||
wantIID int64
|
||||
wantPages []int
|
||||
}{
|
||||
{
|
||||
name: "match after a server-capped first page",
|
||||
title: "issue 110",
|
||||
wantIID: 110,
|
||||
wantPages: []int{1, 2},
|
||||
},
|
||||
{
|
||||
name: "no match checks every page once",
|
||||
title: "missing",
|
||||
wantPages: []int{1, 2},
|
||||
},
|
||||
{
|
||||
name: "max pages stops the search",
|
||||
title: "issue 110",
|
||||
maxPages: 1,
|
||||
wantPages: []int{1},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// GitLab caps per_page at 100, so a configured page size of
|
||||
// 150 still splits 120 issues into two pages.
|
||||
fake := newFakeGitLab(120, 100)
|
||||
server := httptest.NewServer(fake)
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
integration, err := New(&Options{
|
||||
BaseURL: server.URL,
|
||||
Token: "token",
|
||||
ProjectName: "1234",
|
||||
DuplicateIssuePageSize: 150,
|
||||
DuplicateIssueMaxPages: tt.maxPages,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
issue, err := integration.findIssueByTitle(tt.title)
|
||||
require.NoError(t, err)
|
||||
if tt.wantIID == 0 {
|
||||
require.Nil(t, issue)
|
||||
} else {
|
||||
require.NotNil(t, issue)
|
||||
require.Equal(t, tt.wantIID, issue.IID)
|
||||
}
|
||||
require.Equal(t, tt.wantPages, fake.requestedPages())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -55,8 +55,11 @@ func Cluster(list []*Template) [][]*Template {
|
||||
|
||||
// Split up templates that might be clusterable
|
||||
for _, template := range list {
|
||||
// it is not possible to cluster flow and multiprotocol due to dependent execution
|
||||
if template.Flow != "" || template.Options.IsMultiProtocol {
|
||||
// it is not possible to cluster flow and multiprotocol due to dependent execution.
|
||||
// Info placeholders are filled from the event a finding came from, and a
|
||||
// cluster shares one event that already holds the first template's variables
|
||||
// and extracted values, so a template with them runs on its own.
|
||||
if template.Flow != "" || template.Options.IsMultiProtocol || template.Info.HasPlaceholders() {
|
||||
final = append(final, []*Template{template})
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -5,8 +5,10 @@ import (
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -38,6 +40,32 @@ func TestClusterTemplates(t *testing.T) {
|
||||
got := Cluster(tpls)
|
||||
require.ElementsMatch(t, expected, got)
|
||||
})
|
||||
// A cluster shares one event that holds the first template's variables and
|
||||
// extracted values, so a template with info placeholders runs on its own,
|
||||
// even next to one with the same variables.
|
||||
t.Run("no-cluster-with-info-placeholders", func(t *testing.T) {
|
||||
withVariables := func(path, product string) *Template {
|
||||
vars := variables.Variable{InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1)}
|
||||
vars.Set("product", product)
|
||||
tp := &Template{
|
||||
Path: path,
|
||||
RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}},
|
||||
Info: model.Info{Description: "{{product}} panel exposed"},
|
||||
Variables: vars,
|
||||
}
|
||||
tp.Options = execOptions
|
||||
return tp
|
||||
}
|
||||
grafana := withVariables("grafana.yaml", "Grafana")
|
||||
alsoGrafana := withVariables("also-grafana.yaml", "Grafana")
|
||||
static := &Template{Path: "static.yaml", RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}}
|
||||
alsoStatic := &Template{Path: "also-static.yaml", RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}}
|
||||
static.Options = execOptions
|
||||
alsoStatic.Options = execOptions
|
||||
|
||||
got := Cluster([]*Template{grafana, alsoGrafana, static, alsoStatic})
|
||||
require.ElementsMatch(t, [][]*Template{{grafana}, {alsoGrafana}, {static, alsoStatic}}, got)
|
||||
})
|
||||
t.Run("dns-cluster", func(t *testing.T) {
|
||||
tp1 := &Template{Path: "first.yaml", RequestsDNS: []*dns.Request{{Name: "{{Hostname}}"}}}
|
||||
tp2 := &Template{Path: "second.yaml", RequestsDNS: []*dns.Request{{Name: "{{Hostname}}"}}}
|
||||
|
||||
@@ -203,16 +203,18 @@ func init() {
|
||||
MODELInfoDoc.Fields[3].Name = "description"
|
||||
MODELInfoDoc.Fields[3].Type = "string"
|
||||
MODELInfoDoc.Fields[3].Note = ""
|
||||
MODELInfoDoc.Fields[3].Description = "Description of the template.\n\nYou can go in-depth here on what the template actually does."
|
||||
MODELInfoDoc.Fields[3].Description = "Description of the template.\n\nYou can go in-depth here on what the template actually does.\n\nDescription, impact and remediation may hold {{placeholders}} filled for\neach finding: named extractors, payloads, variables and DSL helpers\n(network helpers excluded), and the result fields host, port, ip, url,\nscheme, path, matched-at, matcher-name and extractor-name, which win over\nvalues of the same name. Use join(\", \", name) for an extractor with\nseveral values, and brackets for a hyphenated name inside a helper:\nto_upper([matcher-name]). Unknown placeholders stay as written, and so\ndoes a field whose expression fails. Global matcher templates match on\nother templates' responses, so only the result fields fill their text."
|
||||
MODELInfoDoc.Fields[3].Comments[encoder.LineComment] = "Description of the template."
|
||||
|
||||
MODELInfoDoc.Fields[3].AddExample("", "Bower is a package manager which stores package information in the bower.json file")
|
||||
|
||||
MODELInfoDoc.Fields[3].AddExample("", "Subversion ALM for the enterprise before 8.8.2 allows reflected XSS at multiple locations")
|
||||
|
||||
MODELInfoDoc.Fields[3].AddExample("", "Redis {{version}} on {{host}}:{{port}} accepts the default password")
|
||||
MODELInfoDoc.Fields[4].Name = "impact"
|
||||
MODELInfoDoc.Fields[4].Type = "string"
|
||||
MODELInfoDoc.Fields[4].Note = ""
|
||||
MODELInfoDoc.Fields[4].Description = "Impact of the template.\n\nYou can go in-depth here on impact of the template."
|
||||
MODELInfoDoc.Fields[4].Description = "Impact of the template.\n\nYou can go in-depth here on impact of the template.\n\nSupports the same placeholders as description."
|
||||
MODELInfoDoc.Fields[4].Comments[encoder.LineComment] = "Impact of the template."
|
||||
|
||||
MODELInfoDoc.Fields[4].AddExample("", "Successful exploitation of this vulnerability could allow an attacker to execute arbitrary SQL queries, potentially leading to unauthorized access, data leakage, or data manipulation.")
|
||||
@@ -245,7 +247,7 @@ func init() {
|
||||
MODELInfoDoc.Fields[9].Name = "remediation"
|
||||
MODELInfoDoc.Fields[9].Type = "string"
|
||||
MODELInfoDoc.Fields[9].Note = ""
|
||||
MODELInfoDoc.Fields[9].Description = "Remediation steps for the template.\n\nYou can go in-depth here on how to mitigate the problem found by this template."
|
||||
MODELInfoDoc.Fields[9].Description = "Remediation steps for the template.\n\nYou can go in-depth here on how to mitigate the problem found by this template.\n\nSupports the same placeholders as description."
|
||||
MODELInfoDoc.Fields[9].Comments[encoder.LineComment] = "Remediation steps for the template."
|
||||
|
||||
MODELInfoDoc.Fields[9].AddExample("", "Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties")
|
||||
|
||||
Reference in new issue
Block a user