Compare commits

...
11 Commits
Author SHA1 Message Date
Mzack9999 943ec5df0f Merge dev 2026-10-27 16:12:11 +01:00
Mzack9999 ecdc9e4a35 Merge dev 2026-10-26 13:48:10 +01:00
Mzack9999 cc88919631 Fix negation 2026-10-26 10:54:25 +01:00
Mzack9999 d80cccd076 Fix output 2026-10-26 00:06:44 +01:00
Mzack9999 c0ad64ee5a Merge dev 2026-10-26 00:06:44 +01:00
Mzack9999 682225e1ad fix review 2026-09-24 20:37:32 +02:00
Mzack9999 b0bfc46e01 merge dev 2026-09-24 15:38:35 +02:00
Mzack9999 4489c63f6d harden errors 2026-09-24 00:50:53 +02:00
Mzack9999 0993a669e9 fix matchers 2026-09-23 21:58:13 +02:00
Mzack9999 c565fa9537 merge dev 2026-09-23 20:34:30 +02:00
Mzack9999 84e633d7c5 error matcher 2026-07-23 21:39:40 +04:00
38 changed files with 1290 additions and 45 deletions

No files matched your search

+109
View File
@@ -0,0 +1,109 @@
package operators
import (
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
"github.com/stretchr/testify/require"
)
func TestHasErrorMatchers(t *testing.T) {
t.Run("error type", func(t *testing.T) {
ops := &Operators{Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
Errors: []string{"timeout"},
}}}
require.NoError(t, ops.Compile())
require.True(t, ops.HasErrorMatchers())
})
t.Run("dsl timeout var", func(t *testing.T) {
ops := &Operators{Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
DSL: []string{"timeout == true"},
}}}
require.NoError(t, ops.Compile())
require.True(t, ops.HasErrorMatchers())
})
t.Run("dsl error_type var", func(t *testing.T) {
ops := &Operators{Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
DSL: []string{`error_type == "timeout"`},
}}}
require.NoError(t, ops.Compile())
require.True(t, ops.HasErrorMatchers())
})
t.Run("dsl error var", func(t *testing.T) {
ops := &Operators{Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
DSL: []string{`contains(error, "refused")`},
}}}
require.NoError(t, ops.Compile())
require.True(t, ops.HasErrorMatchers())
})
t.Run("dsl without error vars", func(t *testing.T) {
ops := &Operators{Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
DSL: []string{"status_code == 200"},
}}}
require.NoError(t, ops.Compile())
require.False(t, ops.HasErrorMatchers())
})
t.Run("dsl timeout as identifier prefix does not match", func(t *testing.T) {
ops := &Operators{Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
DSL: []string{"timeout_ms > 100"},
}}}
require.NoError(t, ops.Compile())
require.False(t, ops.HasErrorMatchers())
})
t.Run("dsl quoted error fields do not match", func(t *testing.T) {
for _, expression := range []string{
`contains(body, "timeout")`,
`contains(body, 'error')`,
`contains(body, "escaped \"timeout\"")`,
} {
ops := &Operators{Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
DSL: []string{expression},
}}}
require.NoError(t, ops.Compile())
require.False(t, ops.HasErrorMatchers(), expression)
}
})
t.Run("dsl indexed error fields match", func(t *testing.T) {
for _, expression := range []string{
"timeout_2 == true",
`error_type_10 == "connection"`,
`contains(error_3, "refused")`,
} {
ops := &Operators{Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
DSL: []string{expression},
}}}
require.NoError(t, ops.Compile())
require.True(t, ops.HasErrorMatchers(), expression)
}
})
t.Run("word matcher only", func(t *testing.T) {
ops := &Operators{Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"ok"},
Part: "body",
}}}
require.NoError(t, ops.Compile())
require.False(t, ops.HasErrorMatchers())
})
t.Run("nil operators", func(t *testing.T) {
var ops *Operators
require.False(t, ops.HasErrorMatchers())
})
}
+24 -7
View File
@@ -38,7 +38,7 @@ func (matcher *Matcher) CompileMatchers() error {
}
// By default, match on body if user hasn't provided any specific items
if matcher.Part == "" && matcher.GetType() != DSLMatcher {
if matcher.Part == "" && matcher.GetType() != DSLMatcher && matcher.GetType() != ErrorMatcher {
matcher.Part = "body"
}
@@ -90,17 +90,34 @@ func (matcher *Matcher) CompileMatchers() error {
}
if matcher.CaseInsensitive {
if matcher.GetType() != WordsMatcher {
return fmt.Errorf("case-insensitive flag is supported only for 'word' matchers (not '%s')", matcher.Type)
}
for i := range matcher.Words {
matcher.Words[i] = strings.ToLower(matcher.Words[i])
switch matcher.GetType() {
case WordsMatcher:
for i := range matcher.Words {
matcher.Words[i] = strings.ToLower(matcher.Words[i])
}
case ErrorMatcher:
// Applied at MatchError time so reserved kinds (timeout/connection/any) stay intact.
default:
return fmt.Errorf("case-insensitive flag is supported only for 'word' and 'error' matchers (not '%s')", matcher.Type)
}
}
return nil
}
// GetType returns the condition type of the matcher
// NeedsPart returns true when the matcher requires a response part corpus.
func (matcher *Matcher) NeedsPart() bool {
switch matcher.GetType() {
case DSLMatcher, ErrorMatcher:
return false
case LLMMatcher:
// an llm matcher with inputs compares whole responses instead of one part
return len(matcher.Inputs) == 0
default:
return true
}
}
// GetCondition returns the condition type of the matcher
// todo: the field should be exposed natively
func (matcher *Matcher) GetCondition() ConditionType {
return matcher.condition
+152
View File
@@ -0,0 +1,152 @@
package matchers
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestMatchError(t *testing.T) {
t.Run("any error when errors empty", func(t *testing.T) {
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}}
require.NoError(t, m.CompileMatchers())
ok, snippets := m.MatchError(map[string]interface{}{"error": "boom", "error_type": "unknown", "timeout": false})
require.True(t, m.Result(ok))
require.Equal(t, []string{"boom"}, snippets)
ok, _ = m.MatchError(map[string]interface{}{})
require.False(t, m.Result(ok))
})
t.Run("empty error string does not match", func(t *testing.T) {
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}, Errors: []string{"any"}}
require.NoError(t, m.CompileMatchers())
ok, _ := m.MatchError(map[string]interface{}{"error": "", "error_type": "unknown", "timeout": false})
require.False(t, m.Result(ok))
ok, _ = m.MatchError(map[string]interface{}{"error": nil})
require.False(t, m.Result(ok))
})
t.Run("timeout kind", func(t *testing.T) {
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}, Errors: []string{"timeout"}}
require.NoError(t, m.CompileMatchers())
ok, _ := m.MatchError(map[string]interface{}{"error": "i/o timeout", "error_type": "timeout", "timeout": true})
require.True(t, m.Result(ok))
ok, _ = m.MatchError(map[string]interface{}{"error": "connection refused", "error_type": "connection", "timeout": false})
require.False(t, m.Result(ok))
})
t.Run("connection and connect kinds", func(t *testing.T) {
for _, kind := range []string{"connection", "connect"} {
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}, Errors: []string{kind}}
require.NoError(t, m.CompileMatchers())
ok, _ := m.MatchError(map[string]interface{}{"error": "connection refused", "error_type": "connection", "timeout": false})
require.True(t, m.Result(ok), "kind %q", kind)
ok, _ = m.MatchError(map[string]interface{}{"error": "i/o timeout", "error_type": "timeout", "timeout": true})
require.False(t, m.Result(ok), "kind %q", kind)
}
})
t.Run("any and star kinds", func(t *testing.T) {
for _, kind := range []string{"any", "*"} {
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}, Errors: []string{kind}}
require.NoError(t, m.CompileMatchers())
ok, snippets := m.MatchError(map[string]interface{}{"error": "something went wrong", "error_type": "unknown", "timeout": false})
require.True(t, m.Result(ok), "kind %q", kind)
require.Equal(t, []string{"something went wrong"}, snippets)
}
})
t.Run("substring in error message", func(t *testing.T) {
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}, Errors: []string{"connection refused"}}
require.NoError(t, m.CompileMatchers())
ok, snippets := m.MatchError(map[string]interface{}{"error": "dial tcp 127.0.0.1:1: connection refused", "error_type": "connection", "timeout": false})
require.True(t, m.Result(ok))
require.Equal(t, []string{"connection refused"}, snippets)
ok, _ = m.MatchError(map[string]interface{}{"error": "i/o timeout", "error_type": "timeout", "timeout": true})
require.False(t, m.Result(ok))
})
t.Run("case insensitive substring", func(t *testing.T) {
m := &Matcher{
Type: MatcherTypeHolder{MatcherType: ErrorMatcher},
Errors: []string{"No Such Host"},
CaseInsensitive: true,
}
require.NoError(t, m.CompileMatchers())
ok, snippets := m.MatchError(map[string]interface{}{"error": "lookup example.invalid: no such host", "error_type": "connection", "timeout": false})
require.True(t, m.Result(ok))
require.Equal(t, []string{"No Such Host"}, snippets)
})
t.Run("case sensitive substring misses", func(t *testing.T) {
m := &Matcher{
Type: MatcherTypeHolder{MatcherType: ErrorMatcher},
Errors: []string{"No Such Host"},
}
require.NoError(t, m.CompileMatchers())
ok, _ := m.MatchError(map[string]interface{}{"error": "lookup example.invalid: no such host", "error_type": "connection", "timeout": false})
require.False(t, m.Result(ok))
})
t.Run("or condition across kind and substring", func(t *testing.T) {
m := &Matcher{
Type: MatcherTypeHolder{MatcherType: ErrorMatcher},
Errors: []string{"timeout", "no such host"},
Condition: "or",
}
require.NoError(t, m.CompileMatchers())
ok, _ := m.MatchError(map[string]interface{}{"error": "lookup x: no such host", "error_type": "connection", "timeout": false})
require.True(t, m.Result(ok))
ok, _ = m.MatchError(map[string]interface{}{"error": "i/o timeout", "error_type": "timeout", "timeout": true})
require.True(t, m.Result(ok))
ok, _ = m.MatchError(map[string]interface{}{"error": "broken pipe", "error_type": "connection", "timeout": false})
require.False(t, m.Result(ok))
})
t.Run("and condition requires all", func(t *testing.T) {
m := &Matcher{
Type: MatcherTypeHolder{MatcherType: ErrorMatcher},
Errors: []string{"dial tcp", "connection refused"},
Condition: "and",
}
require.NoError(t, m.CompileMatchers())
ok, snippets := m.MatchError(map[string]interface{}{"error": "dial tcp 127.0.0.1:1: connection refused", "error_type": "connection", "timeout": false})
require.True(t, m.Result(ok))
require.Equal(t, []string{"dial tcp", "connection refused"}, snippets)
ok, _ = m.MatchError(map[string]interface{}{"error": "dial tcp: i/o timeout", "error_type": "timeout", "timeout": true})
require.False(t, m.Result(ok))
})
t.Run("match-all returns every matching snippet", func(t *testing.T) {
m := &Matcher{
Type: MatcherTypeHolder{MatcherType: ErrorMatcher},
Errors: []string{"dial tcp", "connection refused", "missing"},
Condition: "or",
MatchAll: true,
}
require.NoError(t, m.CompileMatchers())
ok, snippets := m.MatchError(map[string]interface{}{"error": "dial tcp 127.0.0.1:1: connection refused", "error_type": "connection", "timeout": false})
require.True(t, m.Result(ok))
require.Equal(t, []string{"dial tcp", "connection refused"}, snippets)
})
t.Run("negative", func(t *testing.T) {
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}, Errors: []string{"timeout"}, Negative: true}
require.NoError(t, m.CompileMatchers())
ok, snippets := m.MatchError(map[string]interface{}{"error": "i/o timeout", "error_type": "timeout", "timeout": true})
matched, out := m.ResultWithMatchedSnippet(ok, snippets)
require.False(t, matched)
require.Empty(t, out)
ok, _ = m.MatchError(map[string]interface{}{})
require.True(t, m.Result(ok))
})
}
func TestHasErrorMatchersViaOperatorsPackage(t *testing.T) {
m := &Matcher{Type: MatcherTypeHolder{MatcherType: ErrorMatcher}, Errors: []string{"timeout"}}
require.NoError(t, m.CompileMatchers())
require.Equal(t, ErrorMatcher, m.GetType())
require.False(t, m.NeedsPart())
}
+9 -1
View File
@@ -13,7 +13,7 @@ const (
)
var (
fuzzMatcherTypes = []MatcherType{WordsMatcher, RegexMatcher, BinaryMatcher, StatusMatcher, SizeMatcher, DSLMatcher, XPathMatcher}
fuzzMatcherTypes = []MatcherType{WordsMatcher, RegexMatcher, BinaryMatcher, StatusMatcher, SizeMatcher, DSLMatcher, XPathMatcher, ErrorMatcher}
fuzzConditions = []string{"", "and", "or"}
fuzzEncodings = []string{"", "hex"}
fuzzParts = []string{"", "body", "raw", "all_headers", "header", "response"}
@@ -193,10 +193,18 @@ func (candidate *fuzzMatcherCandidate) build() (*Matcher, bool) {
matcher.Status = append([]int(nil), candidate.status...)
case SizeMatcher:
matcher.Size = append([]int(nil), candidate.size...)
case ErrorMatcher:
matcher.Part = ""
matcher.Encoding = ""
matcher.CaseInsensitive = false
matcher.Errors = append([]string(nil), candidate.values...)
default:
matcher.Words = append([]string(nil), candidate.values...)
}
if matcher.GetType() == ErrorMatcher {
return matcher, true
}
if matcher.GetType() == StatusMatcher || matcher.GetType() == SizeMatcher {
return matcher, len(matcher.Status) > 0 || len(matcher.Size) > 0
}
+87
View File
@@ -261,6 +261,93 @@ func (matcher *Matcher) MatchDSLWithOptions(data map[string]interface{}, options
return false
}
// MatchError matches request execution errors exposed on the event map.
//
// Errors values may be:
// - kinds: timeout, connection, connect, any, * (classify via error_type / timeout)
// - any other string: substring match against the error message
//
// An empty Errors list matches any request error. Condition (and/or), MatchAll,
// and CaseInsensitive apply the same way as word matchers.
func (matcher *Matcher) MatchError(data map[string]interface{}) (bool, []string) {
errVal, ok := data["error"]
if !ok || errVal == nil {
return false, []string{}
}
errStr, _ := errVal.(string)
if errStr == "" {
return false, []string{}
}
if len(matcher.Errors) == 0 {
return true, []string{errStr}
}
corpus := errStr
if matcher.CaseInsensitive {
corpus = strings.ToLower(corpus)
}
errType, _ := data["error_type"].(string)
isTimeout, _ := data["timeout"].(bool)
var matched []string
for i, item := range matcher.Errors {
item = strings.TrimSpace(item)
if item == "" {
if matcher.condition == ANDCondition {
return false, []string{}
}
continue
}
okMatch, snippet := matcher.matchErrorItem(item, errStr, corpus, errType, isTimeout)
if !okMatch {
if matcher.condition == ANDCondition {
return false, []string{}
}
continue
}
if matcher.condition == ORCondition && !matcher.MatchAll {
return true, []string{snippet}
}
matched = append(matched, snippet)
if len(matcher.Errors)-1 == i && !matcher.MatchAll {
return true, matched
}
}
if len(matched) > 0 && matcher.MatchAll {
return true, matched
}
return false, []string{}
}
func (matcher *Matcher) matchErrorItem(item, errStr, corpus, errType string, isTimeout bool) (bool, string) {
switch strings.ToLower(item) {
case "timeout":
if isTimeout || errType == "timeout" {
return true, item
}
case "connection", "connect":
if errType == "connection" {
return true, item
}
case "any", "*":
return true, errStr
default:
needle := item
if matcher.CaseInsensitive {
needle = strings.ToLower(item)
}
if strings.Contains(corpus, needle) {
return true, item
}
}
return false, ""
}
// MatchXPath matches on a generic map result
func (matcher *Matcher) MatchXPath(corpus string) bool {
if strings.HasPrefix(corpus, "<?xml") {
+15 -1
View File
@@ -11,7 +11,7 @@ import (
type Matcher struct {
// description: |
// Type is the type of the matcher.
Type MatcherTypeHolder `yaml:"type" json:"type" jsonschema:"title=type of matcher,description=Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl"`
Type MatcherTypeHolder `yaml:"type" json:"type" jsonschema:"title=type of matcher,description=Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl,enum=xpath,enum=error"`
// description: |
// Condition is the optional condition between two matcher variables. By default,
// the condition is assumed to be OR.
@@ -105,6 +105,20 @@ type Matcher struct {
// []string{"//a[@target=\"_blank\"]"}
XPath []string `yaml:"xpath,omitempty" json:"xpath,omitempty" jsonschema:"title=xpath queries to match in response,description=xpath are the XPath queries that will be evaluated against the response part of nuclei matching rules"`
// description: |
// Errors are patterns matched against a failed request (no successful response).
//
// Reserved kinds (matched via error classification): timeout, connection, connect, any, *.
// Any other value is treated as a substring of the error message.
// When empty, any request error matches.
// examples:
// - name: Match request timeouts by kind
// value: >
// []string{"timeout"}
// - name: Match a substring in the error message
// value: >
// []string{"connection refused"}
Errors []string `yaml:"errors,omitempty" json:"errors,omitempty" jsonschema:"title=error patterns to match,description=Error kinds (timeout, connection, connect, any, *) or substrings of the error message"`
// description: |
// Prompt is the natural-language question for an llm matcher. The model
// judges the selected response part and returns a verdict; the matcher
// fires when the verdict equals Expect with at least MinConfidence.
+3
View File
@@ -27,6 +27,8 @@ const (
DSLMatcher
// name:xpath
XPathMatcher
// name:error
ErrorMatcher
// name:llm
LLMMatcher
limit
@@ -41,6 +43,7 @@ var MatcherTypes = map[MatcherType]string{
BinaryMatcher: "binary",
DSLMatcher: "dsl",
XPathMatcher: "xpath",
ErrorMatcher: "error",
LLMMatcher: "llm",
}
+2
View File
@@ -57,6 +57,8 @@ func (matcher *Matcher) Validate() error {
case XPathMatcher:
requiredField, valueCount = "xpath", len(matcher.XPath)
expectedFields = append(commonExpectedFields, "XPath", "Part")
case ErrorMatcher:
expectedFields = append(commonExpectedFields, "Errors", "CaseInsensitive")
case LLMMatcher:
requiredField, valueCount = "prompt", len(matcher.Prompt)
expectedFields = append(commonExpectedFields, "Prompt", "Expect", "Options", "MinConfidence", "MaxInputTokens", "AllowSole", "Part", "Inputs")
+4
View File
@@ -34,6 +34,10 @@ func TestValidateRequiresValues(t *testing.T) {
for _, matcherType := range GetSupportedMatcherTypes() {
t.Run(matcherType.String(), func(t *testing.T) {
m := &Matcher{matcherType: matcherType}
if matcherType == ErrorMatcher {
require.NoError(t, m.Validate(), "an empty error matcher matches any request error")
return
}
require.ErrorContains(t, m.Validate(), "requires at least one")
})
}
+87
View File
@@ -81,6 +81,93 @@ func (operators *Operators) HasDSL() bool {
return false
}
// HasErrorMatchers reports whether operators include an error matcher or a DSL
// expression that references request-error fields (timeout / error / error_type).
// When true, protocols should still run matchers on failed requests and should
// not skip the template solely because the host is marked unresponsive.
func (operators *Operators) HasErrorMatchers() bool {
if operators == nil {
return false
}
for _, matcher := range operators.Matchers {
if matcher.GetType() == matchers.ErrorMatcher {
return true
}
if matcher.GetType() == matchers.DSLMatcher {
for _, expr := range matcher.DSL {
if dslReferencesRequestError(expr) {
return true
}
}
}
}
return false
}
func dslReferencesRequestError(expr string) bool {
for i := 0; i < len(expr); {
if expr[i] == '\'' || expr[i] == '"' || expr[i] == '`' {
quote := expr[i]
i++
for i < len(expr) {
if expr[i] == '\\' && quote != '`' {
i += 2
continue
}
if expr[i] == quote {
i++
break
}
i++
}
continue
}
if !isIdentByte(expr[i]) || (expr[i] >= '0' && expr[i] <= '9') {
i++
continue
}
start := i
for i < len(expr) && isIdentByte(expr[i]) {
i++
}
if isRequestErrorIdentifier(strings.ToLower(expr[start:i])) {
return true
}
}
return false
}
func isRequestErrorIdentifier(identifier string) bool {
for _, field := range []string{"timeout", "error_type", "error"} {
if identifier == field {
return true
}
prefix := field + "_"
if !strings.HasPrefix(identifier, prefix) {
continue
}
suffix := identifier[len(prefix):]
if suffix == "" {
continue
}
indexed := true
for i := 0; i < len(suffix); i++ {
if suffix[i] < '0' || suffix[i] > '9' {
indexed = false
break
}
}
if indexed {
return true
}
}
return false
}
func isIdentByte(b byte) bool {
return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_'
}
// GetMatchersCondition returns the condition for the matchers
func (operators *Operators) GetMatchersCondition() matchers.ConditionType {
return operators.matchersCondition
+3 -1
View File
@@ -329,7 +329,9 @@ func (w *StandardWriter) ResultCount() int {
// Write writes the event to file and/or screen.
func (w *StandardWriter) Write(event *ResultEvent) error {
if event.Error != "" && !w.matcherStatus {
// failure events (skipped or errored requests) are only written with -ms;
// a matched result can carry the request error, e.g. from an error matcher
if event.Error != "" && !event.MatcherStatus && !w.matcherStatus {
return nil
}
+15
View File
@@ -54,6 +54,21 @@ func TestStandardWriterRequest(t *testing.T) {
})
}
func TestStandardWriterWriteErroredEvents(t *testing.T) {
outputWriter := &testWriteCloser{}
w, err := NewStandardWriter(&types.Options{})
require.NoError(t, err)
w.outputFile = outputWriter
// a failure event (skipped or errored request) is only written with -ms
require.NoError(t, w.Write(&ResultEvent{TemplateID: "failed", Error: "connection refused"}))
require.Empty(t, outputWriter.String())
// a matched result can carry the request error, e.g. from an error matcher
require.NoError(t, w.Write(&ResultEvent{TemplateID: "matched", MatcherStatus: true, Error: "connection refused"}))
require.Contains(t, outputWriter.String(), "matched")
}
// Only the fields filled from a response are redacted; text the template wrote
// stays as it is, even next to a filled field.
func TestStandardWriterRedactsFilledInfo(t *testing.T) {
+4
View File
@@ -29,6 +29,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
contextutil "github.com/projectdiscovery/utils/context"
@@ -349,6 +350,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
if request.options.Interactsh != nil {
request.options.Interactsh.MakePlaceholders(interactshURLs, data)
}
if err != nil {
requesterr.Annotate(data, err, 0)
}
// todo #1: interactsh async callback should be eliminated as it lead to ton of code duplication
// todo #2: various structs InternalWrappedEvent, InternalEvent should be unwrapped and merged into minimal callbacks and a unique struct (eg. event?)
+3 -1
View File
@@ -18,6 +18,8 @@ func (request *Request) TmplClusterKey() uint64 {
}
// IsClusterable returns true if the request is eligible to be clustered.
// Error matchers are excluded: the cluster executer only shares responses, so
// a failed request would never reach them.
func (request *Request) IsClusterable() bool {
return len(request.Resolvers) <= 0 && !request.Trace && request.ID == ""
return len(request.Resolvers) <= 0 && !request.Trace && request.ID == "" && !request.HasErrorMatchers()
}
+3 -2
View File
@@ -21,8 +21,7 @@ import (
// Match matches a generic data response against a given matcher
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
item, ok := request.getMatchPart(matcher.Part, data)
// an llm matcher with inputs compares whole responses instead of one part
if !ok && matcher.Type.MatcherType != matchers.DSLMatcher && len(matcher.Inputs) == 0 {
if !ok && matcher.NeedsPart() {
return false, []string{}
}
@@ -45,6 +44,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
return matcher.Result(matcher.MatchDSLWithOptions(data, request.options.GetOptions())), []string{}
case matchers.XPathMatcher:
return matcher.Result(matcher.MatchXPath(types.ToString(item))), []string{}
case matchers.ErrorMatcher:
return matcher.ResultWithMatchedSnippet(matcher.MatchError(data))
case matchers.LLMMatcher:
return protocols.MatchLLM(data, matcher, types.ToString(item), request.options)
}
+18
View File
@@ -22,6 +22,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/retryabledns"
iputil "github.com/projectdiscovery/utils/ip"
@@ -186,6 +187,23 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad
request.options.Progress.IncrementRequests()
}
if response == nil {
if err != nil && request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers() {
outputEvent := output.InternalEvent{
"host": domain,
"matched": question,
"request": requestString,
"duration": duration.Seconds(),
"template-id": request.options.TemplateID,
"template-info": request.options.TemplateInfo,
"template-path": request.options.TemplatePath,
"type": request.Type().String(),
}
maps.Copy(outputEvent, previous)
maps.Copy(outputEvent, vars)
requesterr.Annotate(outputEvent, err, duration)
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
callback(event)
}
return errors.Wrap(err, "could not send dns request")
}
+3 -2
View File
@@ -15,8 +15,7 @@ import (
// Match matches a generic data response again a given matcher
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
itemStr, ok := request.getMatchPart(matcher.Part, data)
// an llm matcher with inputs compares whole responses instead of one part
if !ok && matcher.Type.MatcherType != matchers.DSLMatcher && len(matcher.Inputs) == 0 {
if !ok && matcher.NeedsPart() {
return false, []string{}
}
@@ -33,6 +32,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
return matcher.Result(matcher.MatchDSLWithOptions(data, request.options.GetOptions())), []string{}
case matchers.XPathMatcher:
return matcher.Result(matcher.MatchXPath(itemStr)), []string{}
case matchers.ErrorMatcher:
return matcher.ResultWithMatchedSnippet(matcher.MatchError(data))
case matchers.LLMMatcher:
return protocols.MatchLLM(data, matcher, itemStr, request.options)
}
+3 -2
View File
@@ -17,8 +17,7 @@ import (
// Match matches a generic data response again a given matcher
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
itemStr, ok := request.getMatchPart(matcher.Part, data)
// an llm matcher with inputs compares whole responses instead of one part
if !ok && matcher.Type.MatcherType != matchers.DSLMatcher && len(matcher.Inputs) == 0 {
if !ok && matcher.NeedsPart() {
return false, []string{}
}
@@ -41,6 +40,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
return matcher.Result(matcher.MatchDSLWithOptions(data, request.options.GetOptions())), []string{}
case matchers.XPathMatcher:
return matcher.Result(matcher.MatchXPath(itemStr)), []string{}
case matchers.ErrorMatcher:
return matcher.ResultWithMatchedSnippet(matcher.MatchError(data))
case matchers.LLMMatcher:
return protocols.MatchLLM(data, matcher, itemStr, request.options)
}
+27
View File
@@ -26,6 +26,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
urlutil "github.com/projectdiscovery/utils/url"
@@ -188,6 +189,32 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
if err != nil {
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
hasErrorMatchers := request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers()
if hasErrorMatchers || len(interactshURLs) > 0 {
outputEvent := request.responseToDSLMap("", "", "", "", input.MetaInput.Input, input.MetaInput.Input, "")
outputEvent["duration"] = runDuration.Seconds()
maps.Copy(outputEvent, payloads)
requesterr.Annotate(outputEvent, err, runDuration)
var event *output.InternalWrappedEvent
if hasErrorMatchers {
event = eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
} else {
event = &output.InternalWrappedEvent{InternalEvent: outputEvent}
}
if len(interactshURLs) > 0 && request.options.Interactsh != nil {
event.UsesInteractsh = true
request.options.RegisterInteractshRequest(interactshURLs, &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
Event: event,
Operators: request.CompiledOperators,
MatchFunc: request.Match,
ExtractFunc: request.Extract,
})
}
if hasErrorMatchers {
callback(event)
}
}
return errors.Wrap(err, errCouldNotGetHtmlElement)
}
page = createdPage
+3 -1
View File
@@ -16,7 +16,9 @@ func (request *Request) TmplClusterKey() uint64 {
}
// IsClusterable returns true if the request is eligible to be clustered.
// Error matchers are excluded: the cluster executer only shares responses, so
// a failed request would never reach them.
func (request *Request) IsClusterable() bool {
//nolint
return !(len(request.Payloads) > 0 || len(request.Fuzzing) > 0 || len(request.Raw) > 0 || len(request.Body) > 0 || request.Unsafe || request.NeedsRequestCondition() || request.Name != "")
return !(len(request.Payloads) > 0 || len(request.Fuzzing) > 0 || len(request.Raw) > 0 || len(request.Body) > 0 || request.Unsafe || request.NeedsRequestCondition() || request.Name != "" || request.HasErrorMatchers())
}
+6
View File
@@ -26,6 +26,12 @@ func TestIsClusterable(t *testing.T) {
require.True(t, req.IsClusterable())
})
t.Run("error matchers prevent clustering", func(t *testing.T) {
req := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}
req.Matchers = []*matchers.Matcher{{Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher}}}
require.False(t, req.IsClusterable())
})
t.Run("payloads prevent clustering", func(t *testing.T) {
req := &Request{
Path: []string{"{{BaseURL}}"},
+9 -2
View File
@@ -14,6 +14,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
)
@@ -21,8 +22,7 @@ import (
// TODO: Try to consolidate this in protocols.MakeDefaultMatchFunc to avoid any inconsistencies
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
item, ok := request.getMatchPart(matcher.Part, data)
// an llm matcher with inputs compares whole responses instead of one part
if !ok && matcher.Type.MatcherType != matchers.DSLMatcher && len(matcher.Inputs) == 0 {
if !ok && matcher.NeedsPart() {
return false, []string{}
}
@@ -49,6 +49,12 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
return matcher.Result(matcher.MatchDSLWithOptions(data, request.options.GetOptions())), []string{}
case matchers.XPathMatcher:
return matcher.Result(matcher.MatchXPath(item)), []string{}
case matchers.ErrorMatcher:
if !requesterr.IsMarked(data) {
// no request error: a response header named "error" is not one
return matcher.ResultWithMatchedSnippet(false, []string{})
}
return matcher.ResultWithMatchedSnippet(matcher.MatchError(data))
case matchers.LLMMatcher:
return protocols.MatchLLM(data, matcher, item, request.options)
}
@@ -136,6 +142,7 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw
request.setHashOrDefault(data, "all_headers", headers)
request.setHashOrDefault(data, "header", headers)
data["duration"] = duration.Seconds()
data["timeout"] = false
data["template-id"] = request.options.TemplateID
data["template-info"] = request.options.TemplateInfo
data["template-path"] = request.options.TemplatePath
+37 -4
View File
@@ -41,7 +41,7 @@ func TestResponseToDSLMap(t *testing.T) {
matched := "http://example.com/test/?test=1"
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
require.Len(t, event, 15, "could not get correct number of items in dsl map")
require.Len(t, event, 16, "could not get correct number of items in dsl map")
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
}
@@ -67,11 +67,13 @@ func TestHTTPOperatorMatch(t *testing.T) {
resp := &http.Response{}
resp.Header = make(http.Header)
resp.Header.Set("Test", "Test-Response")
resp.Header.Set("Error", "unavailable")
resp.Header.Set("Error-Type", "timeout")
host := "http://example.com/test/"
matched := "http://example.com/test/?test=1"
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
require.Len(t, event, 15, "could not get correct number of items in dsl map")
require.Len(t, event, 18, "could not get correct number of items in dsl map")
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
@@ -118,6 +120,37 @@ func TestHTTPOperatorMatch(t *testing.T) {
require.Equal(t, []string{}, matched)
})
t.Run("response error header is not a request error", func(t *testing.T) {
errorMatcher := &matchers.Matcher{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
Errors: []string{"any"},
}
require.NoError(t, errorMatcher.CompileMatchers())
isMatched, matched := request.Match(event, errorMatcher)
require.False(t, isMatched)
require.Empty(t, matched)
negativeMatcher := &matchers.Matcher{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
Errors: []string{"any"},
Negative: true,
}
require.NoError(t, negativeMatcher.CompileMatchers())
isMatched, _ = request.Match(event, negativeMatcher)
require.True(t, isMatched, "negative error matcher must match a response without request error")
headerMatcher := &matchers.Matcher{
Part: "error",
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"unavailable"},
}
require.NoError(t, headerMatcher.CompileMatchers())
isMatched, matched = request.Match(event, headerMatcher)
require.True(t, isMatched)
require.Equal(t, []string{"unavailable"}, matched)
})
t.Run("caseInsensitive", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "body",
@@ -159,7 +192,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
matched := "http://example.com/test/?test=1"
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
require.Len(t, event, 15, "could not get correct number of items in dsl map")
require.Len(t, event, 16, "could not get correct number of items in dsl map")
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
require.Equal(t, "Test-Response", event["test_header"], "could not get correct resp for header")
@@ -286,7 +319,7 @@ func TestHTTPMakeResult(t *testing.T) {
matched := "http://example.com/test/?test=1"
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
require.Len(t, event, 15, "could not get correct number of items in dsl map")
require.Len(t, event, 16, "could not get correct number of items in dsl map")
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
+38 -12
View File
@@ -44,6 +44,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httputils"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signer"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
@@ -1408,10 +1409,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
request.options.Output.Request(request.options.TemplatePath, formedURL, request.Type().String(), err)
request.options.Progress.IncrementErrorsBy(1)
// In case of interactsh markers and request times out, still send
// a callback event so in case we receive an interaction, correlation is possible.
// Also, to log failed use-cases.
outputEvent := request.responseToDSLMap(&http.Response{}, input.MetaInput.Input, formedURL, convUtil.String(dumpedRequest.data), "", "", "", 0, generatedRequest.meta)
duration := time.Since(timeStart)
outputEvent := request.responseToDSLMap(&http.Response{}, input.MetaInput.Input, formedURL, convUtil.String(dumpedRequest.data), "", "", "", duration, generatedRequest.meta)
requesterr.Annotate(outputEvent, err, duration)
if i := strings.LastIndex(hostname, ":"); i != -1 {
hostname = hostname[:i]
}
@@ -1421,14 +1421,35 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
// try getting cname
request.addCNameIfAvailable(hostname, outputEvent)
}
if len(generatedRequest.interactshURLs) > 0 {
// according to logic we only need to trigger a callback if interactsh was used
// and request failed in hope that later on oast interaction will be received
event := &output.InternalWrappedEvent{}
if request.CompiledOperators != nil && request.CompiledOperators.HasDSL() {
event.InternalEvent = outputEvent
if request.options.Interactsh != nil {
request.options.Interactsh.MakePlaceholders(generatedRequest.interactshURLs, outputEvent)
}
requestEvent := maps.Clone(outputEvent)
if previousEvent != nil {
outputEvent = maps.Clone(previousEvent)
maps.Copy(outputEvent, requestEvent)
}
if request.NeedsRequestCondition() {
for key, value := range requestEvent {
outputEvent[fmt.Sprintf("%s_%d", key, requestCount)] = value
}
}
// mark after the indexed copy so the provenance token gets no "_N" twin
requesterr.Mark(outputEvent)
hasErrorMatchers := request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers()
if hasErrorMatchers || len(generatedRequest.interactshURLs) > 0 {
var event *output.InternalWrappedEvent
if hasErrorMatchers {
event = eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
} else {
// Interactsh-only failure path: keep a lightweight event for later OAST correlation.
event = &output.InternalWrappedEvent{}
if request.CompiledOperators != nil && request.CompiledOperators.HasDSL() {
event.InternalEvent = outputEvent
}
}
requesterr.Unmark(outputEvent)
callback(event)
}
return err
@@ -1941,8 +1962,13 @@ func (request *Request) recordHostResultAndCancelIfUnresponsive(input *contextar
}
}
// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history
// isUnresponsiveAddress checks if the host is marked unresponsive based on
// execution history. Templates with error/timeout matchers intentionally bypass
// this skip so they can still observe and match request failures.
func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool {
if request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers() {
return false
}
if request.options.HostErrorsCache != nil {
return request.options.HostErrorsCache.Check(request.options.ProtocolType.String(), input)
}
+198
View File
@@ -2120,3 +2120,201 @@ func TestHTTPToHTTPSUnrelated400IsNotRetried(t *testing.T) {
require.True(t, matched, "an ordinary 400 is still a response the matcher can see")
require.Equal(t, int32(0), tlsHits.Load())
}
func TestHTTPErrorMatcherOnConnectionFailure(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "http-error-matcher"
request := &Request{
ID: templateID,
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}/"},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
Errors: []string{"connection"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.NoError(t, err)
var matched bool
var event *output.InternalWrappedEvent
ctxArgs := contextargs.NewWithInput(context.Background(), "http://127.0.0.1:1")
previous := output.InternalEvent{
"status_code_1": 200,
"error": "stale",
"error_type": "timeout",
"timeout": true,
}
err = request.ExecuteWithResults(ctxArgs, nil, previous, func(e *output.InternalWrappedEvent) {
event = e
if e.OperatorsResult != nil && e.OperatorsResult.Matched {
matched = true
}
})
require.Error(t, err)
require.NotNil(t, event)
require.True(t, matched, "connection error should match type: error matcher")
require.Equal(t, 200, event.InternalEvent["status_code_1"])
require.NotEqual(t, "stale", event.InternalEvent["error"])
require.Equal(t, "connection", event.InternalEvent["error_type"])
require.Equal(t, false, event.InternalEvent["timeout"])
}
func TestHTTPErrorMatcherSubstring(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "http-error-substring"
request := &Request{
ID: templateID,
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}/"},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
Errors: []string{"refused"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
require.NoError(t, request.Compile(executerOpts))
var matched bool
ctxArgs := contextargs.NewWithInput(context.Background(), "http://127.0.0.1:1")
err := request.ExecuteWithResults(ctxArgs, nil, nil, func(e *output.InternalWrappedEvent) {
if e.OperatorsResult != nil && e.OperatorsResult.Matched {
matched = true
}
})
require.Error(t, err)
require.True(t, matched)
}
func TestHTTPErrorMatcherDSL(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "http-error-dsl"
request := &Request{
ID: templateID,
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}/"},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
DSL: []string{`error_type == "connection"`},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
require.NoError(t, request.Compile(executerOpts))
var matched bool
ctxArgs := contextargs.NewWithInput(context.Background(), "http://127.0.0.1:1")
err := request.ExecuteWithResults(ctxArgs, nil, nil, func(e *output.InternalWrappedEvent) {
if e.OperatorsResult != nil && e.OperatorsResult.Matched {
matched = true
}
})
require.Error(t, err)
require.True(t, matched, "DSL error_type matcher should fire on connection failure")
}
func TestHTTPErrorMatcherBypassesHostErrorsCache(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "http-error-host-cache"
request := &Request{
ID: templateID,
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}/"},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
Errors: []string{"connection"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
// Host is marked unresponsive; without error matchers the request would be skipped.
executerOpts.HostErrorsCache = &spyHostErrorsCache{skip: true}
require.NoError(t, request.Compile(executerOpts))
var matched bool
var callbacks int
ctxArgs := contextargs.NewWithInput(context.Background(), "http://127.0.0.1:1")
err := request.ExecuteWithResults(ctxArgs, nil, nil, func(e *output.InternalWrappedEvent) {
callbacks++
if e.OperatorsResult != nil && e.OperatorsResult.Matched {
matched = true
}
})
require.Error(t, err)
require.Equal(t, 1, callbacks, "error matcher should bypass host-errors skip")
require.True(t, matched)
}
func TestHTTPErrorMatcherTimeout(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.Timeout = 1
testutils.Init(options)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(3 * time.Second)
}))
defer ts.Close()
templateID := "http-error-timeout"
request := &Request{
ID: templateID,
Raw: []string{`@timeout: 1s
GET / HTTP/1.1
Host: {{Hostname}}
`},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
Errors: []string{"timeout"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
require.NoError(t, request.Compile(executerOpts))
var matched bool
var event *output.InternalWrappedEvent
ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
err := request.ExecuteWithResults(ctxArgs, nil, nil, func(e *output.InternalWrappedEvent) {
event = e
if e.OperatorsResult != nil && e.OperatorsResult.Matched {
matched = true
}
})
require.Error(t, err)
require.NotNil(t, event)
require.True(t, matched, "timeout error should match type: error matcher")
require.Equal(t, true, event.InternalEvent["timeout"])
require.Equal(t, "timeout", event.InternalEvent["error_type"])
}
+12 -1
View File
@@ -32,6 +32,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/utils/errkit"
@@ -721,6 +722,10 @@ func (request *Request) executeRequestWithPayloads(
if genErr != nil {
return genErr
}
if err != nil {
// Make classified failure fields available to later protocol steps.
requesterr.Annotate(data, err, 0)
}
// add and get values from templatectx
request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), data)
@@ -737,7 +742,13 @@ func (request *Request) executeRequestWithPayloads(
}
if _, ok := data["error"]; ok {
event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(data, payloadValues), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
eventData := generators.MergeMaps(data, payloadValues)
if err != nil {
// Template context is merged above and may contain stale namespaced
// values, so ensure the emitted event retains the current failure.
requesterr.Annotate(eventData, err, 0)
}
event := eventcreator.CreateEventWithAdditionalOptions(request, eventData, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
wrappedEvent.OperatorsResult.PayloadValues = payload
})
callback(event)
+36
View File
@@ -287,3 +287,39 @@ func TestPreConditionRequestsAreCounted(t *testing.T) {
})
}
}
func TestExecuteFailureAddsClassifiedFieldsToTemplateContext(t *testing.T) {
options := testutils.DefaultOptions.Copy()
testutils.Init(options)
t.Cleanup(func() {
testutils.Cleanup(options)
})
executorOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "javascript-classified-failure"})
executorOptions.JsCompiler = templates.GetJsCompiler()
executorOptions.Verified = true
executorOptions.Flow = "flow()"
request := &javascript.Request{
ID: "js",
Code: `throw new Error("classified failure")`,
}
require.NoError(t, request.Compile(executorOptions))
target := contextargs.NewWithInput(context.Background(), "127.0.0.1:1")
var event *output.InternalWrappedEvent
err := request.ExecuteWithResults(target, nil, nil, func(got *output.InternalWrappedEvent) {
event = got
})
require.ErrorContains(t, err, "classified failure")
require.NotNil(t, event)
require.Contains(t, event.InternalEvent["error"], "classified failure")
require.Equal(t, "unknown", event.InternalEvent["error_type"])
require.Equal(t, false, event.InternalEvent["timeout"])
templateCtx := executorOptions.GetTemplateCtx(target.MetaInput).GetAll()
require.Contains(t, templateCtx["js_error"], "classified failure")
require.Equal(t, "unknown", templateCtx["js_error_type"])
require.Equal(t, false, templateCtx["js_timeout"])
}
+3 -2
View File
@@ -16,8 +16,7 @@ import (
// Match matches a generic data response again a given matcher
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
itemStr, ok := request.getMatchPart(matcher.Part, data)
// an llm matcher with inputs compares whole responses instead of one part
if !ok && matcher.Type.MatcherType != matchers.DSLMatcher && len(matcher.Inputs) == 0 {
if !ok && matcher.NeedsPart() {
return false, []string{}
}
@@ -34,6 +33,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
return matcher.Result(matcher.MatchDSLWithOptions(data, request.options.GetOptions())), []string{}
case matchers.XPathMatcher:
return matcher.Result(matcher.MatchXPath(itemStr)), []string{}
case matchers.ErrorMatcher:
return matcher.ResultWithMatchedSnippet(matcher.MatchError(data))
case matchers.LLMMatcher:
return protocols.MatchLLM(data, matcher, itemStr, request.options)
}
+53 -1
View File
@@ -32,6 +32,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/utils/errkit"
mapsutil "github.com/projectdiscovery/utils/maps"
@@ -360,6 +361,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
if err != nil {
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
request.emitErrorEvent(callback, err, address, actualAddress, payloads, previous, hostname, interactshURLs)
return errors.Wrap(err, "could not connect to server")
}
defer func() {
@@ -418,12 +420,14 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
if _, err := conn.Write(dataInBytes); err != nil {
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
request.emitErrorEvent(callback, err, address, actualAddress, payloads, previous, hostname, interactshURLs)
return errors.Wrap(err, "could not write request to server")
}
if input.Read > 0 {
buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), request.options.Options.GetTimeouts().TcpReadTimeout)
if err != nil {
request.emitErrorEvent(callback, err, address, actualAddress, payloads, previous, hostname, interactshURLs)
return errkit.Wrap(err, "could not read response from connection")
}
stepDurations = append(stepDurations, time.Since(timeStart))
@@ -502,6 +506,9 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
if request.options.Interactsh != nil {
request.options.Interactsh.MakePlaceholders(interactshURLs, outputEvent)
}
if readErr != nil {
requesterr.Annotate(outputEvent, readErr, 0)
}
var event *output.InternalWrappedEvent
if len(interactshURLs) == 0 {
@@ -619,6 +626,47 @@ func ConnReadNWithTimeout(conn net.Conn, n int64, timeout time.Duration) ([]byte
return b[:count], nil
}
// emitErrorEvent creates a matcher event for a failed network I/O operation when
// the template has error/timeout matchers.
func (request *Request) emitErrorEvent(callback protocols.OutputEventCallback, err error, address, actualAddress string, payloads map[string]interface{}, previous output.InternalEvent, hostname string, interactshURLs []string) {
hasErrorMatchers := request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers()
if !hasErrorMatchers && len(interactshURLs) == 0 {
return
}
outputEvent := request.responseToDSLMap("", "", "", address, actualAddress)
maps.Copy(outputEvent, previous)
maps.Copy(outputEvent, payloads)
if hostname != "" {
outputEvent["ip"] = request.dialer.GetDialedIP(hostname)
}
if request.options.Interactsh != nil {
request.options.Interactsh.MakePlaceholders(interactshURLs, outputEvent)
}
requesterr.Annotate(outputEvent, err, 0)
var event *output.InternalWrappedEvent
if hasErrorMatchers {
event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
wrappedEvent.OperatorsResult.PayloadValues = payloads
})
} else {
event = &output.InternalWrappedEvent{InternalEvent: outputEvent}
}
if len(interactshURLs) > 0 && request.options.Interactsh != nil {
event.UsesInteractsh = true
request.options.RegisterInteractshRequest(interactshURLs, &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
Event: event,
Operators: request.CompiledOperators,
MatchFunc: request.Match,
ExtractFunc: request.Extract,
})
}
if hasErrorMatchers {
callback(event)
}
}
// markHostError checks if the error is a unreponsive host error and marks it
func (request *Request) markHostError(input *contextargs.Context, err error) {
if request.options.HostErrorsCache != nil {
@@ -626,8 +674,12 @@ func (request *Request) markHostError(input *contextargs.Context, err error) {
}
}
// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history
// isUnresponsiveAddress checks if the host is marked unresponsive based on
// execution history. Templates with error/timeout matchers bypass this skip.
func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool {
if request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers() {
return false
}
if request.options.HostErrorsCache != nil {
return request.options.HostErrorsCache.Check(request.options.ProtocolType.String(), input)
}
+46
View File
@@ -4,6 +4,7 @@ import (
"context"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"io"
"net"
@@ -276,6 +277,51 @@ func TestNetworkCompileDefersInteractshMarkerHelpersToRuntime(t *testing.T) {
}
}
func TestEmitErrorEventPreservesHistoryAndAnnotations(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
request := &Request{
ID: "network-error-history",
Address: []string{"{{Hostname}}"},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
Errors: []string{"connection"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
require.NoError(t, request.Compile(executerOpts))
previous := output.InternalEvent{
"banner_1": "hello",
"error": "stale",
"error_type": "timeout",
"timeout": true,
}
payloads := map[string]interface{}{
"error": "payload",
"error_type": "unknown",
"timeout": true,
}
var event *output.InternalWrappedEvent
request.emitErrorEvent(func(got *output.InternalWrappedEvent) {
event = got
}, errors.New("dial tcp 127.0.0.1:1: connection refused"), "127.0.0.1:1", "127.0.0.1:1", payloads, previous, "", nil)
require.NotNil(t, event)
require.Equal(t, "hello", event.InternalEvent["banner_1"])
require.Equal(t, "connection", event.InternalEvent["error_type"])
require.Equal(t, false, event.InternalEvent["timeout"])
require.Contains(t, event.InternalEvent["error"], "connection refused")
require.True(t, event.OperatorsResult.Matched)
require.Equal(t, payloads, event.OperatorsResult.PayloadValues)
}
var exampleBody = `<!doctype html>
<html>
<head>
+8 -2
View File
@@ -14,14 +14,14 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
)
// Match matches a generic data response again a given matcher
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
item, ok := getMatchPart(matcher.Part, data)
// an llm matcher with inputs compares whole responses instead of one part
if !ok && matcher.Type.MatcherType != matchers.DSLMatcher && len(matcher.Inputs) == 0 {
if !ok && matcher.NeedsPart() {
return false, []string{}
}
@@ -44,6 +44,12 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
return matcher.Result(matcher.MatchDSLWithOptions(data, request.options.GetOptions())), []string{}
case matchers.XPathMatcher:
return matcher.Result(matcher.MatchXPath(item)), []string{}
case matchers.ErrorMatcher:
if !requesterr.IsMarked(data) {
// no request error: a response header named "error" is not one
return matcher.ResultWithMatchedSnippet(false, []string{})
}
return matcher.ResultWithMatchedSnippet(matcher.MatchError(data))
case matchers.LLMMatcher:
return protocols.MatchLLM(data, matcher, item, request.options)
}
+14 -1
View File
@@ -7,13 +7,13 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
)
func TestResponseToDSLMap(t *testing.T) {
@@ -109,6 +109,19 @@ func TestHTTPOperatorMatch(t *testing.T) {
require.False(t, isMatched, "could match invalid response matcher")
require.Equal(t, []string{}, matched)
})
t.Run("error header is not request failure", func(t *testing.T) {
event["error"] = "server-controlled"
matcher := &matchers.Matcher{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.ErrorMatcher},
Errors: []string{"any"},
}
require.NoError(t, matcher.CompileMatchers())
isMatched, matched := request.Match(event, matcher)
require.False(t, isMatched)
require.Empty(t, matched)
})
}
func TestHTTPOperatorExtract(t *testing.T) {
+3 -1
View File
@@ -627,7 +627,7 @@ func MakeDefaultMatchFuncWithOptions(data map[string]interface{}, matcher *match
}
partItem, ok := data[part]
if !ok && matcher.Type.MatcherType != matchers.DSLMatcher {
if !ok && matcher.NeedsPart() {
return false, nil
}
item := types.ToString(partItem)
@@ -646,6 +646,8 @@ func MakeDefaultMatchFuncWithOptions(data map[string]interface{}, matcher *match
return matcher.Result(matcher.MatchDSLWithOptions(data, options)), nil
case matchers.XPathMatcher:
return matcher.Result(matcher.MatchXPath(item)), []string{}
case matchers.ErrorMatcher:
return matcher.ResultWithMatchedSnippet(matcher.MatchError(data))
}
return false, nil
}
+22 -1
View File
@@ -27,6 +27,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
@@ -108,9 +109,12 @@ func (request *Request) TmplClusterKey() uint64 {
return xxhash.Sum64String(inp)
}
// IsClusterable returns true if the request is eligible to be clustered.
// Error matchers are excluded: the cluster executer only shares responses, so
// a failed request would never reach them.
func (request *Request) IsClusterable() bool {
// nolint
return !(len(request.CipherSuites) > 0 || request.MinVersion != "" || request.MaxVersion != "")
return !(len(request.CipherSuites) > 0 || request.MinVersion != "" || request.MaxVersion != "" || request.HasErrorMatchers())
}
// Compile compiles the request generators preparing any requests possible.
@@ -272,6 +276,23 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
if err != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
if request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers() {
data := make(map[string]interface{})
maps.Copy(data, payloadValues)
data["type"] = request.Type().String()
data["host"] = input.MetaInput.Input
data["matched"] = addressToDial
data["duration"] = duration.Seconds()
data["template-path"] = requestOptions.TemplatePath
data["template-id"] = requestOptions.TemplateID
data["template-info"] = requestOptions.TemplateInfo
if input.MetaInput.CustomIP != "" {
data["ip"] = hostIp
}
requesterr.Annotate(data, err, duration)
event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse)
callback(event)
}
return errkit.Wrap(err, "could not connect to server")
}
@@ -0,0 +1,113 @@
package requesterr
import (
"context"
"errors"
"io"
"net"
"os"
"strings"
"syscall"
"time"
)
// Kind classifies a request execution error for matcher / DSL use.
type Kind string
const (
KindTimeout Kind = "timeout"
KindConnection Kind = "connection"
KindUnknown Kind = "unknown"
)
const provenanceKey = "\x00nuclei-request-error"
var provenanceToken = &struct{ internal byte }{}
// Mark adds internal provenance that the error fields came from request execution.
// The token uses pointer identity so response-controlled fields cannot forge it.
func Mark(event map[string]interface{}) {
if event != nil {
event[provenanceKey] = provenanceToken
}
}
// IsMarked reports whether an event has request-error provenance.
func IsMarked(event map[string]interface{}) bool {
if event == nil {
return false
}
return event[provenanceKey] == provenanceToken
}
// Unmark removes internal request-error provenance before an event is emitted.
func Unmark(event map[string]interface{}) {
delete(event, provenanceKey)
}
// Classify returns the error kind and whether it is a timeout.
func Classify(err error) (Kind, bool) {
if err == nil {
return "", false
}
if isTimeoutErr(err) {
return KindTimeout, true
}
if isConnectionErr(err) {
return KindConnection, false
}
return KindUnknown, false
}
// Annotate adds error, error_type, timeout, and optional duration fields to an event map.
func Annotate(event map[string]interface{}, err error, duration time.Duration) {
if event == nil || err == nil {
return
}
kind, isTimeout := Classify(err)
event["error"] = err.Error()
event["error_type"] = string(kind)
event["timeout"] = isTimeout
if duration > 0 {
event["duration"] = duration.Seconds()
}
}
func isTimeoutErr(err error) bool {
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, os.ErrDeadlineExceeded) {
return true
}
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
return true
}
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "timeout") ||
strings.Contains(msg, "deadline exceeded") ||
strings.Contains(msg, "i/o timeout") ||
strings.Contains(msg, "client.timeout")
}
func isConnectionErr(err error) bool {
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
return true
}
if errors.Is(err, syscall.ECONNREFUSED) ||
errors.Is(err, syscall.ECONNRESET) ||
errors.Is(err, syscall.EPIPE) ||
errors.Is(err, syscall.ENETUNREACH) ||
errors.Is(err, syscall.EHOSTUNREACH) {
return true
}
var opErr *net.OpError
if errors.As(err, &opErr) {
return true
}
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "connection refused") ||
strings.Contains(msg, "actively refused") ||
strings.Contains(msg, "connection reset") ||
strings.Contains(msg, "no such host") ||
strings.Contains(msg, "network is unreachable") ||
strings.Contains(msg, "broken pipe")
}
@@ -0,0 +1,94 @@
package requesterr
import (
"context"
"errors"
"io"
"net"
"os"
"syscall"
"testing"
"time"
)
func TestClassifyNil(t *testing.T) {
kind, isTimeout := Classify(nil)
if kind != "" || isTimeout {
t.Fatalf("expected empty kind for nil, got kind=%s isTimeout=%v", kind, isTimeout)
}
}
func TestClassifyTimeout(t *testing.T) {
cases := []error{
context.DeadlineExceeded,
os.ErrDeadlineExceeded,
errors.New("Client.Timeout exceeded while awaiting headers"),
errors.New("i/o timeout"),
errors.New("context deadline exceeded"),
}
for _, err := range cases {
kind, isTimeout := Classify(err)
if kind != KindTimeout || !isTimeout {
t.Fatalf("expected timeout for %v, got kind=%s isTimeout=%v", err, kind, isTimeout)
}
}
}
func TestClassifyConnection(t *testing.T) {
cases := []error{
syscall.ECONNREFUSED,
syscall.ECONNRESET,
io.EOF,
&net.OpError{Op: "dial", Err: errors.New("connect: connection refused")},
errors.New("connection refused"),
errors.New("connectex: No connection could be made because the target machine actively refused it"),
errors.New("no such host"),
errors.New("network is unreachable"),
}
for _, err := range cases {
kind, isTimeout := Classify(err)
if kind != KindConnection || isTimeout {
t.Fatalf("expected connection for %v, got kind=%s isTimeout=%v", err, kind, isTimeout)
}
}
}
func TestClassifyUnknown(t *testing.T) {
kind, isTimeout := Classify(errors.New("something else entirely"))
if kind != KindUnknown || isTimeout {
t.Fatalf("expected unknown, got kind=%s isTimeout=%v", kind, isTimeout)
}
}
func TestAnnotate(t *testing.T) {
event := map[string]interface{}{}
Annotate(event, context.DeadlineExceeded, 1500*time.Millisecond)
if event["error"] == nil || event["error_type"] != string(KindTimeout) || event["timeout"] != true {
t.Fatalf("unexpected annotate result: %#v", event)
}
if event["duration"].(float64) != 1.5 {
t.Fatalf("unexpected duration: %v", event["duration"])
}
Annotate(nil, context.DeadlineExceeded, time.Second) // no panic
Annotate(map[string]interface{}{}, nil, time.Second) // no fields
}
func TestProvenanceMarker(t *testing.T) {
event := map[string]interface{}{
"error": "server controlled",
"error_type": "timeout",
}
if IsMarked(event) {
t.Fatal("public error fields must not establish request-error provenance")
}
Mark(event)
if !IsMarked(event) {
t.Fatal("expected internally marked event")
}
Unmark(event)
if IsMarked(event) {
t.Fatal("expected provenance to be removed")
}
}
+14
View File
@@ -32,6 +32,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
urlutil "github.com/projectdiscovery/utils/url"
@@ -285,6 +286,19 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
if err != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
if request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers() {
data := make(map[string]interface{})
data["type"] = request.Type().String()
data["host"] = input
data["matched"] = addressToDial
data["duration"] = handshakeDuration.Seconds()
data["ip"] = request.dialer.GetDialedIP(hostname)
maps.Copy(data, previous)
maps.Copy(data, payloadValues)
requesterr.Annotate(data, err, handshakeDuration)
event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse)
callback(event)
}
return errors.Wrap(err, "could not connect to server")
}
+10
View File
@@ -20,6 +20,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/requesterr"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/whois/rdapclientpool"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
@@ -132,6 +133,15 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
res, err := request.client.Do(rdapReq)
duration := time.Since(timeStart)
if err != nil {
if request.CompiledOperators != nil && request.CompiledOperators.HasErrorMatchers() {
data := make(map[string]interface{})
data["type"] = request.Type().String()
data["host"] = query
data["duration"] = duration.Seconds()
requesterr.Annotate(data, err, duration)
event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse)
callback(event)
}
return errors.Wrap(err, "could not make whois request")
}
gologger.Verbose().Msgf("Sent WHOIS request to %s", query)