mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2026-11-07 17:43:04 -10:00
refactor(protocols): template rendering boundary (#7499)
* refactor(protocols): template rendering boundary
Stop treating every marker-shaped string as
template source after it has entered runtime data.
Template-authored text is rendered once, while
values from payload results, previous responses,
overrides, constants, and rendered outputs stay as
data.
Route protocol payload rendering through a shared
render boundary. Authored Interactsh markers still
allocate OAST URLs, including when helper-
transformed, but runtime marker-shaped values stay
literal and do not allocate stale URLs or trigger
another DSL pass.
Keep the same boundary inside helper expressions.
Placeholders embedded in authored govaluate string
literals are escaped before compilation, so a
value used by `{{contains('{{body}}', 'x')}}`
cannot become new expression syntax.
Track fuzz value provenance explicitly instead of
inferring it from merged map values, and share
govaluate string escaping with the matcher
fallback.
Signed-off-by: Dwi Siswanto <[email protected]>
* Update pkg/protocols/websocket/websocket.go
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
* refactor(variables): preserve ownership across eval
Carry variable inputs thru an owner-aware scope
instead of merging template text and runtime data
into the same plain map before eval.
This keeps declared template variables renderable
when later values satisfy their dependencies,
while keeping runtime, extracted, previous, option,
constant, and generated values as terminal data.
A dynamic value can still shadow a declared
variable (though), but the selected value is not
eval'ed again and does not allocate Interactsh
URLs.
Store template-owned values separately in template
context so flow and multiprotocol execution can
reuse declared variable text w/o turning
extracted data into template source.
Migrate fuzz, HTTP, JavaScript, network, headless,
code, DNS, SSL, websocket, whois, flow, and
multiprotocol paths to the owner-aware evaluation
path, and add regressions for equal-value
overrides, lazy declared variables, Interactsh
allocation, and context ownership.
Signed-off-by: Dwi Siswanto <[email protected]>
* fix(protocols): keep interactsh render state explicit
Do not let Interactsh allocation failures
disappear behind a successful render. Return the
partial render state with the error so callers can
keep URLs that were already allocated, and fix the
encoded marker regexp so `|` is not treated as a
brace digit.
Preserve caller-provided Interactsh URLs in fuzz
execution, propagate payload render errors, and
avoid persisting network precompile output when
data introduces a marker that runtime would later
treat as source.
Also keep previous/template-context values in SSL,
WebSocket, and WHOIS render scopes, and make
multiproto compile retain constants and CLI vars
as read-only data without marking them as
template-owned values.
Signed-off-by: Dwi Siswanto <[email protected]>
* refactor(render): centralize interactsh marker scanning
Move default interactsh marker detection into the
common marker pkg and use a small scanner instead
of duplicated regexes in render and interactsh.
Raw and fully URL-encoded markers still work, but
mixed raw/encoded brace pairs and bad numeric
suffixes are rejected.
Keep partially rendered map values when `RenderMap`
stops on an error.
Signed-off-by: Dwi Siswanto <[email protected]>
---------
Signed-off-by: Dwi Siswanto <[email protected]>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
1e294aa7b9
commit
ccbfb12bd0
45 files changed
+3342
-354
No files matched your search
+160
-37
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/marker"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
@@ -149,14 +150,27 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) {
|
||||
}
|
||||
|
||||
baseValues := input.Values
|
||||
baseInteractURLs := append([]string(nil), input.InteractURLs...)
|
||||
if rule.generator == nil {
|
||||
for _, component := range finalComponentList {
|
||||
var dataKeys map[string]struct{}
|
||||
|
||||
optionVars := rule.options.Options.Vars.AsMap()
|
||||
constants := rule.options.Constants
|
||||
dataValues := rule.options.NewVariablesScope(baseValues, optionVars, constants)
|
||||
// get vars from variables while replacing interactsh urls
|
||||
evaluatedValues, interactURLs := rule.options.Variables.EvaluateWithInteractsh(baseValues, rule.options.Interactsh)
|
||||
input.Values = generators.MergeMaps(evaluatedValues, baseValues, rule.options.Options.Vars.AsMap(), rule.options.Constants)
|
||||
// evaluate all vars with interactsh
|
||||
input.Values, interactURLs = rule.evaluateVarsWithInteractsh(input.Values, interactURLs)
|
||||
input.InteractURLs = interactURLs
|
||||
evaluation := rule.options.Variables.EvaluateWithInteractshScope(dataValues, rule.options.Interactsh)
|
||||
evaluatedValues, interactURLs := evaluation.Values, evaluation.InteractURLs
|
||||
input.Values, dataKeys = mergeFuzzValueLayers(
|
||||
fuzzValueLayer{values: evaluatedValues, kind: fuzzValueTemplate},
|
||||
fuzzValueLayer{values: baseValues, kind: fuzzValueData},
|
||||
fuzzValueLayer{values: optionVars, kind: fuzzValueData},
|
||||
fuzzValueLayer{values: constants, kind: fuzzValueData},
|
||||
)
|
||||
// Replace template-text interactsh markers without evaluating runtime values.
|
||||
input.Values, interactURLs = rule.evaluateVarsWithInteractsh(input.Values, interactURLs, dataKeys)
|
||||
input.InteractURLs = mergeInteractURLs(baseInteractURLs, interactURLs)
|
||||
|
||||
err := rule.executeRuleValues(input, component)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -172,12 +186,9 @@ mainLoop:
|
||||
if !next {
|
||||
continue mainLoop
|
||||
}
|
||||
// get vars from variables while replacing interactsh urls
|
||||
evaluatedValues, interactURLs := rule.options.Variables.EvaluateWithInteractsh(generators.MergeMaps(values, baseValues), rule.options.Interactsh)
|
||||
input.Values = generators.MergeMaps(values, evaluatedValues, baseValues, rule.options.Options.Vars.AsMap(), rule.options.Constants)
|
||||
// evaluate all vars with interactsh
|
||||
input.Values, interactURLs = rule.evaluateVarsWithInteractsh(input.Values, interactURLs)
|
||||
input.InteractURLs = interactURLs
|
||||
var interactURLs []string
|
||||
input.Values, interactURLs = rule.prepareGeneratorValues(values, baseValues)
|
||||
input.InteractURLs = mergeInteractURLs(baseInteractURLs, interactURLs)
|
||||
|
||||
if err := rule.executeRuleValues(input, component); err != nil {
|
||||
if err == io.EOF {
|
||||
@@ -191,6 +202,93 @@ mainLoop:
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeInteractURLs(base, urls []string) []string {
|
||||
if len(base)+len(urls) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
merged := make([]string, 0, len(base)+len(urls))
|
||||
seen := make(map[string]struct{}, len(base)+len(urls))
|
||||
|
||||
for _, url := range base {
|
||||
if _, ok := seen[url]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
seen[url] = struct{}{}
|
||||
merged = append(merged, url)
|
||||
}
|
||||
|
||||
for _, url := range urls {
|
||||
if _, ok := seen[url]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
seen[url] = struct{}{}
|
||||
merged = append(merged, url)
|
||||
}
|
||||
|
||||
return merged
|
||||
}
|
||||
|
||||
func (rule *Rule) prepareGeneratorValues(values, baseValues map[string]interface{}) (map[string]interface{}, []string) {
|
||||
optionVars := rule.options.Options.Vars.AsMap()
|
||||
constants := rule.options.Constants
|
||||
evaluationValues := rule.options.NewVariablesScope(values, baseValues, optionVars, constants)
|
||||
evaluation := rule.options.Variables.EvaluateWithInteractshScope(evaluationValues, rule.options.Interactsh)
|
||||
evaluatedValues, interactURLs := evaluation.Values, evaluation.InteractURLs
|
||||
inputValues, dataKeys := mergeFuzzValueLayers(
|
||||
fuzzValueLayer{values: values, kind: fuzzValueTemplate},
|
||||
fuzzValueLayer{values: evaluatedValues, kind: fuzzValueTemplate},
|
||||
fuzzValueLayer{values: baseValues, kind: fuzzValueData},
|
||||
fuzzValueLayer{values: optionVars, kind: fuzzValueData},
|
||||
fuzzValueLayer{values: constants, kind: fuzzValueData},
|
||||
)
|
||||
|
||||
// Generator values are template-authored payload definitions until this
|
||||
// render. Runtime/base, CLI option, and constant values remain data.
|
||||
return rule.evaluateVarsWithInteractsh(inputValues, interactURLs, dataKeys)
|
||||
}
|
||||
|
||||
type fuzzValueKind uint8
|
||||
|
||||
const (
|
||||
fuzzValueTemplate fuzzValueKind = iota
|
||||
fuzzValueData
|
||||
)
|
||||
|
||||
type fuzzValueLayer struct {
|
||||
values map[string]interface{}
|
||||
kind fuzzValueKind
|
||||
}
|
||||
|
||||
func mergeFuzzValueLayers(layers ...fuzzValueLayer) (map[string]interface{}, map[string]struct{}) {
|
||||
size := 0
|
||||
for _, layer := range layers {
|
||||
size += len(layer.values)
|
||||
}
|
||||
|
||||
values := make(map[string]interface{}, size)
|
||||
dataKeys := make(map[string]struct{})
|
||||
|
||||
for _, layer := range layers {
|
||||
for key, value := range layer.values {
|
||||
values[key] = value
|
||||
if layer.kind == fuzzValueData {
|
||||
dataKeys[key] = struct{}{}
|
||||
continue
|
||||
}
|
||||
delete(dataKeys, key)
|
||||
}
|
||||
}
|
||||
|
||||
if len(dataKeys) == 0 {
|
||||
return values, nil
|
||||
}
|
||||
|
||||
return values, dataKeys
|
||||
}
|
||||
|
||||
// evaluateVars evaluates variables in a string using available executor options
|
||||
func (rule *Rule) evaluateVars(input string) (string, error) {
|
||||
if rule.options == nil {
|
||||
@@ -210,16 +308,19 @@ func (rule *Rule) evaluateVars(input string) (string, error) {
|
||||
return input, err
|
||||
}
|
||||
|
||||
eval, err := expressions.Evaluate(input, data)
|
||||
result, err := render.Render(render.Input{
|
||||
Text: input,
|
||||
Values: data,
|
||||
})
|
||||
if err != nil {
|
||||
return input, err
|
||||
}
|
||||
|
||||
return eval, nil
|
||||
return result.Text, nil
|
||||
}
|
||||
|
||||
// evaluateVarsWithInteractsh evaluates the variables with Interactsh URLs and updates them accordingly.
|
||||
func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, interactshUrls []string) (map[string]interface{}, []string) {
|
||||
// evaluateVarsWithInteractsh renders template-text values and leaves runtime data untouched.
|
||||
func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, interactshUrls []string, dataKeys map[string]struct{}) (map[string]interface{}, []string) {
|
||||
// Check if Interactsh options are configured
|
||||
if rule.options.Interactsh != nil {
|
||||
data = maps.Clone(data)
|
||||
@@ -228,35 +329,35 @@ func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, intera
|
||||
for _, url := range interactshUrls {
|
||||
interactshUrlsMap[url] = struct{}{}
|
||||
}
|
||||
|
||||
interactshUrls = mapsutil.GetKeys(interactshUrlsMap)
|
||||
// Iterate through the data to replace and evaluate variables with Interactsh URLs
|
||||
|
||||
// Iterate through template-text data to replace Interactsh URL markers.
|
||||
for k, v := range data {
|
||||
value := fmt.Sprint(v)
|
||||
// Replace variables with Interactsh URLs and collect new URLs
|
||||
got, oastUrls := rule.options.Interactsh.Replace(value, interactshUrls)
|
||||
if got != value {
|
||||
data[k] = got
|
||||
if _, ok := dataKeys[k]; ok {
|
||||
continue
|
||||
}
|
||||
// Append new OAST URLs if any
|
||||
if len(oastUrls) > 0 {
|
||||
for _, url := range oastUrls {
|
||||
if _, ok := interactshUrlsMap[url]; !ok {
|
||||
interactshUrlsMap[url] = struct{}{}
|
||||
interactshUrls = append(interactshUrls, url)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Evaluate the replaced data
|
||||
evaluatedData, err := expressions.Evaluate(got, data)
|
||||
|
||||
got, err := render.Render(render.Input{
|
||||
Text: fmt.Sprint(v),
|
||||
Values: data,
|
||||
Interactsh: rule.options.Interactsh,
|
||||
InteractURLs: interactshUrls,
|
||||
})
|
||||
if err == nil {
|
||||
// Update the data if there is a change after evaluation
|
||||
if evaluatedData != got {
|
||||
data[k] = evaluatedData
|
||||
data[k] = got.Text
|
||||
for _, url := range got.InteractURLs {
|
||||
if _, ok := interactshUrlsMap[url]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
interactshUrlsMap[url] = struct{}{}
|
||||
interactshUrls = append(interactshUrls, url)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Return the updated data and Interactsh URLs without any error
|
||||
|
||||
return data, interactshUrls
|
||||
}
|
||||
|
||||
@@ -282,15 +383,18 @@ func (rule *Rule) executeRuleValues(input *ExecuteRuleInput, ruleComponent compo
|
||||
// ex: fuzzing string value in a json int field
|
||||
continue
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// if we are fuzzing both keys and values
|
||||
if rule.Fuzz.KV != nil {
|
||||
var gotErr error
|
||||
|
||||
rule.Fuzz.KV.Iterate(func(key, value string) bool {
|
||||
if err := rule.executePartRule(input, ValueOrKeyValue{Key: key, Value: value}, ruleComponent); err != nil {
|
||||
if component.IsErrSetValue(err) {
|
||||
@@ -299,28 +403,47 @@ func (rule *Rule) executeRuleValues(input *ExecuteRuleInput, ruleComponent compo
|
||||
return true
|
||||
}
|
||||
gotErr = err
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
})
|
||||
|
||||
// if mode is multiple now build and execute it
|
||||
if rule.modeType == multipleModeType {
|
||||
rule.Fuzz.KV.Iterate(func(key, value string) bool {
|
||||
var evaluated string
|
||||
evaluated, input.InteractURLs = rule.executeEvaluate(input, key, "", value, input.InteractURLs)
|
||||
var err error
|
||||
|
||||
evaluated, input.InteractURLs, err = rule.executeEvaluate(input, key, "", value, input.InteractURLs)
|
||||
if err != nil {
|
||||
gotErr = err
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
if err := ruleComponent.SetValue(key, evaluated); err != nil {
|
||||
return true
|
||||
}
|
||||
|
||||
return true
|
||||
})
|
||||
|
||||
if gotErr != nil {
|
||||
return gotErr
|
||||
}
|
||||
|
||||
req, err := ruleComponent.Rebuild()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, "", "", "", "", "", ""); gotErr != nil {
|
||||
return gotErr
|
||||
}
|
||||
}
|
||||
|
||||
return gotErr
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestEvaluateVarsWithInteractsh_RaceCondition(t *testing.T) {
|
||||
@@ -25,8 +26,27 @@ func TestEvaluateVarsWithInteractsh_RaceCondition(t *testing.T) {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
rule.evaluateVarsWithInteractsh(sharedData, nil)
|
||||
rule.evaluateVarsWithInteractsh(sharedData, nil, nil)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestEvaluateVarsWithInteractshDoesNotEvaluateRuntimeData(t *testing.T) {
|
||||
rule := &Rule{}
|
||||
rule.options = &protocols.ExecutorOptions{
|
||||
Interactsh: &interactsh.Client{},
|
||||
}
|
||||
data := map[string]interface{}{
|
||||
"from_response": "{{secret}}",
|
||||
"secret": "leaked-secret",
|
||||
}
|
||||
|
||||
got, urls := rule.evaluateVarsWithInteractsh(data, nil, map[string]struct{}{
|
||||
"from_response": {},
|
||||
"secret": {},
|
||||
})
|
||||
|
||||
require.Empty(t, urls)
|
||||
require.Equal(t, "{{secret}}", got["from_response"])
|
||||
}
|
||||
+37
-12
@@ -6,8 +6,8 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/component"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
sliceutil "github.com/projectdiscovery/utils/slice"
|
||||
@@ -58,10 +58,19 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS
|
||||
}
|
||||
|
||||
var evaluated, originalEvaluated string
|
||||
evaluated, input.InteractURLs = rule.executeEvaluate(input, key, valueStr, payloadStr, input.InteractURLs)
|
||||
var err error
|
||||
|
||||
evaluated, input.InteractURLs, err = rule.executeEvaluate(input, key, valueStr, payloadStr, input.InteractURLs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if input.ApplyPayloadInitialTransformation != nil {
|
||||
evaluated = input.ApplyPayloadInitialTransformation(evaluated, input.AnalyzerParams)
|
||||
originalEvaluated, _ = rule.executeEvaluate(input, key, valueStr, originalPayload, input.InteractURLs)
|
||||
originalEvaluated, _, err = rule.executeEvaluate(input, key, valueStr, originalPayload, input.InteractURLs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := ruleComponent.SetValue(key, evaluated); err != nil {
|
||||
@@ -78,14 +87,16 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS
|
||||
if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key, valueStr, originalEvaluated, valueStr, key, evaluated); qerr != nil {
|
||||
return qerr
|
||||
}
|
||||
// fmt.Printf("executed with value: %s\n", evaluated)
|
||||
|
||||
err = ruleComponent.SetValue(key, valueStr) // change back to previous value for temp
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if finalErr != nil {
|
||||
return finalErr
|
||||
}
|
||||
@@ -97,11 +108,13 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, "", "", "", "", "", ""); qerr != nil {
|
||||
err = qerr
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -122,7 +135,11 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu
|
||||
// iterate over given kv instead of component ones
|
||||
return func(key, value string) error {
|
||||
var evaluated string
|
||||
evaluated, input.InteractURLs = rule.executeEvaluate(input, key, "", value, input.InteractURLs)
|
||||
var err error
|
||||
evaluated, input.InteractURLs, err = rule.executeEvaluate(input, key, "", value, input.InteractURLs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ruleComponent.SetValue(key, evaluated); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -191,16 +208,24 @@ func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.
|
||||
// executeEvaluate executes evaluation of payload on a key and value and
|
||||
// returns completed values to be replaced and processed
|
||||
// for fuzzing.
|
||||
func (rule *Rule) executeEvaluate(input *ExecuteRuleInput, _, value, payload string, interactshURLs []string) (string, []string) {
|
||||
// TODO: Handle errors
|
||||
func (rule *Rule) executeEvaluate(input *ExecuteRuleInput, _, value, payload string, interactshURLs []string) (string, []string, error) {
|
||||
values := generators.MergeMaps(rule.options.Variables.GetAll(), map[string]interface{}{
|
||||
"value": value,
|
||||
}, rule.options.Options.Vars.AsMap(), input.Values)
|
||||
firstpass, _ := expressions.Evaluate(payload, values)
|
||||
interactData, interactshURLs := rule.options.Interactsh.Replace(firstpass, interactshURLs)
|
||||
evaluated, _ := expressions.Evaluate(interactData, values)
|
||||
replaced := rule.executeRuleTypes(input, value, evaluated)
|
||||
return replaced, interactshURLs
|
||||
|
||||
result, err := render.Render(render.Input{
|
||||
Text: payload,
|
||||
Values: values,
|
||||
Interactsh: rule.options.Interactsh,
|
||||
InteractURLs: interactshURLs,
|
||||
})
|
||||
if err != nil {
|
||||
return payload, result.InteractURLs, err
|
||||
}
|
||||
|
||||
replaced := rule.executeRuleTypes(input, value, result.Text)
|
||||
|
||||
return replaced, result.InteractURLs, nil
|
||||
}
|
||||
|
||||
// executeRuleTypes executes replacement for a key and value
|
||||
|
||||
+301
-1
@@ -1,2 +1,302 @@
|
||||
// TODO: Write tests
|
||||
package fuzz
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newFuzzTestInteractshClient(t *testing.T) *interactsh.Client {
|
||||
t.Helper()
|
||||
|
||||
client, err := interactsh.New(&interactsh.Options{
|
||||
ServerURL: "oast.fun",
|
||||
CacheSize: 100,
|
||||
Eviction: 60 * time.Second,
|
||||
CooldownPeriod: 0,
|
||||
PollDuration: 5 * time.Second,
|
||||
DisableHttpFallback: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Cleanup(func() {
|
||||
client.Close()
|
||||
})
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
func newFuzzTestVariables() variables.Variable {
|
||||
return variables.Variable{
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(0),
|
||||
}
|
||||
}
|
||||
|
||||
func newFuzzGeneratorTestRule(t *testing.T, templateVars variables.Variable) *Rule {
|
||||
t.Helper()
|
||||
|
||||
return &Rule{
|
||||
ruleType: replaceRuleType,
|
||||
options: &protocols.ExecutorOptions{
|
||||
Variables: templateVars,
|
||||
Options: &types.Options{},
|
||||
Interactsh: newFuzzTestInteractshClient(t),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareGeneratorValuesRendersDirectPayloadInteractshMarker(t *testing.T) {
|
||||
rule := newFuzzGeneratorTestRule(t, newFuzzTestVariables())
|
||||
|
||||
values, urls := rule.prepareGeneratorValues(map[string]interface{}{
|
||||
"ssrf": "{{interactsh-url}}",
|
||||
}, nil)
|
||||
got, urls, err := rule.executeEvaluate(&ExecuteRuleInput{Values: values}, "", "original", "https://{{ssrf}}", urls)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, urls, 1)
|
||||
require.Equal(t, "https://"+urls[0], got)
|
||||
require.NotContains(t, got, "{{interactsh-url}}")
|
||||
}
|
||||
|
||||
func TestPrepareGeneratorValuesRendersHelperEncodedPayloadInteractshMarker(t *testing.T) {
|
||||
rule := newFuzzGeneratorTestRule(t, newFuzzTestVariables())
|
||||
|
||||
values, urls := rule.prepareGeneratorValues(map[string]interface{}{
|
||||
"ssrf": "{{url_encode('{{interactsh-url}}')}}",
|
||||
}, nil)
|
||||
got, urls, err := rule.executeEvaluate(&ExecuteRuleInput{Values: values}, "", "original", "{{ssrf}}", urls)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, urls, 1)
|
||||
require.Equal(t, urls[0], got)
|
||||
require.NotContains(t, got, "{{interactsh-url}}")
|
||||
require.NotContains(t, got, "%7B%7Binteractsh-url%7D%7D")
|
||||
}
|
||||
|
||||
func TestPrepareGeneratorValuesRendersVariableIndirectPayloadInteractshMarker(t *testing.T) {
|
||||
templateVars := newFuzzTestVariables()
|
||||
templateVars.Set("marker", "{{interactsh-url}}")
|
||||
rule := newFuzzGeneratorTestRule(t, templateVars)
|
||||
|
||||
values, urls := rule.prepareGeneratorValues(map[string]interface{}{
|
||||
"interaction": "|nslookup {{marker}}|curl {{marker}}",
|
||||
}, nil)
|
||||
got, urls, err := rule.executeEvaluate(&ExecuteRuleInput{Values: values}, "", "original", "{{interaction}}", urls)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, urls, 1)
|
||||
require.Equal(t, "|nslookup "+urls[0]+"|curl "+urls[0], got)
|
||||
require.NotContains(t, got, "{{marker}}")
|
||||
require.NotContains(t, got, "{{interactsh-url}}")
|
||||
}
|
||||
|
||||
func TestPrepareGeneratorValuesDoesNotAllocateRuntimeInteractshMarker(t *testing.T) {
|
||||
rule := newFuzzGeneratorTestRule(t, newFuzzTestVariables())
|
||||
|
||||
values, urls := rule.prepareGeneratorValues(
|
||||
map[string]interface{}{
|
||||
"payload": "{{server_value}}",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"server_value": "{{interactsh-url}}",
|
||||
},
|
||||
)
|
||||
got, urls, err := rule.executeEvaluate(&ExecuteRuleInput{Values: values}, "", "original", "{{payload}}", urls)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, urls)
|
||||
require.Equal(t, "{{interactsh-url}}", got)
|
||||
}
|
||||
|
||||
func TestPrepareGeneratorValuesDoesNotRenderConstantOverride(t *testing.T) {
|
||||
rule := newFuzzGeneratorTestRule(t, newFuzzTestVariables())
|
||||
rule.options.Constants = map[string]interface{}{
|
||||
"payload": "{{interactsh-url}}",
|
||||
}
|
||||
|
||||
values, urls := rule.prepareGeneratorValues(
|
||||
map[string]interface{}{
|
||||
"payload": "{{interactsh-url}}",
|
||||
},
|
||||
nil,
|
||||
)
|
||||
got, urls, err := rule.executeEvaluate(&ExecuteRuleInput{Values: values}, "", "original", "{{payload}}", urls)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, urls)
|
||||
require.Equal(t, "{{interactsh-url}}", got)
|
||||
}
|
||||
|
||||
func TestMergeInteractURLsPreservesCallerURLs(t *testing.T) {
|
||||
got := mergeInteractURLs(
|
||||
[]string{"https://caller.example/a", "https://shared.example"},
|
||||
[]string{"https://generated.example/b", "https://shared.example"},
|
||||
)
|
||||
|
||||
require.Equal(t, []string{
|
||||
"https://caller.example/a",
|
||||
"https://shared.example",
|
||||
"https://generated.example/b",
|
||||
}, got)
|
||||
}
|
||||
|
||||
func TestExecuteEvaluateDoesNotReevaluateRenderedPayload(t *testing.T) {
|
||||
templateVars := variables.Variable{
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(0),
|
||||
}
|
||||
rule := &Rule{
|
||||
ruleType: replaceRuleType,
|
||||
options: &protocols.ExecutorOptions{
|
||||
Variables: templateVars,
|
||||
Options: &types.Options{},
|
||||
Interactsh: &interactsh.Client{},
|
||||
},
|
||||
}
|
||||
|
||||
input := &ExecuteRuleInput{
|
||||
Values: map[string]interface{}{
|
||||
"server_value": "{{secret}}",
|
||||
"secret": "leaked-secret",
|
||||
},
|
||||
}
|
||||
|
||||
got, urls, err := rule.executeEvaluate(input, "", "original", "{{server_value}}", nil)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, urls)
|
||||
require.Equal(t, "{{secret}}", got)
|
||||
}
|
||||
|
||||
func TestExecuteEvaluateReturnsRenderError(t *testing.T) {
|
||||
templateVars := variables.Variable{
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(0),
|
||||
}
|
||||
rule := &Rule{
|
||||
ruleType: replaceRuleType,
|
||||
options: &protocols.ExecutorOptions{
|
||||
Variables: templateVars,
|
||||
Options: &types.Options{},
|
||||
},
|
||||
}
|
||||
|
||||
got, urls, err := rule.executeEvaluate(&ExecuteRuleInput{}, "", "original", "{{md5(missing)}}", nil)
|
||||
|
||||
require.Error(t, err)
|
||||
require.Empty(t, urls)
|
||||
require.Equal(t, "{{md5(missing)}}", got)
|
||||
}
|
||||
|
||||
func TestExecuteEvaluateReplacesHelperEncodedTemplateInteractshMarker(t *testing.T) {
|
||||
templateVars := variables.Variable{
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(0),
|
||||
}
|
||||
rule := &Rule{
|
||||
ruleType: replaceRuleType,
|
||||
options: &protocols.ExecutorOptions{
|
||||
Variables: templateVars,
|
||||
Options: &types.Options{},
|
||||
Interactsh: newFuzzTestInteractshClient(t),
|
||||
},
|
||||
}
|
||||
|
||||
got, urls, err := rule.executeEvaluate(&ExecuteRuleInput{}, "", "original", "{{url_encode('{{interactsh-url}}')}}", nil)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, urls, 1)
|
||||
require.Equal(t, urls[0], got)
|
||||
require.NotContains(t, got, "{{interactsh-url}}")
|
||||
require.NotContains(t, got, "%7B%7Binteractsh-url%7D%7D")
|
||||
}
|
||||
|
||||
func TestExecuteEvaluateDoesNotReplaceRuntimeInteractshMarkers(t *testing.T) {
|
||||
items := []struct {
|
||||
name string
|
||||
value string
|
||||
}{
|
||||
{name: "raw marker", value: "{{interactsh-url}}"},
|
||||
{name: "url encoded marker", value: "%7B%7Binteractsh-url%7D%7D"},
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
t.Run(item.name, func(t *testing.T) {
|
||||
templateVars := variables.Variable{
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(0),
|
||||
}
|
||||
rule := &Rule{
|
||||
ruleType: replaceRuleType,
|
||||
options: &protocols.ExecutorOptions{
|
||||
Variables: templateVars,
|
||||
Options: &types.Options{},
|
||||
Interactsh: newFuzzTestInteractshClient(t),
|
||||
},
|
||||
}
|
||||
input := &ExecuteRuleInput{
|
||||
Values: map[string]interface{}{
|
||||
"server_value": item.value,
|
||||
},
|
||||
}
|
||||
|
||||
got, urls, err := rule.executeEvaluate(input, "", "original", "{{server_value}}", nil)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, urls)
|
||||
require.Equal(t, item.value, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteEvaluateUsesRenderedPayloadForAllRuleTypes(t *testing.T) {
|
||||
items := []struct {
|
||||
name string
|
||||
ruleType ruleType
|
||||
value string
|
||||
replaceRegex string
|
||||
expected string
|
||||
}{
|
||||
{name: "prefix", ruleType: prefixRuleType, value: "base", expected: "{{secret}}base"},
|
||||
{name: "postfix", ruleType: postfixRuleType, value: "base", expected: "base{{secret}}"},
|
||||
{name: "infix", ruleType: infixRuleType, value: "base", expected: "ba{{secret}}se"},
|
||||
{name: "replace", ruleType: replaceRuleType, value: "base", expected: "{{secret}}"},
|
||||
{name: "replace-regex", ruleType: replaceRegexRuleType, value: "base123", replaceRegex: `[0-9]+`, expected: "base{{secret}}"},
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
t.Run(item.name, func(t *testing.T) {
|
||||
templateVars := variables.Variable{
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(0),
|
||||
}
|
||||
rule := &Rule{
|
||||
ruleType: item.ruleType,
|
||||
options: &protocols.ExecutorOptions{
|
||||
Variables: templateVars,
|
||||
Options: &types.Options{},
|
||||
Interactsh: &interactsh.Client{},
|
||||
},
|
||||
}
|
||||
if item.replaceRegex != "" {
|
||||
rule.replaceRegex = regexp.MustCompile(item.replaceRegex)
|
||||
}
|
||||
|
||||
input := &ExecuteRuleInput{
|
||||
Values: map[string]interface{}{
|
||||
"server_value": "{{secret}}",
|
||||
"secret": "leaked-secret",
|
||||
},
|
||||
}
|
||||
|
||||
got, urls, err := rule.executeEvaluate(input, "", item.value, "{{server_value}}", nil)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, urls)
|
||||
require.Equal(t, item.expected, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package matchers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
)
|
||||
|
||||
var dslStringMarkerRegex = regexp.MustCompile(`\{\{([^{}]+)\}\}`)
|
||||
|
||||
// dslStringMarker represents a span of an unresolved DSL expression within a
|
||||
// string literal.
|
||||
type dslStringMarker struct {
|
||||
start int
|
||||
end int
|
||||
expr string
|
||||
quote byte
|
||||
}
|
||||
|
||||
type textSpan struct {
|
||||
start int
|
||||
end int
|
||||
}
|
||||
|
||||
type stringLiteralSpan struct {
|
||||
start int
|
||||
end int
|
||||
quote byte
|
||||
}
|
||||
|
||||
func resolveDSLStringMarkers(expression string, data map[string]interface{}) (string, error) {
|
||||
// Resolve only marker spans already present in the compiled DSL source.
|
||||
// Values are escaped for the surrounding string literal before recompilation.
|
||||
stringSpans := findStringLiteralSpans(expression)
|
||||
markers, err := findDSLStringMarkers(expression, data, stringSpans)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if len(markers) == 0 {
|
||||
return "", fmt.Errorf("unresolved DSL placeholders must be inside string literals")
|
||||
}
|
||||
|
||||
sort.Slice(markers, func(i, j int) bool {
|
||||
return markers[i].start > markers[j].start
|
||||
})
|
||||
|
||||
resolved := expression
|
||||
for _, marker := range markers {
|
||||
result, err := render.Render(render.Input{
|
||||
Text: "{{" + marker.expr + "}}",
|
||||
Values: data,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
replacement := expressions.EscapeStringValue(result.Text, marker.quote)
|
||||
resolved = resolved[:marker.start] + replacement + resolved[marker.end:]
|
||||
}
|
||||
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func findDSLStringMarkers(expression string, data map[string]interface{}, stringSpans []stringLiteralSpan) ([]dslStringMarker, error) {
|
||||
var markers []dslStringMarker
|
||||
var occupiedSpans []textSpan
|
||||
|
||||
seenComplex := make(map[string]struct{})
|
||||
|
||||
for _, expr := range expressions.FindExpressions(expression, "{{", "}}", data) {
|
||||
if _, ok := seenComplex[expr]; ok {
|
||||
continue
|
||||
}
|
||||
seenComplex[expr] = struct{}{}
|
||||
|
||||
marker := "{{" + expr + "}}"
|
||||
for start := 0; ; {
|
||||
index := strings.Index(expression[start:], marker)
|
||||
if index < 0 {
|
||||
break
|
||||
}
|
||||
|
||||
index += start
|
||||
end := index + len(marker)
|
||||
|
||||
stringSpan, ok := stringLiteralForSpan(stringSpans, index, end)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unresolved DSL placeholder %q is not inside a string literal", expr)
|
||||
}
|
||||
|
||||
got := dslStringMarker{start: index, end: end, expr: expr, quote: stringSpan.quote}
|
||||
markers = append(markers, got)
|
||||
occupiedSpans = append(occupiedSpans, textSpan{start: index, end: end})
|
||||
start = end
|
||||
}
|
||||
}
|
||||
|
||||
for _, match := range dslStringMarkerRegex.FindAllStringSubmatchIndex(expression, -1) {
|
||||
if len(match) < 4 || markerWithinSpans(match[0], match[1], occupiedSpans) {
|
||||
continue
|
||||
}
|
||||
|
||||
stringSpan, ok := stringLiteralForSpan(stringSpans, match[0], match[1])
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unresolved DSL placeholder %q is not inside a string literal", expression[match[2]:match[3]])
|
||||
}
|
||||
|
||||
markers = append(markers, dslStringMarker{
|
||||
start: match[0],
|
||||
end: match[1],
|
||||
expr: expression[match[2]:match[3]],
|
||||
quote: stringSpan.quote,
|
||||
})
|
||||
}
|
||||
|
||||
return markers, nil
|
||||
}
|
||||
|
||||
func markerWithinSpans(start, end int, spans []textSpan) bool {
|
||||
for _, span := range spans {
|
||||
if start >= span.start && end <= span.end {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func findStringLiteralSpans(expression string) []stringLiteralSpan {
|
||||
var spans []stringLiteralSpan
|
||||
var quote byte
|
||||
start := 0
|
||||
escaped := false
|
||||
|
||||
for i := 0; i < len(expression); i++ {
|
||||
char := expression[i]
|
||||
if escaped {
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
|
||||
if char == '\\' {
|
||||
escaped = true
|
||||
continue
|
||||
}
|
||||
|
||||
if quote != 0 {
|
||||
if char == quote {
|
||||
spans = append(spans, stringLiteralSpan{start: start, end: i, quote: quote})
|
||||
quote = 0
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if char == '\'' || char == '"' {
|
||||
quote = char
|
||||
start = i + 1
|
||||
}
|
||||
}
|
||||
|
||||
if quote != 0 {
|
||||
spans = append(spans, stringLiteralSpan{start: start, end: len(expression), quote: quote})
|
||||
}
|
||||
|
||||
return spans
|
||||
}
|
||||
|
||||
func stringLiteralForSpan(spans []stringLiteralSpan, start, end int) (stringLiteralSpan, bool) {
|
||||
for _, span := range spans {
|
||||
if start >= span.start && end <= span.end {
|
||||
return span, true
|
||||
}
|
||||
}
|
||||
|
||||
return stringLiteralSpan{}, false
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
)
|
||||
|
||||
@@ -64,13 +65,14 @@ func (matcher *Matcher) MatchWords(corpus string, data map[string]interface{}) (
|
||||
data = make(map[string]interface{})
|
||||
}
|
||||
|
||||
var err error
|
||||
word, err = expressions.Evaluate(word, data)
|
||||
result, err := render.Render(render.Input{Text: word, Values: data})
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("Error while evaluating word matcher: %q", word)
|
||||
if matcher.condition == ANDCondition {
|
||||
return false, []string{}
|
||||
}
|
||||
} else {
|
||||
word = result.Text
|
||||
}
|
||||
// Continue if the word doesn't match
|
||||
if !strings.Contains(corpus, word) {
|
||||
@@ -195,7 +197,7 @@ func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool {
|
||||
// Iterate over all the expressions accepted as valid
|
||||
for i, expression := range matcher.dslCompiled {
|
||||
if varErr := expressions.ContainsUnresolvedVariables(expression.String()); varErr != nil {
|
||||
resolvedExpression, err := expressions.Evaluate(expression.String(), data)
|
||||
resolvedExpression, err := resolveDSLStringMarkers(expression.String(), data)
|
||||
if err != nil {
|
||||
logExpressionEvaluationFailure(matcher.Name, err)
|
||||
return false
|
||||
|
||||
@@ -8,6 +8,31 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func withMatcherTestHelperFunction(t *testing.T, name string, fn govaluate.ExpressionFunction) {
|
||||
t.Helper()
|
||||
|
||||
originalFn, hadFn := dsl.HelperFunctions[name]
|
||||
dsl.HelperFunctions[name] = fn
|
||||
t.Cleanup(func() {
|
||||
if hadFn {
|
||||
dsl.HelperFunctions[name] = originalFn
|
||||
return
|
||||
}
|
||||
delete(dsl.HelperFunctions, name)
|
||||
})
|
||||
}
|
||||
|
||||
func mustCompileDSLMatcher(t *testing.T, expression string) *Matcher {
|
||||
t.Helper()
|
||||
|
||||
matcher := &Matcher{
|
||||
Type: MatcherTypeHolder{MatcherType: DSLMatcher},
|
||||
DSL: []string{expression},
|
||||
}
|
||||
require.NoError(t, matcher.CompileMatchers())
|
||||
return matcher
|
||||
}
|
||||
|
||||
func TestWordANDCondition(t *testing.T) {
|
||||
m := &Matcher{condition: ANDCondition, Words: []string{"a", "b"}}
|
||||
|
||||
@@ -90,6 +115,148 @@ func TestMatcher_MatchDSL(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatcherMatchDSLDoesNotExecuteHelpersFromResolvedValues(t *testing.T) {
|
||||
var waitForCalls int
|
||||
withMatcherTestHelperFunction(t, "wait_for", func(args ...interface{}) (interface{}, error) {
|
||||
waitForCalls++
|
||||
return true, nil
|
||||
})
|
||||
|
||||
items := []struct {
|
||||
name string
|
||||
expression string
|
||||
value string
|
||||
}{
|
||||
{
|
||||
name: "single quoted placeholder",
|
||||
expression: "contains(body, '{{server_token}}')",
|
||||
value: "') && wait_for(5) && contains(body, '",
|
||||
},
|
||||
{
|
||||
name: "double quoted placeholder",
|
||||
expression: `contains(body, "{{server_token}}")`,
|
||||
value: `") && wait_for(5) && contains(body, "`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
t.Run(item.name, func(t *testing.T) {
|
||||
matcher := mustCompileDSLMatcher(t, item.expression)
|
||||
|
||||
require.False(t, matcher.MatchDSL(map[string]interface{}{
|
||||
"template-id": "test-template",
|
||||
"body": "safe body",
|
||||
"server_token": item.value,
|
||||
}))
|
||||
require.Zero(t, waitForCalls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatcherMatchDSLMatchesResolvedValuesLiterally(t *testing.T) {
|
||||
var waitForCalls int
|
||||
withMatcherTestHelperFunction(t, "wait_for", func(args ...interface{}) (interface{}, error) {
|
||||
waitForCalls++
|
||||
return true, nil
|
||||
})
|
||||
|
||||
items := []struct {
|
||||
name string
|
||||
expression string
|
||||
value string
|
||||
}{
|
||||
{
|
||||
name: "single quoted placeholder",
|
||||
expression: "contains(body, '{{server_token}}')",
|
||||
value: "') && wait_for(5) && contains(body, '",
|
||||
},
|
||||
{
|
||||
name: "double quoted placeholder",
|
||||
expression: `contains(body, "{{server_token}}")`,
|
||||
value: `") && wait_for(5) && contains(body, "`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
t.Run(item.name, func(t *testing.T) {
|
||||
matcher := mustCompileDSLMatcher(t, item.expression)
|
||||
|
||||
require.True(t, matcher.MatchDSL(map[string]interface{}{
|
||||
"template-id": "test-template",
|
||||
"body": "prefix " + item.value + " suffix",
|
||||
"server_token": item.value,
|
||||
}))
|
||||
require.Zero(t, waitForCalls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatcherMatchDSLResolvedValuesPreserveBytes(t *testing.T) {
|
||||
items := []struct {
|
||||
name string
|
||||
value string
|
||||
}{
|
||||
{
|
||||
name: "newline",
|
||||
value: "a\nb",
|
||||
},
|
||||
{
|
||||
name: "tab",
|
||||
value: "a\tb",
|
||||
},
|
||||
{
|
||||
name: "carriage return",
|
||||
value: "a\rb",
|
||||
},
|
||||
{
|
||||
name: "backslash",
|
||||
value: `a\b`,
|
||||
},
|
||||
{
|
||||
name: "single quote",
|
||||
value: "a'b",
|
||||
},
|
||||
{
|
||||
name: "double quote",
|
||||
value: `a"b`,
|
||||
},
|
||||
{
|
||||
name: "mixed quotes and controls",
|
||||
value: "a\nb\tc\rd\\e'f\"g",
|
||||
},
|
||||
}
|
||||
|
||||
expressions := []struct {
|
||||
name string
|
||||
expression string
|
||||
}{
|
||||
{
|
||||
name: "single quoted placeholder",
|
||||
expression: "contains(body, '{{server_token}}')",
|
||||
},
|
||||
{
|
||||
name: "double quoted placeholder",
|
||||
expression: `contains(body, "{{server_token}}")`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, expression := range expressions {
|
||||
t.Run(expression.name, func(t *testing.T) {
|
||||
for _, item := range items {
|
||||
t.Run(item.name, func(t *testing.T) {
|
||||
matcher := mustCompileDSLMatcher(t, expression.expression)
|
||||
|
||||
require.True(t, matcher.MatchDSL(map[string]interface{}{
|
||||
"template-id": "test-template",
|
||||
"body": "prefix " + item.value + " suffix",
|
||||
"server_token": item.value,
|
||||
}))
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatcher_MatchXPath_HTML(t *testing.T) {
|
||||
body := `<!doctype html>
|
||||
<html>
|
||||
|
||||
@@ -8,9 +8,9 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/goja"
|
||||
"github.com/alecthomas/chroma/quick"
|
||||
"github.com/ditashi/jsbeautifier-go/jsbeautifier"
|
||||
"github.com/projectdiscovery/goja"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/gozero"
|
||||
@@ -188,8 +188,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
||||
}
|
||||
}()
|
||||
|
||||
var interactshURLs []string
|
||||
|
||||
// inject all template context values as gozero env allvars
|
||||
allvars := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil)
|
||||
// add template context values if available
|
||||
@@ -200,13 +198,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
||||
allvars = generators.MergeMaps(allvars, dynamicValues, previous)
|
||||
// optionvars are vars passed from CLI or env variables
|
||||
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
|
||||
variablesMap := request.options.Variables.Evaluate(allvars)
|
||||
scope := request.options.NewVariablesScope(allvars, optionVars, request.options.Constants)
|
||||
evaluation := request.options.Variables.EvaluateWithInteractshScope(scope, request.options.Interactsh)
|
||||
variablesMap, interactshURLs := evaluation.Values, evaluation.InteractURLs
|
||||
// since we evaluate variables using allvars, give precedence to variablesMap
|
||||
allvars = generators.MergeMaps(allvars, variablesMap, optionVars, request.options.Constants)
|
||||
for name, value := range allvars {
|
||||
v := fmt.Sprint(value)
|
||||
v, interactshURLs = request.options.Interactsh.Replace(v, interactshURLs)
|
||||
// if value is updated by interactsh, update allvars to reflect the change downstream
|
||||
allvars[name] = v
|
||||
metaSrc.AddVariable(gozerotypes.Variable{Name: name, Value: v})
|
||||
}
|
||||
|
||||
@@ -5,14 +5,20 @@ package code
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
)
|
||||
|
||||
func newTestRequest(t *testing.T, engine []string, source string) (*Request, *testutils.TemplateInfo) {
|
||||
@@ -80,3 +86,71 @@ func TestCodeProtocolMixedOutput(t *testing.T) {
|
||||
require.Equal(t, "stdout-data", gotEvent["response"])
|
||||
require.Contains(t, gotEvent["stderr"], "stderr-data")
|
||||
}
|
||||
|
||||
func TestCodeProtocolDoesNotTrackOptionOverriddenInteractshVariable(t *testing.T) {
|
||||
gotEvent := executeCodeProtocolWithCallbackOverride(t, func(options *types.Options, _ *protocols.ExecutorOptions) {
|
||||
require.NoError(t, options.Vars.Set("callback=option-callback"))
|
||||
})
|
||||
|
||||
require.Equal(t, "option-callback", gotEvent["callback"])
|
||||
require.Equal(t, "option-callback", gotEvent["response"])
|
||||
require.NotContains(t, gotEvent, "interactsh-url")
|
||||
require.NotContains(t, gotEvent, "interactsh-id")
|
||||
}
|
||||
|
||||
func TestCodeProtocolDoesNotTrackConstantOverriddenInteractshVariable(t *testing.T) {
|
||||
gotEvent := executeCodeProtocolWithCallbackOverride(t, func(_ *types.Options, executerOpts *protocols.ExecutorOptions) {
|
||||
executerOpts.Constants = map[string]interface{}{
|
||||
"callback": "constant-callback",
|
||||
}
|
||||
})
|
||||
|
||||
require.Equal(t, "constant-callback", gotEvent["callback"])
|
||||
require.Equal(t, "constant-callback", gotEvent["response"])
|
||||
require.NotContains(t, gotEvent, "interactsh-url")
|
||||
require.NotContains(t, gotEvent, "interactsh-id")
|
||||
}
|
||||
|
||||
func executeCodeProtocolWithCallbackOverride(t *testing.T, configure func(*types.Options, *protocols.ExecutorOptions)) output.InternalEvent {
|
||||
t.Helper()
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.InteractionsCoolDownPeriod = 0
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() {
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
request, info := newTestRequest(t, []string{"sh"}, "echo $callback")
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, info)
|
||||
configure(options, executerOpts)
|
||||
executerOpts.Variables = variables.Variable{
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
|
||||
}
|
||||
executerOpts.Variables.Set("callback", "{{interactsh-url}}")
|
||||
|
||||
client, err := interactsh.New(&interactsh.Options{
|
||||
ServerURL: options.InteractshURL,
|
||||
CacheSize: options.InteractionsCacheSize,
|
||||
Eviction: time.Duration(options.InteractionsEviction) * time.Second,
|
||||
CooldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second,
|
||||
PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second,
|
||||
DisableHttpFallback: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
client.Close()
|
||||
})
|
||||
executerOpts.Interactsh = client
|
||||
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
var gotEvent output.InternalEvent
|
||||
ctxArgs := contextargs.NewWithInput(context.Background(), "")
|
||||
err = request.ExecuteWithResults(ctxArgs, nil, nil, func(event *output.InternalWrappedEvent) {
|
||||
gotEvent = event.InternalEvent
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
return gotEvent
|
||||
}
|
||||
@@ -30,7 +30,8 @@ type Context struct {
|
||||
CookieJar *cookiejar.Jar
|
||||
|
||||
// Args is a workflow shared key-value store
|
||||
args *mapsutil.SyncLockMap[string, interface{}]
|
||||
args *mapsutil.SyncLockMap[string, interface{}]
|
||||
templateVariables *mapsutil.SyncLockMap[string, struct{}]
|
||||
}
|
||||
|
||||
// Create a new contextargs instance
|
||||
@@ -61,6 +62,10 @@ func NewWithInput(ctx context.Context, input string) *Context {
|
||||
Map: make(map[string]interface{}),
|
||||
ReadOnly: atomic.Bool{},
|
||||
},
|
||||
templateVariables: &mapsutil.SyncLockMap[string, struct{}]{
|
||||
Map: make(map[string]struct{}),
|
||||
ReadOnly: atomic.Bool{},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,6 +77,7 @@ func (ctx *Context) Context() context.Context {
|
||||
// Set the specific key-value pair
|
||||
func (ctx *Context) Set(key string, value interface{}) {
|
||||
_ = ctx.args.Set(key, value)
|
||||
ctx.clearTemplateVariable(key)
|
||||
}
|
||||
|
||||
func (ctx *Context) hasArgs() bool {
|
||||
@@ -81,6 +87,17 @@ func (ctx *Context) hasArgs() bool {
|
||||
// Merge the key-value pairs
|
||||
func (ctx *Context) Merge(args map[string]interface{}) {
|
||||
_ = ctx.args.Merge(args)
|
||||
for key := range args {
|
||||
ctx.clearTemplateVariable(key)
|
||||
}
|
||||
}
|
||||
|
||||
// MergeTemplateVariables merges values produced from template variables.
|
||||
func (ctx *Context) MergeTemplateVariables(args map[string]interface{}) {
|
||||
_ = ctx.args.Merge(args)
|
||||
for key := range args {
|
||||
ctx.setTemplateVariable(key)
|
||||
}
|
||||
}
|
||||
|
||||
// Add the specific key-value pair
|
||||
@@ -177,6 +194,41 @@ func (ctx *Context) GetAll() map[string]interface{} {
|
||||
return ctx.args.Clone().Map
|
||||
}
|
||||
|
||||
// GetTemplateVariables returns values currently owned by template variable
|
||||
// evaluation.
|
||||
func (ctx *Context) GetTemplateVariables() map[string]interface{} {
|
||||
if ctx.templateVariables == nil || ctx.templateVariables.IsEmpty() {
|
||||
return nil
|
||||
}
|
||||
|
||||
values := make(map[string]interface{})
|
||||
for key := range ctx.templateVariables.Clone().Map {
|
||||
value, ok := ctx.Get(key)
|
||||
if ok {
|
||||
values[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
return values
|
||||
}
|
||||
|
||||
func (ctx *Context) setTemplateVariable(key string) {
|
||||
if ctx.templateVariables == nil {
|
||||
ctx.templateVariables = &mapsutil.SyncLockMap[string, struct{}]{
|
||||
Map: make(map[string]struct{}),
|
||||
ReadOnly: atomic.Bool{},
|
||||
}
|
||||
}
|
||||
_ = ctx.templateVariables.Set(key, struct{}{})
|
||||
}
|
||||
|
||||
func (ctx *Context) clearTemplateVariable(key string) {
|
||||
if ctx.templateVariables == nil {
|
||||
return
|
||||
}
|
||||
ctx.templateVariables.Delete(key)
|
||||
}
|
||||
|
||||
func (ctx *Context) ForEach(f func(string, interface{})) {
|
||||
_ = ctx.args.Iterate(func(k string, v interface{}) error {
|
||||
f(k, v)
|
||||
@@ -195,10 +247,11 @@ func (ctx *Context) HasArgs() bool {
|
||||
|
||||
func (ctx *Context) Clone() *Context {
|
||||
newCtx := &Context{
|
||||
ctx: ctx.ctx,
|
||||
MetaInput: ctx.MetaInput.Clone(),
|
||||
args: ctx.args.Clone(),
|
||||
CookieJar: ctx.CookieJar,
|
||||
ctx: ctx.ctx,
|
||||
MetaInput: ctx.MetaInput.Clone(),
|
||||
args: ctx.args.Clone(),
|
||||
templateVariables: ctx.templateVariables.Clone(),
|
||||
CookieJar: ctx.CookieJar,
|
||||
}
|
||||
return newCtx
|
||||
}
|
||||
|
||||
@@ -224,6 +224,41 @@ func TestUseNetworkPortIPv6(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemplateVariableOwnershipClearsOnDataWrites(t *testing.T) {
|
||||
ctx := New(context.Background())
|
||||
|
||||
ctx.MergeTemplateVariables(map[string]interface{}{
|
||||
"token": "{{extracted_token}}",
|
||||
})
|
||||
require.Equal(t, map[string]interface{}{"token": "{{extracted_token}}"}, ctx.GetTemplateVariables())
|
||||
|
||||
ctx.Set("token", "{{runtime_token}}")
|
||||
require.Empty(t, ctx.GetTemplateVariables())
|
||||
|
||||
ctx.MergeTemplateVariables(map[string]interface{}{
|
||||
"token": "{{extracted_token}}",
|
||||
})
|
||||
ctx.Merge(map[string]interface{}{
|
||||
"token": "{{runtime_token}}",
|
||||
})
|
||||
require.Empty(t, ctx.GetTemplateVariables())
|
||||
require.Equal(t, map[string]interface{}{"token": "{{runtime_token}}"}, ctx.GetAll())
|
||||
}
|
||||
|
||||
func TestTemplateVariableOwnershipIsCloned(t *testing.T) {
|
||||
ctx := New(context.Background())
|
||||
ctx.MergeTemplateVariables(map[string]interface{}{
|
||||
"token": "{{extracted_token}}",
|
||||
})
|
||||
|
||||
cloned := ctx.Clone()
|
||||
require.Equal(t, map[string]interface{}{"token": "{{extracted_token}}"}, cloned.GetTemplateVariables())
|
||||
|
||||
cloned.Set("token", "{{runtime_token}}")
|
||||
require.Empty(t, cloned.GetTemplateVariables())
|
||||
require.Equal(t, map[string]interface{}{"token": "{{extracted_token}}"}, ctx.GetTemplateVariables())
|
||||
}
|
||||
|
||||
// TestUseNetworkPortServiceNameExclude verifies excludePorts accepts service
|
||||
// names (resolved via portutil), e.g. "http" -> "80".
|
||||
func TestUseNetworkPortServiceNameExclude(t *testing.T) {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/marker"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
)
|
||||
|
||||
@@ -54,8 +55,10 @@ func evaluate(data string, base map[string]interface{}) (string, error) {
|
||||
// literals like {{2+2}} are not considered expressions
|
||||
for _, expression := range expressions {
|
||||
originalExpression := expression
|
||||
// replace variable placeholders with base values
|
||||
expression = replacer.Replace(expression, base)
|
||||
// data has already had simple placeholders replaced; keep the same
|
||||
// marker shape for output replacement, but never compile this string.
|
||||
replacedExpression := replacer.Replace(expression, base)
|
||||
expression = replaceStringPlaceholders(expression, base)
|
||||
|
||||
// turns expressions (either helper functions+base values or base values)
|
||||
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions)
|
||||
@@ -86,11 +89,115 @@ func evaluate(data string, base map[string]interface{}) (string, error) {
|
||||
}
|
||||
|
||||
// replace incrementally
|
||||
data = replacer.ReplaceOne(data, expression, replacement)
|
||||
data = replacer.ReplaceOne(data, replacedExpression, replacement)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func replaceStringPlaceholders(expression string, base map[string]interface{}) string {
|
||||
if len(base) == 0 || (!strings.Contains(expression, marker.ParenthesisOpen) && !strings.Contains(expression, marker.General)) {
|
||||
return expression
|
||||
}
|
||||
|
||||
var builder strings.Builder
|
||||
builder.Grow(len(expression))
|
||||
|
||||
var quote byte
|
||||
escaped := false
|
||||
for i := 0; i < len(expression); {
|
||||
char := expression[i]
|
||||
|
||||
if escaped {
|
||||
builder.WriteByte(char)
|
||||
escaped = false
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
if quote != 0 {
|
||||
if char == '\\' {
|
||||
builder.WriteByte(char)
|
||||
escaped = true
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
if char == quote {
|
||||
builder.WriteByte(char)
|
||||
quote = 0
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
replacement, next, ok := stringPlaceholderReplacement(expression, i, quote, base, marker.ParenthesisOpen, marker.ParenthesisClose)
|
||||
if ok {
|
||||
builder.WriteString(replacement)
|
||||
i = next
|
||||
continue
|
||||
}
|
||||
|
||||
replacement, next, ok = stringPlaceholderReplacement(expression, i, quote, base, marker.General, marker.General)
|
||||
if ok {
|
||||
builder.WriteString(replacement)
|
||||
i = next
|
||||
continue
|
||||
}
|
||||
|
||||
builder.WriteByte(char)
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
if char == '\'' || char == '"' {
|
||||
quote = char
|
||||
}
|
||||
|
||||
builder.WriteByte(char)
|
||||
i++
|
||||
}
|
||||
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func stringPlaceholderReplacement(expression string, index int, quote byte, base map[string]interface{}, open, close string) (string, int, bool) {
|
||||
if !strings.HasPrefix(expression[index:], open) {
|
||||
return "", 0, false
|
||||
}
|
||||
|
||||
markerStart := index + len(open)
|
||||
markerEnd := strings.Index(expression[markerStart:], close)
|
||||
if markerEnd < 0 {
|
||||
return "", 0, false
|
||||
}
|
||||
|
||||
key := expression[markerStart : markerStart+markerEnd]
|
||||
value, ok := base[key]
|
||||
if !ok {
|
||||
return "", 0, false
|
||||
}
|
||||
|
||||
replacement := EscapeStringValue(types.ToString(value), quote)
|
||||
next := markerStart + markerEnd + len(close)
|
||||
return replacement, next, true
|
||||
}
|
||||
|
||||
// EscapeStringValue escapes value for insertion into a govaluate string literal
|
||||
// that is already delimited by quote.
|
||||
func EscapeStringValue(value string, quote byte) string {
|
||||
var builder strings.Builder
|
||||
builder.Grow(len(value))
|
||||
|
||||
for i := 0; i < len(value); i++ {
|
||||
char := value[i]
|
||||
if char == '\\' || char == quote {
|
||||
builder.WriteByte('\\')
|
||||
}
|
||||
builder.WriteByte(char)
|
||||
}
|
||||
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
// maxIterations to avoid infinite loop
|
||||
const maxIterations = 250
|
||||
|
||||
|
||||
@@ -122,6 +122,73 @@ func TestEvaluateDoesNotExecuteHelpersFromResolvedValues(t *testing.T) {
|
||||
require.Zero(t, calls)
|
||||
}
|
||||
|
||||
func TestEvaluateDoesNotCompileResolvedValuesInsideHelperExpressions(t *testing.T) {
|
||||
var calls int
|
||||
|
||||
withTestHelperFunction(t, "test_side_effect", func(args ...interface{}) (interface{}, error) {
|
||||
calls++
|
||||
return true, nil
|
||||
})
|
||||
|
||||
items := []struct {
|
||||
name string
|
||||
expression string
|
||||
}{
|
||||
{
|
||||
name: "parenthesis marker",
|
||||
expression: "{{contains('{{body}}', 'needle')}}",
|
||||
},
|
||||
{
|
||||
name: "general marker",
|
||||
expression: "{{contains('§body§', 'needle')}}",
|
||||
},
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
t.Run(item.name, func(t *testing.T) {
|
||||
calls = 0
|
||||
|
||||
value, err := Evaluate(item.expression, map[string]interface{}{
|
||||
"body": "value', 'missing') || test_side_effect() || contains('",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "false", value)
|
||||
require.Zero(t, calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluatePreservesStringLiteralPlaceholderBytes(t *testing.T) {
|
||||
withTestHelperFunction(t, "test_echo", func(args ...interface{}) (interface{}, error) {
|
||||
require.Len(t, args, 1)
|
||||
return args[0], nil
|
||||
})
|
||||
|
||||
items := []struct {
|
||||
name string
|
||||
expression string
|
||||
}{
|
||||
{
|
||||
name: "parenthesis marker",
|
||||
expression: "{{test_echo('{{body}}')}}",
|
||||
},
|
||||
{
|
||||
name: "general marker",
|
||||
expression: "{{test_echo('§body§')}}",
|
||||
},
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
t.Run(item.name, func(t *testing.T) {
|
||||
value, err := Evaluate(item.expression, map[string]interface{}{
|
||||
"body": "a\nb\tc\rd\\e'f\"g",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "a\nb\tc\rd\\e'f\"g", value)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateHyphenatedPlaceholderNames(t *testing.T) {
|
||||
value, err := Evaluate("foo-bar: {{foo-bar}}", map[string]interface{}{
|
||||
"foo-bar": "baz",
|
||||
|
||||
@@ -2,20 +2,17 @@ package interactsh
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"regexp"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
defaultInteractionDuration = 60 * time.Second
|
||||
interactshURLMarkerRegex = regexp.MustCompile(`(%7[B|b]|\{){2}(interactsh-url(?:_[0-9]+){0,3})(%7[D|d]|\}){2}`)
|
||||
|
||||
ErrInteractshClientNotInitialized = errors.New("interactsh client not initialized")
|
||||
)
|
||||
|
||||
const (
|
||||
stopAtFirstMatchAttribute = "stop-at-first-match"
|
||||
templateIdAttribute = "template-id"
|
||||
|
||||
stopAtFirstMatchAttribute = "stop-at-first-match"
|
||||
templateIdAttribute = "template-id"
|
||||
defaultMaxInteractionsCount = 5000
|
||||
)
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/marker"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
@@ -273,12 +274,18 @@ func (c *Client) Close() bool {
|
||||
return c.matched.Load()
|
||||
}
|
||||
|
||||
// ReplaceMarkers replaces the default {{interactsh-url}} placeholders with interactsh urls
|
||||
// Replace replaces the default Interactsh URL placeholders with interactsh URLs.
|
||||
func (c *Client) Replace(data string, interactshURLs []string) (string, []string) {
|
||||
return c.ReplaceWithMarker(data, interactshURLMarkerRegex, interactshURLs)
|
||||
for _, interactshURLMarker := range marker.FindInteractshURLMarkers(data) {
|
||||
if url, err := c.NewURLWithData(interactshURLMarker); err == nil {
|
||||
interactshURLs = append(interactshURLs, url)
|
||||
data = strings.Replace(data, interactshURLMarker, url, 1)
|
||||
}
|
||||
}
|
||||
return data, interactshURLs
|
||||
}
|
||||
|
||||
// ReplaceMarkers replaces the placeholders with interactsh urls and appends them to interactshURLs
|
||||
// ReplaceWithMarker replaces custom regex matches with Interactsh URLs and appends them to interactshURLs.
|
||||
func (c *Client) ReplaceWithMarker(data string, regex *regexp.Regexp, interactshURLs []string) (string, []string) {
|
||||
for _, interactshURLMarker := range regex.FindAllString(data, -1) {
|
||||
if url, err := c.NewURLWithData(interactshURLMarker); err == nil {
|
||||
@@ -395,7 +402,7 @@ func HasMatchers(op *operators.Operators) bool {
|
||||
|
||||
// HasMarkers checks if the text contains interactsh markers
|
||||
func HasMarkers(data string) bool {
|
||||
return interactshURLMarkerRegex.Match([]byte(data))
|
||||
return marker.HasInteractshURLMarker(data)
|
||||
}
|
||||
|
||||
func (c *Client) debugPrintInteraction(interaction *server.Interaction, event *operators.Result) {
|
||||
|
||||
@@ -15,6 +15,27 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHasMarkersDoesNotTreatPipeAsEncodedBrace(t *testing.T) {
|
||||
require.True(t, HasMarkers("{{interactsh-url}}"))
|
||||
require.True(t, HasMarkers("%7B%7Binteractsh-url%7D%7D"))
|
||||
require.False(t, HasMarkers("%7|%7|interactsh-url%7|%7|"))
|
||||
}
|
||||
|
||||
func TestHasMarkersDoesNotTreatMixedRawEncodedBracesAsMarker(t *testing.T) {
|
||||
items := []string{
|
||||
"%7B{interactsh-url}}",
|
||||
"{{interactsh-url}%7D",
|
||||
"%7B%7Binteractsh-url}}",
|
||||
"{{interactsh-url%7D%7D",
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
t.Run(item, func(t *testing.T) {
|
||||
require.False(t, HasMarkers(item))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessInteractionForRequestConcurrentEventUpdate(t *testing.T) {
|
||||
const (
|
||||
keyCount = 4096
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package marker
|
||||
|
||||
import "strings"
|
||||
|
||||
const (
|
||||
// General marker (open/close)
|
||||
General = "§"
|
||||
@@ -8,3 +10,114 @@ const (
|
||||
// ParenthesisClose marker - end of a placeholder
|
||||
ParenthesisClose = "}}"
|
||||
)
|
||||
|
||||
const interactshURL = "interactsh-url"
|
||||
|
||||
// HasInteractshURLMarker returns true when data contains a complete Interactsh
|
||||
// URL marker. It accepts raw markers ({{interactsh-url}}) and fully URL-encoded
|
||||
// brace markers (%7B%7Binteractsh-url%7D%7D), but rejects mixed raw/encoded
|
||||
// brace pairs.
|
||||
func HasInteractshURLMarker(data string) bool {
|
||||
_, _, ok := NextInteractshURLMarker(data, 0)
|
||||
|
||||
return ok
|
||||
}
|
||||
|
||||
// FindInteractshURLMarkers returns complete Interactsh URL markers in data in
|
||||
// left-to-right order.
|
||||
func FindInteractshURLMarkers(data string) []string {
|
||||
var markers []string
|
||||
|
||||
for start := 0; start < len(data); {
|
||||
markerStart, markerEnd, ok := NextInteractshURLMarker(data, start)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
||||
markers = append(markers, data[markerStart:markerEnd])
|
||||
start = markerEnd
|
||||
}
|
||||
|
||||
return markers
|
||||
}
|
||||
|
||||
// NextInteractshURLMarker returns the next complete Interactsh URL marker span
|
||||
// at or after start.
|
||||
func NextInteractshURLMarker(data string, start int) (int, int, bool) {
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
|
||||
for i := start; i < len(data); i++ {
|
||||
if strings.HasPrefix(data[i:], ParenthesisOpen) {
|
||||
if end, ok := matchInteractshURLMarker(data, i+len(ParenthesisOpen), hasRawClose); ok {
|
||||
return i, end, true
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if hasEncodedOpen(data, i) {
|
||||
if end, ok := matchInteractshURLMarker(data, i+len("%7B%7B"), hasEncodedClose); ok {
|
||||
return i, end, true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0, 0, false
|
||||
}
|
||||
|
||||
func matchInteractshURLMarker(data string, bodyStart int, hasClose func(string, int) (int, bool)) (int, bool) {
|
||||
if !strings.HasPrefix(data[bodyStart:], interactshURL) {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
i := bodyStart + len(interactshURL)
|
||||
for suffixes := 0; suffixes < 3 && i < len(data) && data[i] == '_'; suffixes++ {
|
||||
next := i + 1
|
||||
if next >= len(data) || !isDigit(data[next]) {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
for next < len(data) && isDigit(data[next]) {
|
||||
next++
|
||||
}
|
||||
i = next
|
||||
}
|
||||
|
||||
return hasClose(data, i)
|
||||
}
|
||||
|
||||
func hasRawClose(data string, start int) (int, bool) {
|
||||
if strings.HasPrefix(data[start:], ParenthesisClose) {
|
||||
return start + len(ParenthesisClose), true
|
||||
}
|
||||
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func hasEncodedOpen(data string, start int) bool {
|
||||
return hasEncodedBrace(data, start, 'B') && hasEncodedBrace(data, start+len("%7B"), 'B')
|
||||
}
|
||||
|
||||
func hasEncodedClose(data string, start int) (int, bool) {
|
||||
if hasEncodedBrace(data, start, 'D') && hasEncodedBrace(data, start+len("%7D"), 'D') {
|
||||
return start + len("%7D%7D"), true
|
||||
}
|
||||
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func hasEncodedBrace(data string, start int, brace byte) bool {
|
||||
if start+3 > len(data) || data[start] != '%' || data[start+1] != '7' {
|
||||
return false
|
||||
}
|
||||
|
||||
got := data[start+2]
|
||||
|
||||
return got == brace || got == brace+'a'-'A'
|
||||
}
|
||||
|
||||
func isDigit(value byte) bool {
|
||||
return value >= '0' && value <= '9'
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package marker
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFindInteractshURLMarkers(t *testing.T) {
|
||||
data := "a {{interactsh-url}} b %7B%7binteractsh-url_1_22_333%7d%7D c"
|
||||
|
||||
require.Equal(t, []string{
|
||||
"{{interactsh-url}}",
|
||||
"%7B%7binteractsh-url_1_22_333%7d%7D",
|
||||
}, FindInteractshURLMarkers(data))
|
||||
}
|
||||
|
||||
func TestHasInteractshURLMarkerRejectsMixedRawEncodedBraces(t *testing.T) {
|
||||
items := []string{
|
||||
"%7B{interactsh-url}}",
|
||||
"{{interactsh-url}%7D",
|
||||
"%7B%7Binteractsh-url}}",
|
||||
"{{interactsh-url%7D%7D",
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
t.Run(item, func(t *testing.T) {
|
||||
require.False(t, HasInteractshURLMarker(item))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasInteractshURLMarkerRejectsMalformedSuffixes(t *testing.T) {
|
||||
items := []string{
|
||||
"{{interactsh-url_}}",
|
||||
"{{interactsh-url_a}}",
|
||||
"{{interactsh-url_1_2_3_4}}",
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
t.Run(item, func(t *testing.T) {
|
||||
require.False(t, HasInteractshURLMarker(item))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
// Package render centralizes rendering of template text with runtime values.
|
||||
package render
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/marker"
|
||||
)
|
||||
|
||||
// Values contains render values. Values are inserted as data; they are not
|
||||
// scanned as new template text after insertion.
|
||||
type Values = map[string]interface{}
|
||||
|
||||
// URLSource allocates URLs for Interactsh markers found in template text.
|
||||
type URLSource interface {
|
||||
NewURLWithData(data string) (string, error)
|
||||
}
|
||||
|
||||
// Input describes a render operation. Text is template text, Values are data,
|
||||
// and Interactsh is optional source-level marker handling.
|
||||
type Input struct {
|
||||
Text string
|
||||
Values Values
|
||||
Interactsh URLSource
|
||||
InteractURLs []string
|
||||
}
|
||||
|
||||
// Result is the terminal rendered data and any Interactsh URLs allocated while
|
||||
// rendering template text.
|
||||
type Result struct {
|
||||
Text string
|
||||
InteractURLs []string
|
||||
}
|
||||
|
||||
// MapInput describes a map render operation. Source values are template text.
|
||||
// Data values are terminal data that override Source and are not rendered.
|
||||
type MapInput struct {
|
||||
Source Values
|
||||
Data Values
|
||||
// Values is the evaluation context used while rendering Source.
|
||||
Values Values
|
||||
// Interactsh and InteractURLs carry source-level Interactsh allocations.
|
||||
Interactsh URLSource
|
||||
InteractURLs []string
|
||||
}
|
||||
|
||||
// MapResult contains the complete value map after Source rendering and Data
|
||||
// overlay, plus any Interactsh URLs allocated while rendering Source values.
|
||||
type MapResult struct {
|
||||
Values Values
|
||||
InteractURLs []string
|
||||
}
|
||||
|
||||
// Render evaluates template text with values and returns rendered data.
|
||||
//
|
||||
// Interactsh markers in Text are source-level render effects and are allocated
|
||||
// before DSL helper evaluation, so helper transforms apply to generated URLs.
|
||||
// Values are data and rendered output is terminal data; neither is rescanned for
|
||||
// DSL expressions or Interactsh markers.
|
||||
func Render(input Input) (Result, error) {
|
||||
prepared, err := ReplaceInteractshMarkers(input.Text, input.Interactsh, input.InteractURLs)
|
||||
if err != nil {
|
||||
return prepared, err
|
||||
}
|
||||
|
||||
result, err := expressions.Evaluate(prepared.Text, input.Values)
|
||||
if err != nil {
|
||||
return prepared, err
|
||||
}
|
||||
|
||||
return Result{Text: result, InteractURLs: prepared.InteractURLs}, nil
|
||||
}
|
||||
|
||||
// RenderMap renders Source values as template text, overlays Data values, and
|
||||
// returns a complete value map. Data keys are skipped while rendering so they
|
||||
// cannot produce DSL or Interactsh side effects.
|
||||
//
|
||||
// It is intended for already-expanded payload iteration maps, not raw payload
|
||||
// definitions containing lists.
|
||||
func RenderMap(input MapInput) (MapResult, error) {
|
||||
values := make(Values, len(input.Source)+len(input.Data))
|
||||
urls := append([]string(nil), input.InteractURLs...)
|
||||
|
||||
for key, value := range input.Source {
|
||||
if _, ok := input.Data[key]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
result, err := Render(Input{
|
||||
Text: fmt.Sprint(value),
|
||||
Values: input.Values,
|
||||
Interactsh: input.Interactsh,
|
||||
InteractURLs: urls,
|
||||
})
|
||||
if err != nil {
|
||||
return MapResult{Values: values, InteractURLs: result.InteractURLs}, err
|
||||
}
|
||||
|
||||
values[key] = result.Text
|
||||
urls = result.InteractURLs
|
||||
}
|
||||
|
||||
for key, value := range input.Data {
|
||||
values[key] = value
|
||||
}
|
||||
|
||||
return MapResult{Values: values, InteractURLs: urls}, nil
|
||||
}
|
||||
|
||||
// ReplaceInteractshMarkers allocates Interactsh URLs for markers that appear in
|
||||
// template text. It does not evaluate DSL expressions or inspect values.
|
||||
func ReplaceInteractshMarkers(text string, source URLSource, interactURLs []string) (Result, error) {
|
||||
urls := append([]string(nil), interactURLs...)
|
||||
if source == nil {
|
||||
return Result{Text: text, InteractURLs: urls}, nil
|
||||
}
|
||||
|
||||
for _, interactshURLMarker := range marker.FindInteractshURLMarkers(text) {
|
||||
url, err := source.NewURLWithData(interactshURLMarker)
|
||||
if err != nil {
|
||||
return Result{Text: text, InteractURLs: urls}, fmt.Errorf("replace interactsh marker %q: %w", interactshURLMarker, err)
|
||||
}
|
||||
|
||||
urls = append(urls, url)
|
||||
text = strings.Replace(text, interactshURLMarker, url, 1)
|
||||
}
|
||||
|
||||
return Result{Text: text, InteractURLs: urls}, nil
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
package render
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/govaluate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type testURLSource struct {
|
||||
urls []string
|
||||
data []string
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *testURLSource) NewURLWithData(data string) (string, error) {
|
||||
s.data = append(s.data, data)
|
||||
if s.err != nil {
|
||||
return "", s.err
|
||||
}
|
||||
url := "https://scan.example/a b"
|
||||
if len(s.urls) > 0 {
|
||||
url = s.urls[0]
|
||||
s.urls = s.urls[1:]
|
||||
}
|
||||
return url, nil
|
||||
}
|
||||
|
||||
func withRenderTestHelperFunction(t *testing.T, name string, fn govaluate.ExpressionFunction) {
|
||||
t.Helper()
|
||||
|
||||
originalFn, hadFn := dsl.HelperFunctions[name]
|
||||
dsl.HelperFunctions[name] = fn
|
||||
t.Cleanup(func() {
|
||||
if hadFn {
|
||||
dsl.HelperFunctions[name] = originalFn
|
||||
return
|
||||
}
|
||||
delete(dsl.HelperFunctions, name)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRenderEvaluatesTemplateTextHelpers(t *testing.T) {
|
||||
result, err := Render(Input{
|
||||
Text: "{{hex_encode(value)}}",
|
||||
Values: Values{"value": "PING"},
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "50494e47", result.Text)
|
||||
require.Empty(t, result.InteractURLs)
|
||||
}
|
||||
|
||||
func TestRenderTreatsValuesAsData(t *testing.T) {
|
||||
var calls int
|
||||
withRenderTestHelperFunction(t, "test_side_effect", func(args ...interface{}) (interface{}, error) {
|
||||
calls++
|
||||
return "executed", nil
|
||||
})
|
||||
|
||||
result, err := Render(Input{
|
||||
Text: "{{body}}",
|
||||
Values: Values{
|
||||
"body": "{{test_side_effect(1)}}",
|
||||
},
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "{{test_side_effect(1)}}", result.Text)
|
||||
require.Zero(t, calls)
|
||||
}
|
||||
|
||||
func TestRenderTreatsValuesInsideHelperStringsAsData(t *testing.T) {
|
||||
var calls int
|
||||
withRenderTestHelperFunction(t, "test_side_effect", func(args ...interface{}) (interface{}, error) {
|
||||
calls++
|
||||
return true, nil
|
||||
})
|
||||
|
||||
result, err := Render(Input{
|
||||
Text: "{{contains('{{body}}', 'needle')}}",
|
||||
Values: Values{
|
||||
"body": "value', 'missing') || test_side_effect() || contains('",
|
||||
},
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "false", result.Text)
|
||||
require.Zero(t, calls)
|
||||
}
|
||||
|
||||
func TestRenderDoesNotEvaluateRenderedOutput(t *testing.T) {
|
||||
result, err := Render(Input{
|
||||
Text: "{{body}}",
|
||||
Values: Values{
|
||||
"body": "{{secret}}",
|
||||
"secret": "leaked-secret",
|
||||
},
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "{{secret}}", result.Text)
|
||||
}
|
||||
|
||||
func TestRenderReplacesTemplateInteractshMarkers(t *testing.T) {
|
||||
source := &testURLSource{urls: []string{"https://scan.example/raw"}}
|
||||
|
||||
result, err := Render(Input{
|
||||
Text: "callback={{interactsh-url}}",
|
||||
Values: Values{},
|
||||
Interactsh: source,
|
||||
InteractURLs: []string{"https://existing.example"},
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "callback=https://scan.example/raw", result.Text)
|
||||
require.Equal(t, []string{"https://existing.example", "https://scan.example/raw"}, result.InteractURLs)
|
||||
require.Equal(t, []string{"{{interactsh-url}}"}, source.data)
|
||||
}
|
||||
|
||||
func TestRenderReplacesEncodedTemplateInteractshMarkers(t *testing.T) {
|
||||
source := &testURLSource{urls: []string{"https://scan.example/encoded"}}
|
||||
|
||||
result, err := Render(Input{
|
||||
Text: "callback=%7B%7Binteractsh-url%7D%7D",
|
||||
Values: Values{},
|
||||
Interactsh: source,
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "callback=https://scan.example/encoded", result.Text)
|
||||
require.Equal(t, []string{"https://scan.example/encoded"}, result.InteractURLs)
|
||||
require.Equal(t, []string{"%7B%7Binteractsh-url%7D%7D"}, source.data)
|
||||
}
|
||||
|
||||
func TestRenderRejectsFailedInteractshAllocation(t *testing.T) {
|
||||
source := &testURLSource{err: errors.New("allocation failed")}
|
||||
|
||||
result, err := Render(Input{
|
||||
Text: "callback={{interactsh-url}}",
|
||||
Values: Values{},
|
||||
Interactsh: source,
|
||||
InteractURLs: []string{"https://existing.example"},
|
||||
})
|
||||
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "replace interactsh marker")
|
||||
require.Equal(t, "callback={{interactsh-url}}", result.Text)
|
||||
require.Equal(t, []string{"https://existing.example"}, result.InteractURLs)
|
||||
require.Equal(t, []string{"{{interactsh-url}}"}, source.data)
|
||||
}
|
||||
|
||||
func TestRenderTransformsTemplateInteractshMarkersThroughHelpers(t *testing.T) {
|
||||
source := &testURLSource{urls: []string{"https://scan.example/a b"}}
|
||||
|
||||
result, err := Render(Input{
|
||||
Text: "{{url_encode('{{interactsh-url}}')}}",
|
||||
Values: Values{},
|
||||
Interactsh: source,
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "https%3A%2F%2Fscan.example%2Fa%20b", result.Text)
|
||||
require.Equal(t, []string{"https://scan.example/a b"}, result.InteractURLs)
|
||||
require.Equal(t, []string{"{{interactsh-url}}"}, source.data)
|
||||
}
|
||||
|
||||
func TestRenderDoesNotTreatPipeAsEncodedInteractshBrace(t *testing.T) {
|
||||
source := &testURLSource{urls: []string{"https://scan.example/pipe"}}
|
||||
|
||||
result, err := Render(Input{
|
||||
Text: "%7|%7|interactsh-url%7|%7|",
|
||||
Values: Values{},
|
||||
Interactsh: source,
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "%7|%7|interactsh-url%7|%7|", result.Text)
|
||||
require.Empty(t, result.InteractURLs)
|
||||
require.Empty(t, source.data)
|
||||
}
|
||||
|
||||
func TestRenderDoesNotTreatMixedRawEncodedBracesAsInteractshMarkers(t *testing.T) {
|
||||
items := []struct {
|
||||
name string
|
||||
text string
|
||||
}{
|
||||
{name: "mixed opening pair", text: "%7B{interactsh-url}}"},
|
||||
{name: "mixed closing pair", text: "{{interactsh-url}%7D"},
|
||||
{name: "encoded opening raw closing", text: "%7B%7Binteractsh-url}}"},
|
||||
{name: "raw opening encoded closing", text: "{{interactsh-url%7D%7D"},
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
t.Run(item.name, func(t *testing.T) {
|
||||
source := &testURLSource{urls: []string{"https://scan.example/mixed"}}
|
||||
|
||||
result, err := Render(Input{
|
||||
Text: item.text,
|
||||
Values: Values{},
|
||||
Interactsh: source,
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, item.text, result.Text)
|
||||
require.Empty(t, result.InteractURLs)
|
||||
require.Empty(t, source.data)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderDoesNotReplaceInteractshMarkersFromValues(t *testing.T) {
|
||||
items := []struct {
|
||||
name string
|
||||
value string
|
||||
}{
|
||||
{name: "raw", value: "{{interactsh-url}}"},
|
||||
{name: "encoded", value: "%7B%7Binteractsh-url%7D%7D"},
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
t.Run(item.name, func(t *testing.T) {
|
||||
source := &testURLSource{urls: []string{"https://scan.example/value"}}
|
||||
|
||||
result, err := Render(Input{
|
||||
Text: "{{body}}",
|
||||
Values: Values{"body": item.value},
|
||||
Interactsh: source,
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, item.value, result.Text)
|
||||
require.Empty(t, result.InteractURLs)
|
||||
require.Empty(t, source.data)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderMapRendersSourceValuesOnce(t *testing.T) {
|
||||
source := &testURLSource{urls: []string{"https://scan.example/payload"}}
|
||||
|
||||
result, err := RenderMap(MapInput{
|
||||
Source: Values{
|
||||
"payload": "{{url_encode('{{interactsh-url}}')}}",
|
||||
},
|
||||
Values: Values{},
|
||||
Interactsh: source,
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "https%3A%2F%2Fscan.example%2Fpayload", result.Values["payload"])
|
||||
require.Equal(t, []string{"https://scan.example/payload"}, result.InteractURLs)
|
||||
require.Equal(t, []string{"{{interactsh-url}}"}, source.data)
|
||||
}
|
||||
|
||||
func TestRenderMapTreatsValuesAsData(t *testing.T) {
|
||||
source := &testURLSource{urls: []string{"https://scan.example/runtime"}}
|
||||
|
||||
result, err := RenderMap(MapInput{
|
||||
Source: Values{
|
||||
"payload": "{{server_value}}",
|
||||
},
|
||||
Data: Values{
|
||||
"runtime": "kept",
|
||||
},
|
||||
Values: Values{
|
||||
"server_value": "{{interactsh-url}}",
|
||||
"runtime": "kept",
|
||||
},
|
||||
Interactsh: source,
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "{{interactsh-url}}", result.Values["payload"])
|
||||
require.Equal(t, "kept", result.Values["runtime"])
|
||||
require.Empty(t, result.InteractURLs)
|
||||
require.Empty(t, source.data)
|
||||
}
|
||||
|
||||
func TestRenderMapPreservesAccumulatedValuesOnError(t *testing.T) {
|
||||
calls := 0
|
||||
withRenderTestHelperFunction(t, "test_render_map_error", func(args ...interface{}) (interface{}, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return "rendered", nil
|
||||
}
|
||||
return nil, errors.New("render failed")
|
||||
})
|
||||
|
||||
result, err := RenderMap(MapInput{
|
||||
Source: Values{
|
||||
"first": "{{test_render_map_error()}}",
|
||||
"second": "{{test_render_map_error()}}",
|
||||
},
|
||||
Values: Values{},
|
||||
})
|
||||
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "render failed")
|
||||
require.Len(t, result.Values, 1)
|
||||
for _, value := range result.Values {
|
||||
require.Equal(t, "rendered", value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderMapOverlaysDataWithoutRenderingIt(t *testing.T) {
|
||||
source := &testURLSource{urls: []string{"https://scan.example/shadowed"}}
|
||||
|
||||
result, err := RenderMap(MapInput{
|
||||
Source: Values{
|
||||
"payload": "{{interactsh-url}}",
|
||||
},
|
||||
Data: Values{
|
||||
"payload": "runtime-value",
|
||||
},
|
||||
Values: Values{
|
||||
"payload": "runtime-value",
|
||||
},
|
||||
Interactsh: source,
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, Values{"payload": "runtime-value"}, result.Values)
|
||||
require.Empty(t, result.InteractURLs)
|
||||
require.Empty(t, source.data)
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package variables
|
||||
|
||||
// Scope contains values used while evaluating template variables.
|
||||
//
|
||||
// Data values are terminal values from runtime, options, constants, payloads,
|
||||
// or protocol context. Template values are produced from the template variables
|
||||
// section and may be re-rendered when later data becomes available.
|
||||
type Scope struct {
|
||||
values map[string]scopeValue
|
||||
}
|
||||
|
||||
type scopeValueKind uint8
|
||||
|
||||
const (
|
||||
scopeValueData scopeValueKind = iota
|
||||
scopeValueTemplate
|
||||
)
|
||||
|
||||
type scopeValue struct {
|
||||
value interface{}
|
||||
kind scopeValueKind
|
||||
}
|
||||
|
||||
// NewScope creates an empty variable evaluation scope.
|
||||
func NewScope() *Scope {
|
||||
return &Scope{values: make(map[string]scopeValue)}
|
||||
}
|
||||
|
||||
// AddData adds terminal data values to the scope.
|
||||
func (s *Scope) AddData(values ...map[string]interface{}) *Scope {
|
||||
if s == nil {
|
||||
return s
|
||||
}
|
||||
|
||||
for _, valueMap := range values {
|
||||
for key, value := range valueMap {
|
||||
s.AddDataValue(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// AddDataValue adds one terminal data value to the scope.
|
||||
func (s *Scope) AddDataValue(key string, value interface{}) *Scope {
|
||||
if s == nil {
|
||||
return s
|
||||
}
|
||||
|
||||
s.values[key] = scopeValue{value: value, kind: scopeValueData}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// AddTemplate adds values produced from template variables.
|
||||
func (s *Scope) AddTemplate(values map[string]interface{}) *Scope {
|
||||
if s == nil {
|
||||
return s
|
||||
}
|
||||
|
||||
for key, value := range values {
|
||||
s.AddTemplateValue(key, value)
|
||||
}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// AddTemplateValue adds one value produced from template variables.
|
||||
func (s *Scope) AddTemplateValue(key string, value interface{}) *Scope {
|
||||
if s == nil {
|
||||
return s
|
||||
}
|
||||
|
||||
s.values[key] = scopeValue{value: value, kind: scopeValueTemplate}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// Values returns a plain value map for render evaluation.
|
||||
func (s *Scope) Values() map[string]interface{} {
|
||||
if s == nil || len(s.values) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
values := make(map[string]interface{}, len(s.values))
|
||||
for key, value := range s.values {
|
||||
values[key] = value.value
|
||||
}
|
||||
|
||||
return values
|
||||
}
|
||||
|
||||
func (s *Scope) clone() *Scope {
|
||||
if s == nil {
|
||||
return NewScope()
|
||||
}
|
||||
|
||||
clone := NewScope()
|
||||
for key, value := range s.values {
|
||||
clone.values[key] = value
|
||||
}
|
||||
|
||||
return clone
|
||||
}
|
||||
|
||||
func (s *Scope) get(key string) (scopeValue, bool) {
|
||||
if s == nil {
|
||||
return scopeValue{}, false
|
||||
}
|
||||
|
||||
value, ok := s.values[key]
|
||||
|
||||
return value, ok
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/marker"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
@@ -26,6 +27,14 @@ type Variable struct {
|
||||
utils.InsertionOrderedStringMap `yaml:"-" json:"-"`
|
||||
}
|
||||
|
||||
// Evaluation contains variable values and the subset produced from template
|
||||
// variable text.
|
||||
type Evaluation struct {
|
||||
Values map[string]interface{}
|
||||
TemplateValues map[string]interface{}
|
||||
InteractURLs []string
|
||||
}
|
||||
|
||||
func (variables Variable) JSONSchema() *jsonschema.Schema {
|
||||
gotType := &jsonschema.Schema{
|
||||
Type: "object",
|
||||
@@ -69,26 +78,41 @@ func (variables *Variable) UnmarshalJSON(data []byte) error {
|
||||
|
||||
// Evaluate returns a finished map of variables based on set values
|
||||
func (variables *Variable) Evaluate(values map[string]interface{}) map[string]interface{} {
|
||||
return variables.EvaluateScope(NewScope().AddData(values)).Values
|
||||
}
|
||||
|
||||
// EvaluateScope returns evaluated variables and records which returned values
|
||||
// were produced from template variable text.
|
||||
func (variables *Variable) EvaluateScope(scope *Scope) Evaluation {
|
||||
result := make(map[string]interface{}, variables.Len())
|
||||
combined := make(map[string]interface{}, len(values)+variables.Len())
|
||||
generators.MergeMapsInto(combined, values)
|
||||
templateValues := make(map[string]interface{}, variables.Len())
|
||||
combined := scope.clone()
|
||||
|
||||
variables.ForEach(func(key string, value interface{}) {
|
||||
if existingValue, ok := combined.get(key); ok && existingValue.kind == scopeValueData {
|
||||
result[key] = existingValue.value
|
||||
combined.AddDataValue(key, existingValue.value)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if sliceValue, ok := value.([]interface{}); ok {
|
||||
// slices cannot be evaluated
|
||||
result[key] = sliceValue
|
||||
combined[key] = sliceValue
|
||||
templateValues[key] = sliceValue
|
||||
combined.AddTemplateValue(key, sliceValue)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
valueString := types.ToString(value)
|
||||
if existingValue, ok := combined[key]; ok {
|
||||
valueString = types.ToString(existingValue)
|
||||
}
|
||||
evaluated := evaluateVariableValueWithMap(valueString, combined)
|
||||
evaluated := evaluateVariableValueWithMap(valueString, combined.Values())
|
||||
result[key] = evaluated
|
||||
combined[key] = evaluated
|
||||
templateValues[key] = evaluated
|
||||
combined.AddTemplateValue(key, evaluated)
|
||||
})
|
||||
return result
|
||||
|
||||
return Evaluation{Values: result, TemplateValues: templateValues}
|
||||
}
|
||||
|
||||
// GetAll returns all variables as a map
|
||||
@@ -97,35 +121,52 @@ func (variables *Variable) GetAll() map[string]interface{} {
|
||||
variables.ForEach(func(key string, value interface{}) {
|
||||
result[key] = value
|
||||
})
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// EvaluateWithInteractsh returns evaluation results of variables with interactsh
|
||||
func (variables *Variable) EvaluateWithInteractsh(values map[string]interface{}, interact *interactsh.Client) (map[string]interface{}, []string) {
|
||||
evaluation := variables.EvaluateWithInteractshScope(NewScope().AddData(values), interact)
|
||||
return evaluation.Values, evaluation.InteractURLs
|
||||
}
|
||||
|
||||
// EvaluateWithInteractshScope returns evaluated variables, source-level
|
||||
// Interactsh URLs, and the subset of values produced from template variable
|
||||
// text.
|
||||
func (variables *Variable) EvaluateWithInteractshScope(scope *Scope, interact render.URLSource) Evaluation {
|
||||
result := make(map[string]interface{}, variables.Len())
|
||||
combined := make(map[string]interface{}, len(values)+variables.Len())
|
||||
generators.MergeMapsInto(combined, values)
|
||||
templateValues := make(map[string]interface{}, variables.Len())
|
||||
combined := scope.clone()
|
||||
|
||||
var interactURLs []string
|
||||
|
||||
variables.ForEach(func(key string, value interface{}) {
|
||||
if existingValue, ok := combined.get(key); ok && existingValue.kind == scopeValueData {
|
||||
result[key] = existingValue.value
|
||||
combined.AddDataValue(key, existingValue.value)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if sliceValue, ok := value.([]interface{}); ok {
|
||||
// slices cannot be evaluated
|
||||
result[key] = sliceValue
|
||||
combined[key] = sliceValue
|
||||
templateValues[key] = sliceValue
|
||||
combined.AddTemplateValue(key, sliceValue)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
valueString := types.ToString(value)
|
||||
if existingValue, ok := combined[key]; ok {
|
||||
valueString = types.ToString(existingValue)
|
||||
}
|
||||
if strings.Contains(valueString, "interactsh-url") {
|
||||
valueString, interactURLs = interact.Replace(valueString, interactURLs)
|
||||
}
|
||||
evaluated := evaluateVariableValueWithMap(valueString, combined)
|
||||
evaluated, gotURLs := renderVariableValueWithInteractsh(valueString, combined.Values(), interact, interactURLs)
|
||||
result[key] = evaluated
|
||||
combined[key] = evaluated
|
||||
templateValues[key] = evaluated
|
||||
combined.AddTemplateValue(key, evaluated)
|
||||
interactURLs = gotURLs
|
||||
})
|
||||
return result, interactURLs
|
||||
|
||||
return Evaluation{Values: result, TemplateValues: templateValues, InteractURLs: interactURLs}
|
||||
}
|
||||
|
||||
// evaluateVariableValue expression and returns final value.
|
||||
@@ -134,22 +175,42 @@ func (variables *Variable) EvaluateWithInteractsh(values map[string]interface{},
|
||||
// merging overhead.
|
||||
func evaluateVariableValue(expression string, values, processing map[string]interface{}) string { // nolint
|
||||
finalMap := generators.MergeMaps(values, processing)
|
||||
result, err := expressions.Evaluate(expression, finalMap)
|
||||
result, err := render.Render(render.Input{
|
||||
Text: expression,
|
||||
Values: finalMap,
|
||||
})
|
||||
if err != nil {
|
||||
return expression
|
||||
}
|
||||
|
||||
return result
|
||||
return result.Text
|
||||
}
|
||||
|
||||
// evaluateVariableValueWithMap evaluates an expression with a pre-merged map.
|
||||
func evaluateVariableValueWithMap(expression string, combinedMap map[string]interface{}) string {
|
||||
result, err := expressions.Evaluate(expression, combinedMap)
|
||||
result, err := render.Render(render.Input{
|
||||
Text: expression,
|
||||
Values: combinedMap,
|
||||
})
|
||||
if err != nil {
|
||||
return expression
|
||||
}
|
||||
|
||||
return result
|
||||
return result.Text
|
||||
}
|
||||
|
||||
func renderVariableValueWithInteractsh(expression string, combinedMap map[string]interface{}, interact render.URLSource, interactURLs []string) (string, []string) {
|
||||
result, err := render.Render(render.Input{
|
||||
Text: expression,
|
||||
Values: combinedMap,
|
||||
Interactsh: interact,
|
||||
InteractURLs: interactURLs,
|
||||
})
|
||||
if err != nil {
|
||||
return result.Text, result.InteractURLs
|
||||
}
|
||||
|
||||
return result.Text, result.InteractURLs
|
||||
}
|
||||
|
||||
// checkForLazyEval checks if the variables have any lazy evaluation i.e any dsl function
|
||||
|
||||
@@ -4,13 +4,37 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/govaluate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func withVariableTestHelperFunction(t *testing.T, name string, fn govaluate.ExpressionFunction) {
|
||||
t.Helper()
|
||||
|
||||
originalFn, hadFn := dsl.HelperFunctions[name]
|
||||
dsl.HelperFunctions[name] = fn
|
||||
t.Cleanup(func() {
|
||||
if hadFn {
|
||||
dsl.HelperFunctions[name] = originalFn
|
||||
return
|
||||
}
|
||||
delete(dsl.HelperFunctions, name)
|
||||
})
|
||||
}
|
||||
|
||||
type variableTestURLSource struct {
|
||||
calls int
|
||||
}
|
||||
|
||||
func (v *variableTestURLSource) NewURLWithData(string) (string, error) {
|
||||
v.calls++
|
||||
return "https://example.oast.fun", nil
|
||||
}
|
||||
|
||||
func TestVariablesEvaluate(t *testing.T) {
|
||||
data := `a2: "{{md5('test')}}"
|
||||
a3: "this_is_random_text"
|
||||
@@ -239,6 +263,204 @@ func TestVariablesEvaluateChained(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestVariablesEvaluateDynamicOverrideValuesAreData(t *testing.T) {
|
||||
var waitForCalls int
|
||||
withVariableTestHelperFunction(t, "wait_for", func(args ...interface{}) (interface{}, error) {
|
||||
waitForCalls++
|
||||
return true, nil
|
||||
})
|
||||
|
||||
items := []struct {
|
||||
name string
|
||||
value string
|
||||
values map[string]interface{}
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "environment marker remains data",
|
||||
value: "{{HOME}}",
|
||||
values: map[string]interface{}{"HOME": "/home/scanner"},
|
||||
expected: "{{HOME}}",
|
||||
},
|
||||
{
|
||||
name: "helper marker remains data",
|
||||
value: "{{wait_for(5)}}",
|
||||
values: map[string]interface{}{},
|
||||
expected: "{{wait_for(5)}}",
|
||||
},
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
t.Run(item.name, func(t *testing.T) {
|
||||
variables := &Variable{
|
||||
LazyEval: true,
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
|
||||
}
|
||||
variables.Set("token", "declared-token")
|
||||
|
||||
inputValues := map[string]interface{}{"token": item.value}
|
||||
for key, value := range item.values {
|
||||
inputValues[key] = value
|
||||
}
|
||||
|
||||
result := variables.Evaluate(inputValues)
|
||||
require.Equal(t, item.expected, result["token"])
|
||||
})
|
||||
}
|
||||
|
||||
require.Zero(t, waitForCalls, "dynamic override helpers must not execute")
|
||||
}
|
||||
|
||||
func TestVariablesEvaluateMatchingDynamicHelperOverrideIsData(t *testing.T) {
|
||||
var waitForCalls int
|
||||
withVariableTestHelperFunction(t, "wait_for", func(args ...interface{}) (interface{}, error) {
|
||||
waitForCalls++
|
||||
return true, nil
|
||||
})
|
||||
|
||||
variables := &Variable{
|
||||
LazyEval: true,
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
|
||||
}
|
||||
variables.Set("token", "{{wait_for(5)}}")
|
||||
|
||||
result := variables.Evaluate(map[string]interface{}{
|
||||
"token": "{{wait_for(5)}}",
|
||||
})
|
||||
|
||||
require.Equal(t, "{{wait_for(5)}}", result["token"])
|
||||
require.Zero(t, waitForCalls, "matching dynamic override helpers must not execute")
|
||||
}
|
||||
|
||||
func TestVariablesEvaluateDataValueInsertedIntoVariableIsData(t *testing.T) {
|
||||
var waitForCalls int
|
||||
withVariableTestHelperFunction(t, "wait_for", func(args ...interface{}) (interface{}, error) {
|
||||
waitForCalls++
|
||||
return true, nil
|
||||
})
|
||||
|
||||
variables := &Variable{
|
||||
LazyEval: true,
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
|
||||
}
|
||||
variables.Set("header", "Bearer {{extracted_token}}")
|
||||
|
||||
result := variables.EvaluateScope(NewScope().AddData(map[string]interface{}{
|
||||
"extracted_token": "{{wait_for(5)}}",
|
||||
})).Values
|
||||
|
||||
require.Equal(t, "Bearer {{wait_for(5)}}", result["header"])
|
||||
require.Zero(t, waitForCalls, "data inserted into template variables must not execute")
|
||||
}
|
||||
|
||||
func TestVariablesEvaluateReevaluatesStoredDeclaredValue(t *testing.T) {
|
||||
variables := &Variable{
|
||||
LazyEval: true,
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
|
||||
}
|
||||
variables.Set("auth_header", "Bearer {{extracted_token}}")
|
||||
|
||||
result := variables.EvaluateScope(NewScope().AddTemplate(map[string]interface{}{
|
||||
"auth_header": "Bearer {{extracted_token}}",
|
||||
}).AddData(map[string]interface{}{
|
||||
"extracted_token": "secret123",
|
||||
})).Values
|
||||
|
||||
require.Equal(t, "Bearer secret123", result["auth_header"])
|
||||
}
|
||||
|
||||
func TestVariablesEvaluateReevaluatesStoredDeclaredHelperValue(t *testing.T) {
|
||||
variables := &Variable{
|
||||
LazyEval: true,
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
|
||||
}
|
||||
variables.Set("cname_filtered", `{{trim_suffix(dns_cname,".vercel-dns.com")}}`)
|
||||
|
||||
result := variables.EvaluateScope(NewScope().AddTemplate(map[string]interface{}{
|
||||
"cname_filtered": `{{trim_suffix(dns_cname,".vercel-dns.com")}}`,
|
||||
}).AddData(map[string]interface{}{
|
||||
"dns_cname": "cname.vercel-dns.com",
|
||||
})).Values
|
||||
|
||||
require.Equal(t, "cname", result["cname_filtered"])
|
||||
}
|
||||
|
||||
func TestVariablesEvaluateWithInteractshMatchingRuntimeOverrideIsData(t *testing.T) {
|
||||
variables := &Variable{
|
||||
LazyEval: true,
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
|
||||
}
|
||||
variables.Set("callback", "{{interactsh-url}}")
|
||||
|
||||
source := &variableTestURLSource{}
|
||||
evaluation := variables.EvaluateWithInteractshScope(NewScope().AddData(map[string]interface{}{
|
||||
"callback": "{{interactsh-url}}",
|
||||
}), source)
|
||||
result, urls := evaluation.Values, evaluation.InteractURLs
|
||||
|
||||
require.Empty(t, urls)
|
||||
require.Zero(t, source.calls, "matching runtime override marker must not allocate interactsh URLs")
|
||||
require.Equal(t, "{{interactsh-url}}", result["callback"])
|
||||
}
|
||||
|
||||
func TestVariablesEvaluateWithInteractshDynamicOverrideIsData(t *testing.T) {
|
||||
variables := &Variable{
|
||||
LazyEval: true,
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
|
||||
}
|
||||
variables.Set("callback", "https://declared.{{interactsh-url}}/path")
|
||||
|
||||
result, urls := variables.EvaluateWithInteractsh(map[string]interface{}{
|
||||
"callback": "https://dynamic.{{interactsh-url}}/path",
|
||||
}, nil)
|
||||
|
||||
require.Empty(t, urls)
|
||||
require.Equal(t, "https://dynamic.{{interactsh-url}}/path", result["callback"])
|
||||
}
|
||||
|
||||
func TestRenderVariableValueWithInteractshKeepsURLsOnRenderError(t *testing.T) {
|
||||
source := &variableTestURLSource{}
|
||||
|
||||
result, urls := renderVariableValueWithInteractsh("{{interactsh-url}} {{md5(missing)}}", nil, source, nil)
|
||||
|
||||
require.Equal(t, 1, source.calls)
|
||||
require.Len(t, urls, 1)
|
||||
require.Contains(t, result, "https://example.oast.fun")
|
||||
require.NotContains(t, result, "{{interactsh-url}}")
|
||||
}
|
||||
|
||||
func TestVariablesEvaluatePreservesDynamicShadowingCompatibility(t *testing.T) {
|
||||
t.Run("login-token-replaces-declared-empty-token", func(t *testing.T) {
|
||||
variables := &Variable{
|
||||
LazyEval: true,
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
|
||||
}
|
||||
variables.Set("token", "")
|
||||
|
||||
result := variables.Evaluate(map[string]interface{}{
|
||||
"token": "server-issued-token",
|
||||
})
|
||||
|
||||
require.Equal(t, "server-issued-token", result["token"])
|
||||
})
|
||||
|
||||
t.Run("edited-filename-replaces-initial-generated-filename", func(t *testing.T) {
|
||||
variables := &Variable{
|
||||
LazyEval: true,
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
|
||||
}
|
||||
variables.Set("image_filename", "initial-upload-name")
|
||||
|
||||
initial := variables.Evaluate(map[string]interface{}{})
|
||||
require.Equal(t, "initial-upload-name", initial["image_filename"])
|
||||
|
||||
afterExtraction := variables.Evaluate(map[string]interface{}{
|
||||
"image_filename": "server-edited-name-e1",
|
||||
})
|
||||
require.Equal(t, "server-edited-name-e1", afterExtraction["image_filename"])
|
||||
})
|
||||
}
|
||||
|
||||
func TestEvaluateWithInteractshOverrideOrder(t *testing.T) {
|
||||
// This test demonstrates a bug where interactsh URL replacement is wasted
|
||||
// when an input value exists for the same variable key.
|
||||
@@ -252,40 +474,28 @@ func TestEvaluateWithInteractshOverrideOrder(t *testing.T) {
|
||||
// Expected behavior: Input override should be checked FIRST, then interactsh
|
||||
// replacement should happen on the final valueString.
|
||||
|
||||
t.Run("interactsh-replacement-with-input-override", func(t *testing.T) {
|
||||
t.Run("input-override-is-data", func(t *testing.T) {
|
||||
variables := &Variable{
|
||||
LazyEval: true,
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
|
||||
}
|
||||
variables.Set("callback", "{{interactsh-url}}")
|
||||
|
||||
// Input provides an override that also contains interactsh-url
|
||||
// Input provides an override that also contains interactsh-url.
|
||||
// Overrides are dynamic data, so the marker must not allocate an
|
||||
// interactsh URL or get evaluated as template source.
|
||||
inputValues := map[string]interface{}{
|
||||
"callback": "https://custom.{{interactsh-url}}/path",
|
||||
}
|
||||
|
||||
// Create a real interactsh client for testing
|
||||
client, err := interactsh.New(&interactsh.Options{
|
||||
ServerURL: "oast.fun",
|
||||
CacheSize: 100,
|
||||
Eviction: 60 * time.Second,
|
||||
CooldownPeriod: 5 * time.Second,
|
||||
PollDuration: 5 * time.Second,
|
||||
DisableHttpFallback: true,
|
||||
})
|
||||
require.NoError(t, err, "could not create interactsh client")
|
||||
defer client.Close()
|
||||
source := &variableTestURLSource{}
|
||||
|
||||
result, urls := variables.EvaluateWithInteractsh(inputValues, client)
|
||||
evaluation := variables.EvaluateWithInteractshScope(NewScope().AddData(inputValues), source)
|
||||
result, urls := evaluation.Values, evaluation.InteractURLs
|
||||
|
||||
// The input override contains interactsh-url, so it should be replaced
|
||||
// and we should have exactly 1 URL from the input override
|
||||
require.Len(t, urls, 1, "should have 1 interactsh URL from input override")
|
||||
|
||||
// The result should use the input override (with interactsh replaced)
|
||||
require.Contains(t, result["callback"], "https://custom.", "should use input override pattern")
|
||||
require.Contains(t, result["callback"], "/path", "should use input override pattern")
|
||||
require.NotContains(t, result["callback"], "{{interactsh-url}}", "interactsh should be replaced")
|
||||
require.Empty(t, urls, "dynamic override marker should not allocate interactsh URLs")
|
||||
require.Zero(t, source.calls, "dynamic override marker should not allocate interactsh URLs")
|
||||
require.Equal(t, "https://custom.{{interactsh-url}}/path", result["callback"])
|
||||
})
|
||||
|
||||
t.Run("interactsh-replacement-without-input-override", func(t *testing.T) {
|
||||
@@ -300,21 +510,14 @@ func TestEvaluateWithInteractshOverrideOrder(t *testing.T) {
|
||||
"other_key": "other_value",
|
||||
}
|
||||
|
||||
client, err := interactsh.New(&interactsh.Options{
|
||||
ServerURL: "oast.fun",
|
||||
CacheSize: 100,
|
||||
Eviction: 60 * time.Second,
|
||||
CooldownPeriod: 5 * time.Second,
|
||||
PollDuration: 5 * time.Second,
|
||||
DisableHttpFallback: true,
|
||||
})
|
||||
require.NoError(t, err, "could not create interactsh client")
|
||||
defer client.Close()
|
||||
source := &variableTestURLSource{}
|
||||
|
||||
result, urls := variables.EvaluateWithInteractsh(inputValues, client)
|
||||
evaluation := variables.EvaluateWithInteractshScope(NewScope().AddData(inputValues), source)
|
||||
result, urls := evaluation.Values, evaluation.InteractURLs
|
||||
|
||||
// Should have 1 URL from the variable definition
|
||||
require.Len(t, urls, 1, "should have 1 interactsh URL")
|
||||
require.Equal(t, 1, source.calls, "should allocate from template variable source")
|
||||
|
||||
// The result should be the replaced interactsh URL
|
||||
require.NotContains(t, result["callback"], "{{interactsh-url}}", "interactsh should be replaced")
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool"
|
||||
"github.com/projectdiscovery/retryabledns"
|
||||
@@ -204,11 +205,11 @@ func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadat
|
||||
// Defer resolution to the per-request runtime path.
|
||||
continue
|
||||
}
|
||||
evaluated, err := expressions.Evaluate(resolver, metadata)
|
||||
result, err := render.Render(render.Input{Text: resolver, Values: metadata})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not resolve resolvers expressions")
|
||||
}
|
||||
resolver = evaluated
|
||||
resolver = result.Text
|
||||
}
|
||||
resolvers = append(resolvers, resolver)
|
||||
}
|
||||
|
||||
@@ -45,11 +45,15 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
|
||||
vars := protocolutils.GenerateDNSVariables(domain)
|
||||
// optionvars are vars passed from CLI or env variables
|
||||
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
|
||||
// merge with metadata (eg. from workflow context)
|
||||
scope := request.options.NewVariablesScope(vars, metadata, optionVars)
|
||||
request.options.AddTemplateCtxToVariablesScope(input.MetaInput, scope)
|
||||
scope.AddData(request.options.Constants)
|
||||
variablesMap := request.options.Variables.EvaluateScope(scope).Values
|
||||
if request.options.HasTemplateCtx(input.MetaInput) {
|
||||
vars = generators.MergeMaps(vars, metadata, optionVars, request.options.GetTemplateCtx(input.MetaInput).GetAll())
|
||||
} else {
|
||||
vars = generators.MergeMaps(vars, metadata, optionVars)
|
||||
}
|
||||
variablesMap := request.options.Variables.Evaluate(vars)
|
||||
vars = generators.MergeMaps(vars, variablesMap, request.options.Constants)
|
||||
|
||||
// if request threads matches global payload concurrency we follow it
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
filepathutil "github.com/projectdiscovery/nuclei/v3/pkg/utils/filepath"
|
||||
contextutil "github.com/projectdiscovery/utils/context"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
@@ -976,11 +977,11 @@ func (p *Page) getActionArg(action *Action, arg string) (string, error) {
|
||||
|
||||
argValue := action.GetArg(arg)
|
||||
|
||||
if p.instance.interactsh != nil {
|
||||
var interactshURLs []string
|
||||
argValue, interactshURLs = p.instance.interactsh.Replace(argValue, p.InteractshURLs)
|
||||
p.addInteractshURL(interactshURLs...)
|
||||
prepared, err := render.ReplaceInteractshMarkers(argValue, p.instance.interactsh, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("could not replace interactsh marker for argument %q: %w", arg, err)
|
||||
}
|
||||
argValue = prepared.Text
|
||||
|
||||
exprs := getExpressions(argValue, p.variables)
|
||||
|
||||
@@ -989,10 +990,15 @@ func (p *Page) getActionArg(action *Action, arg string) (string, error) {
|
||||
return "", errkit.Wrapf(err, "argument %q, value: %q", arg, argValue)
|
||||
}
|
||||
|
||||
argValue, err = expressions.Evaluate(argValue, p.variables)
|
||||
result, err := render.Render(render.Input{
|
||||
Text: argValue,
|
||||
Values: p.variables,
|
||||
InteractURLs: prepared.InteractURLs,
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("could not get value for argument %q: %s", arg, err)
|
||||
}
|
||||
p.addInteractshURL(result.InteractURLs...)
|
||||
|
||||
return argValue, nil
|
||||
return result.Text, nil
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -21,12 +22,58 @@ import (
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testheadless"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
envutil "github.com/projectdiscovery/utils/env"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
)
|
||||
|
||||
func TestGetActionArgTreatsResolvedValuesAsData(t *testing.T) {
|
||||
page := &Page{
|
||||
instance: &Instance{},
|
||||
mutex: &sync.RWMutex{},
|
||||
variables: map[string]interface{}{
|
||||
"body": "{{secret}}",
|
||||
"secret": "leaked-secret",
|
||||
},
|
||||
}
|
||||
|
||||
got, err := page.getActionArg(&Action{Data: map[string]string{"value": "{{body}}"}}, "value")
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "{{secret}}", got)
|
||||
require.Empty(t, page.InteractshURLs)
|
||||
}
|
||||
|
||||
func TestGetActionArgRendersTemplateInteractshBeforeValidation(t *testing.T) {
|
||||
client, err := interactsh.New(&interactsh.Options{
|
||||
ServerURL: "oast.fun",
|
||||
CacheSize: 100,
|
||||
Eviction: 60 * time.Second,
|
||||
CooldownPeriod: 0,
|
||||
PollDuration: 5 * time.Second,
|
||||
DisableHttpFallback: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer client.Close()
|
||||
|
||||
page := &Page{
|
||||
instance: &Instance{interactsh: client},
|
||||
mutex: &sync.RWMutex{},
|
||||
variables: map[string]interface{}{},
|
||||
}
|
||||
|
||||
got, err := page.getActionArg(&Action{Data: map[string]string{
|
||||
"value": "{{url_encode('{{interactsh-url}}')}}",
|
||||
}}, "value")
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, page.InteractshURLs, 1)
|
||||
require.NotContains(t, got, "{{interactsh-url}}")
|
||||
require.NotContains(t, got, "%7B%7Binteractsh-url%7D%7D")
|
||||
}
|
||||
|
||||
func TestActionNavigate(t *testing.T) {
|
||||
response := `
|
||||
<html>
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
|
||||
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
@@ -52,12 +53,18 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
|
||||
|
||||
vars := protocolutils.GenerateVariablesWithContextArgs(input, false)
|
||||
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
|
||||
scope := request.options.NewVariablesScope(vars)
|
||||
// add templatecontext variables to varMap
|
||||
if request.options.HasTemplateCtx(input.MetaInput) {
|
||||
request.options.AddTemplateCtxToVariablesScope(input.MetaInput, scope)
|
||||
}
|
||||
|
||||
scope.AddData(metadata, optionVars, request.options.Constants)
|
||||
evaluation := request.options.Variables.EvaluateWithInteractshScope(scope, request.options.Interactsh)
|
||||
variablesMap, interactshURLs := evaluation.Values, evaluation.InteractURLs
|
||||
if request.options.HasTemplateCtx(input.MetaInput) {
|
||||
vars = generators.MergeMaps(vars, request.options.GetTemplateCtx(input.MetaInput).GetAll())
|
||||
}
|
||||
|
||||
variablesMap := request.options.Variables.Evaluate(vars)
|
||||
vars = generators.MergeMaps(vars, metadata, optionVars, variablesMap, request.options.Constants)
|
||||
|
||||
// check for operator matches by wrapping callback
|
||||
@@ -79,20 +86,33 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
||||
if gotmatches && (request.StopAtFirstMatch || request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch) {
|
||||
return nil
|
||||
}
|
||||
value = generators.MergeMaps(value, vars)
|
||||
if err := request.executeRequestWithPayloads(input, value, previous, wrappedCallback); err != nil {
|
||||
|
||||
renderedValue, err := render.RenderMap(render.MapInput{
|
||||
Source: value,
|
||||
Data: vars,
|
||||
Values: generators.MergeMaps(value, vars),
|
||||
Interactsh: request.options.Interactsh,
|
||||
InteractURLs: interactshURLs,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not evaluate payload helper expressions")
|
||||
}
|
||||
|
||||
if err := request.executeRequestWithPayloads(input, renderedValue.Values, previous, renderedValue.InteractURLs, wrappedCallback); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
value := maps.Clone(vars)
|
||||
if err := request.executeRequestWithPayloads(input, value, previous, wrappedCallback); err != nil {
|
||||
if err := request.executeRequestWithPayloads(input, value, previous, interactshURLs, wrappedCallback); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -111,7 +131,7 @@ func extractBaseURLFromActions(steps []*engine.Action) (string, error) {
|
||||
return "", errors.New("no navigation action found")
|
||||
}
|
||||
|
||||
func (request *Request) executeRequestWithPayloads(input *contextargs.Context, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
func (request *Request) executeRequestWithPayloads(input *contextargs.Context, payloads map[string]interface{}, previous output.InternalEvent, interactshURLs []string, callback protocols.OutputEventCallback) error {
|
||||
instance, err := request.options.Browser.NewInstance()
|
||||
if err != nil {
|
||||
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
|
||||
@@ -147,6 +167,8 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
|
||||
}
|
||||
defer page.Close()
|
||||
|
||||
page.InteractshURLs = append(interactshURLs, page.InteractshURLs...)
|
||||
|
||||
reqLog := instance.GetRequestLog()
|
||||
navigatedURL := request.getLastNavigationURLWithLog(reqLog) // also known as matchedURL if there is a match
|
||||
|
||||
@@ -245,19 +267,23 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads
|
||||
}
|
||||
newInput := input.Clone()
|
||||
newInput.MetaInput.Input = gr.Request.String()
|
||||
if err := request.executeRequestWithPayloads(newInput, gr.DynamicValues, previous, callback); err != nil {
|
||||
|
||||
if err := request.executeRequestWithPayloads(newInput, gr.DynamicValues, previous, gr.InteractURLs, callback); err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
if _, err := urlutil.Parse(input.MetaInput.Input); err != nil {
|
||||
return errors.Wrap(err, "could not parse url")
|
||||
}
|
||||
|
||||
baseRequest, err := retryablehttp.NewRequest("GET", input.MetaInput.Input, nil)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not create base request")
|
||||
}
|
||||
|
||||
for _, rule := range request.Fuzzing {
|
||||
err := rule.Execute(&fuzz.ExecuteRuleInput{
|
||||
Input: input,
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/race"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/raw"
|
||||
@@ -154,6 +155,7 @@ func (g *generatedRequest) URL() string {
|
||||
// It returns ErrNoMoreRequests as error when all the requests have been exhausted.
|
||||
func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, reqData string, payloads, dynamicValues map[string]interface{}) (gr *generatedRequest, err error) {
|
||||
origReqData := reqData
|
||||
r.interactshURLs = nil
|
||||
defer func() {
|
||||
if gr != nil {
|
||||
gr.setReqURLPattern(origReqData)
|
||||
@@ -163,28 +165,13 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
|
||||
// 1. If request is raw request = reqData contains raw request (i.e http request dump)
|
||||
// 2. If request is Normal ( simply put not a raw request) (Ex: with placeholders `path`) = reqData contains relative path
|
||||
|
||||
// add template context values to dynamicValues (this takes care of self-contained and other types of requests)
|
||||
// Note: `iterate-all` and flow are mutually exclusive. flow uses templateCtx and iterate-all uses dynamicValues
|
||||
if r.request.options.HasTemplateCtx(input.MetaInput) {
|
||||
// skip creating template context if not available
|
||||
dynamicValues = generators.MergeMaps(dynamicValues, r.request.options.GetTemplateCtx(input.MetaInput).GetAll())
|
||||
}
|
||||
|
||||
isRawRequest := len(r.request.Raw) > 0
|
||||
// replace interactsh variables with actual interactsh urls
|
||||
if r.options.Interactsh != nil {
|
||||
reqData, r.interactshURLs = r.options.Interactsh.Replace(reqData, []string{})
|
||||
for payloadName, payloadValue := range payloads {
|
||||
payloads[payloadName], r.interactshURLs = r.options.Interactsh.Replace(types.ToString(payloadValue), r.interactshURLs)
|
||||
}
|
||||
} else {
|
||||
for payloadName, payloadValue := range payloads {
|
||||
payloads[payloadName] = types.ToStringNSlice(payloadValue)
|
||||
}
|
||||
for payloadName, payloadValue := range payloads {
|
||||
payloads[payloadName] = types.ToStringNSlice(payloadValue)
|
||||
}
|
||||
|
||||
if r.request.SelfContained {
|
||||
return r.makeSelfContainedRequest(ctx, reqData, payloads, dynamicValues)
|
||||
return r.makeSelfContainedRequest(ctx, input, reqData, payloads, dynamicValues)
|
||||
}
|
||||
|
||||
// Parse target url
|
||||
@@ -208,17 +195,29 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
|
||||
// optionvars are vars passed from CLI or env variables
|
||||
optionVars := generators.BuildPayloadFromOptions(r.request.options.Options)
|
||||
|
||||
variablesMap, interactURLs := r.options.Variables.EvaluateWithInteractsh(generators.MergeMaps(dynamicValues, defaultReqVars, optionVars), r.options.Interactsh)
|
||||
scope := r.options.NewVariablesScope(dynamicValues)
|
||||
r.options.AddTemplateCtxToVariablesScope(input.MetaInput, scope)
|
||||
scope.AddData(defaultReqVars, optionVars, r.options.Constants)
|
||||
evaluation := r.options.Variables.EvaluateWithInteractshScope(scope, r.options.Interactsh)
|
||||
|
||||
variablesMap, interactURLs := evaluation.Values, evaluation.InteractURLs
|
||||
if len(interactURLs) > 0 {
|
||||
r.interactshURLs = append(r.interactshURLs, interactURLs...)
|
||||
}
|
||||
|
||||
// allVars contains all variables from all sources
|
||||
allVars := generators.MergeMaps(dynamicValues, defaultReqVars, optionVars, variablesMap, r.options.Constants)
|
||||
allVars := generators.MergeMaps(dynamicValues)
|
||||
|
||||
if r.request.options.HasTemplateCtx(input.MetaInput) {
|
||||
allVars = generators.MergeMaps(allVars, r.request.options.GetTemplateCtx(input.MetaInput).GetAll())
|
||||
}
|
||||
|
||||
allVars = generators.MergeMaps(allVars, defaultReqVars, optionVars, variablesMap, r.options.Constants)
|
||||
|
||||
// Evaluate payload variables
|
||||
// eg: payload variables can be username: jon.doe@{{Hostname}}
|
||||
for payloadName, payloadValue := range payloads {
|
||||
payloads[payloadName], err = expressions.Evaluate(types.ToString(payloadValue), allVars)
|
||||
payloads[payloadName], err = r.renderText(types.ToString(payloadValue), allVars)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "could not evaluate helper expressions")
|
||||
}
|
||||
@@ -235,7 +234,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
|
||||
// should be avoided since it is dependent on `urlutil` core logic
|
||||
|
||||
// Evaluate (replace) variable with final values
|
||||
reqData, err = expressions.Evaluate(reqData, finalVars)
|
||||
reqData, err = r.renderText(reqData, finalVars)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "could not evaluate helper expressions")
|
||||
}
|
||||
@@ -248,16 +247,18 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
|
||||
if err != nil {
|
||||
return nil, errkit.Newf("failed to parse url %v while creating http request", reqData)
|
||||
}
|
||||
|
||||
// while merging parameters first preference is given to target params
|
||||
finalparams := parsed.Params
|
||||
finalparams.Merge(reqURL.Params.Encode())
|
||||
reqURL.Params = finalparams
|
||||
|
||||
return r.generateHttpRequest(ctx, reqURL, finalVars, payloads)
|
||||
}
|
||||
|
||||
// selfContained templates do not need/use target data and all values i.e {{Hostname}} , {{BaseURL}} etc are already available
|
||||
// in template . makeSelfContainedRequest parses and creates variables map and then creates corresponding http request or raw request
|
||||
func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
|
||||
func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, input *contextargs.Context, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
|
||||
isRawRequest := r.request.isRaw()
|
||||
|
||||
values := generators.MergeMaps(
|
||||
@@ -266,21 +267,46 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
|
||||
payloads, // payloads should override other variables in case of duplicate vars
|
||||
)
|
||||
// adds all variables from `variables` section in template
|
||||
variablesMap := r.request.options.Variables.Evaluate(values)
|
||||
values = generators.MergeMaps(variablesMap, values)
|
||||
scope := r.request.options.NewVariablesScope(generators.BuildPayloadFromOptions(r.request.options.Options), dynamicValues, payloads)
|
||||
r.request.options.AddTemplateCtxToVariablesScope(input.MetaInput, scope)
|
||||
scope.AddData(r.options.Constants)
|
||||
evaluation := r.request.options.Variables.EvaluateWithInteractshScope(scope, r.request.options.Interactsh)
|
||||
|
||||
variablesMap, interactURLs := evaluation.Values, evaluation.InteractURLs
|
||||
if len(interactURLs) > 0 {
|
||||
r.interactshURLs = append(r.interactshURLs, interactURLs...)
|
||||
}
|
||||
|
||||
if r.request.options.HasTemplateCtx(input.MetaInput) {
|
||||
values = generators.MergeMaps(values, r.request.options.GetTemplateCtx(input.MetaInput).GetAll())
|
||||
}
|
||||
|
||||
values = generators.MergeMaps(variablesMap, values)
|
||||
signerVars := GetDefaultSignerVars(r.request.Signature.Value)
|
||||
|
||||
// this will ensure that default signer variables are overwritten by other variables
|
||||
values = generators.MergeMaps(signerVars, values, r.options.Constants)
|
||||
|
||||
for payloadName, payloadValue := range payloads {
|
||||
renderedPayload, err := r.renderText(types.ToString(payloadValue), values)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "could not evaluate payload helper expressions")
|
||||
}
|
||||
|
||||
payloads[payloadName] = renderedPayload
|
||||
}
|
||||
|
||||
values = generators.MergeMaps(values, payloads, r.options.Constants)
|
||||
|
||||
// priority of variables is as follows (from low to high) for self contained templates
|
||||
// default signer vars < variables < cli vars < payload < dynamic values < constants
|
||||
// default signer vars < variables < cli vars < dynamic values < payload < constants
|
||||
|
||||
// evaluate request
|
||||
data, err := expressions.Evaluate(data, values)
|
||||
data, err := r.renderText(data, values)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "could not evaluate helper expressions")
|
||||
}
|
||||
|
||||
// If the request is a raw request, get the URL from the request
|
||||
// header and use it to make the request.
|
||||
if isRawRequest {
|
||||
@@ -291,6 +317,7 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not read request: %w", err)
|
||||
}
|
||||
|
||||
// ignore all annotations
|
||||
if stringsutil.HasPrefixAny(s, "@") {
|
||||
goto read_line
|
||||
@@ -309,15 +336,18 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not parse request URL: %w", err)
|
||||
}
|
||||
|
||||
values = generators.MergeMaps(
|
||||
generators.MergeMaps(dynamicValues, protocolutils.GenerateVariables(parsed, false, nil)),
|
||||
values,
|
||||
)
|
||||
|
||||
// Evaluate (replace) variable with final values
|
||||
data, err = expressions.Evaluate(data, values)
|
||||
data, err = r.renderText(data, values)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "could not evaluate helper expressions")
|
||||
}
|
||||
|
||||
return r.generateRawRequest(ctx, data, parsed, values, payloads)
|
||||
}
|
||||
if err := expressions.ContainsUnresolvedVariables(data); err != nil && !r.request.SkipVariablesCheck {
|
||||
@@ -330,6 +360,7 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
|
||||
if err != nil {
|
||||
return nil, errkit.Wrapf(err, "failed to parse %v in self contained request", data)
|
||||
}
|
||||
|
||||
return r.generateHttpRequest(ctx, urlx, values, payloads)
|
||||
}
|
||||
|
||||
@@ -337,10 +368,11 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
|
||||
// finalVars = contains all variables including generator and protocol specific variables
|
||||
// generatorValues = contains variables used in fuzzing or other generator specific values
|
||||
func (r *requestGenerator) generateHttpRequest(ctx context.Context, urlx *urlutil.URL, finalVars, generatorValues map[string]interface{}) (*generatedRequest, error) {
|
||||
method, err := expressions.Evaluate(r.request.Method.String(), finalVars)
|
||||
method, err := r.renderText(r.request.Method.String(), finalVars)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "failed to evaluate while generating http request")
|
||||
}
|
||||
|
||||
// Build a request on the specified URL
|
||||
req, err := retryablehttp.NewRequestFromURLWithContext(ctx, method, urlx, nil)
|
||||
if err != nil {
|
||||
@@ -351,6 +383,7 @@ func (r *requestGenerator) generateHttpRequest(ctx context.Context, urlx *urluti
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalVars, interactshURLs: r.interactshURLs}, nil
|
||||
}
|
||||
|
||||
@@ -358,9 +391,9 @@ func (r *requestGenerator) generateHttpRequest(ctx context.Context, urlx *urluti
|
||||
// finalVars = contains all variables including generator and protocol specific variables
|
||||
// generatorValues = contains variables used in fuzzing or other generator specific values
|
||||
func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest string, baseURL *urlutil.URL, finalVars, generatorValues map[string]interface{}) (*generatedRequest, error) {
|
||||
|
||||
var rawRequestData *raw.Request
|
||||
var err error
|
||||
|
||||
if r.request.SelfContained {
|
||||
// in self contained requests baseURL is extracted from raw request itself
|
||||
rawRequestData, err = raw.ParseRawRequest(rawRequest, r.request.Unsafe)
|
||||
@@ -436,6 +469,7 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
|
||||
if r.request.Threads > 0 && r.request.Race {
|
||||
req.Body = race.NewOpenGateWithTimeout(req.Body, time.Duration(2)*time.Second)
|
||||
}
|
||||
|
||||
for key, value := range rawRequestData.Headers {
|
||||
if key == "" {
|
||||
continue
|
||||
@@ -445,6 +479,7 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
|
||||
req.Host = value
|
||||
}
|
||||
}
|
||||
|
||||
request, err := r.fillRequest(req, finalVars)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -471,13 +506,11 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
|
||||
func (r *requestGenerator) fillRequest(req *retryablehttp.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
|
||||
// Set the header values requested
|
||||
for header, value := range r.request.Headers {
|
||||
if r.options.Interactsh != nil {
|
||||
value, r.interactshURLs = r.options.Interactsh.Replace(value, r.interactshURLs)
|
||||
}
|
||||
value, err := expressions.Evaluate(value, values)
|
||||
value, err := r.renderText(value, values)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "failed to evaluate while adding headers to request")
|
||||
}
|
||||
|
||||
req.Header[header] = []string{value}
|
||||
if header == "Host" {
|
||||
req.Host = value
|
||||
@@ -508,18 +541,16 @@ func (r *requestGenerator) fillRequest(req *retryablehttp.Request, values map[st
|
||||
|
||||
// Check if the user requested a request body
|
||||
if r.request.Body != "" {
|
||||
body := r.request.Body
|
||||
if r.options.Interactsh != nil {
|
||||
body, r.interactshURLs = r.options.Interactsh.Replace(r.request.Body, r.interactshURLs)
|
||||
}
|
||||
body, err := expressions.Evaluate(body, values)
|
||||
body, err := r.renderText(r.request.Body, values)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "could not evaluate helper expressions")
|
||||
}
|
||||
|
||||
if err := req.SetBodyString(body); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to set request body")
|
||||
}
|
||||
}
|
||||
|
||||
if !r.request.Unsafe {
|
||||
userAgent := useragent.PickRandom()
|
||||
httputil.SetHeader(req, "User-Agent", userAgent.Raw)
|
||||
@@ -550,3 +581,19 @@ func (r *requestGenerator) fillRequest(req *retryablehttp.Request, values map[st
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func (r *requestGenerator) renderText(text string, values map[string]interface{}) (string, error) {
|
||||
result, err := render.Render(render.Input{
|
||||
Text: text,
|
||||
Values: values,
|
||||
Interactsh: r.options.Interactsh,
|
||||
InteractURLs: r.interactshURLs,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
r.interactshURLs = result.InteractURLs
|
||||
|
||||
return result.Text, nil
|
||||
}
|
||||
@@ -2,17 +2,20 @@ package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
)
|
||||
|
||||
@@ -240,6 +243,148 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) {
|
||||
require.True(t, areUnique(got.interactshURLs), "interactsh urls are not unique")
|
||||
}
|
||||
|
||||
func TestMakeSelfContainedRequestRendersPayloadInteractshMarker(t *testing.T) {
|
||||
options := testutils.DefaultOptions
|
||||
|
||||
testutils.Init(options)
|
||||
templateID := "testing-self-contained-payload-interactsh"
|
||||
request := &Request{
|
||||
ID: templateID,
|
||||
Name: "testing",
|
||||
SelfContained: true,
|
||||
Path: []string{"http://{{payload}}/callback"},
|
||||
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
|
||||
Payloads: map[string]interface{}{
|
||||
"payload": []string{"{{interactsh-url}}"},
|
||||
},
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: templateID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
client, err := interactsh.New(&interactsh.Options{
|
||||
ServerURL: options.InteractshURL,
|
||||
CacheSize: options.InteractionsCacheSize,
|
||||
Eviction: time.Duration(options.InteractionsEviction) * time.Second,
|
||||
CooldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second,
|
||||
PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second,
|
||||
DisableHttpFallback: true,
|
||||
})
|
||||
require.Nil(t, err, "could not create interactsh client")
|
||||
t.Cleanup(func() {
|
||||
client.Close()
|
||||
})
|
||||
executerOpts.Interactsh = client
|
||||
|
||||
err = request.Compile(executerOpts)
|
||||
require.Nil(t, err, "could not compile http request")
|
||||
|
||||
generator := request.newGenerator(false)
|
||||
inputData, payloads, _ := generator.nextValue()
|
||||
got, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), ""), inputData, payloads, map[string]interface{}{})
|
||||
require.Nil(t, err, "could not make self-contained http request")
|
||||
|
||||
require.Len(t, got.interactshURLs, 1)
|
||||
require.Equal(t, "http://"+got.interactshURLs[0]+"/callback", got.request.String())
|
||||
require.NotContains(t, got.request.String(), "{{interactsh-url}}")
|
||||
}
|
||||
|
||||
func TestMakeSelfContainedRequestDoesNotAllocateRuntimeInteractshMarkerFromPayload(t *testing.T) {
|
||||
options := testutils.DefaultOptions
|
||||
|
||||
testutils.Init(options)
|
||||
templateID := "testing-self-contained-runtime-interactsh"
|
||||
request := &Request{
|
||||
ID: templateID,
|
||||
Name: "testing",
|
||||
SelfContained: true,
|
||||
Path: []string{"http://example.com/callback"},
|
||||
Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
|
||||
Body: "{{payload}}",
|
||||
Payloads: map[string]interface{}{
|
||||
"payload": []string{"{{server_value}}"},
|
||||
},
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: templateID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
client, err := interactsh.New(&interactsh.Options{
|
||||
ServerURL: options.InteractshURL,
|
||||
CacheSize: options.InteractionsCacheSize,
|
||||
Eviction: time.Duration(options.InteractionsEviction) * time.Second,
|
||||
CooldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second,
|
||||
PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second,
|
||||
DisableHttpFallback: true,
|
||||
})
|
||||
require.Nil(t, err, "could not create interactsh client")
|
||||
t.Cleanup(func() {
|
||||
client.Close()
|
||||
})
|
||||
executerOpts.Interactsh = client
|
||||
|
||||
err = request.Compile(executerOpts)
|
||||
require.Nil(t, err, "could not compile http request")
|
||||
|
||||
generator := request.newGenerator(false)
|
||||
inputData, payloads, _ := generator.nextValue()
|
||||
got, err := generator.Make(
|
||||
context.Background(),
|
||||
contextargs.NewWithInput(context.Background(), ""),
|
||||
inputData,
|
||||
payloads,
|
||||
map[string]interface{}{
|
||||
"server_value": "{{interactsh-url}}",
|
||||
},
|
||||
)
|
||||
require.Nil(t, err, "could not make self-contained http request")
|
||||
body, err := got.request.BodyBytes()
|
||||
require.Nil(t, err, "could not read request body")
|
||||
|
||||
require.Empty(t, got.interactshURLs)
|
||||
require.Equal(t, "{{interactsh-url}}", string(body))
|
||||
}
|
||||
|
||||
func TestExecuteRequestDoesNotReevaluateGeneratedPayloadMetadata(t *testing.T) {
|
||||
options := testutils.DefaultOptions
|
||||
|
||||
testutils.Init(options)
|
||||
templateID := "testing-http-payload-metadata-data"
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
request := &Request{
|
||||
ID: templateID,
|
||||
Name: "testing",
|
||||
Path: []string{server.URL},
|
||||
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: templateID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
err := request.Compile(executerOpts)
|
||||
require.Nil(t, err, "could not compile http request")
|
||||
|
||||
generator := request.newGenerator(false)
|
||||
inputData, payloads, _ := generator.nextValue()
|
||||
input := contextargs.NewWithInput(context.Background(), server.URL)
|
||||
generated, err := generator.Make(context.Background(), input, inputData, payloads, map[string]interface{}{
|
||||
"secret": "leaked-secret",
|
||||
})
|
||||
require.Nil(t, err, "could not make http request")
|
||||
generated.meta = map[string]interface{}{
|
||||
"payload": "{{secret}}",
|
||||
}
|
||||
|
||||
err = request.executeRequest(input, generated, nil, false, func(*output.InternalWrappedEvent) {}, 0)
|
||||
|
||||
require.Nil(t, err, "could not execute http request")
|
||||
require.Equal(t, "{{secret}}", generated.meta["payload"])
|
||||
}
|
||||
|
||||
// areUnique checks if the elements of string slice are unique
|
||||
func areUnique(elements []string) bool {
|
||||
encountered := map[string]bool{}
|
||||
|
||||
@@ -682,20 +682,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
||||
|
||||
request.setCustomHeaders(generatedRequest)
|
||||
|
||||
// Try to evaluate any payloads before replacement
|
||||
finalMap := generators.MergeMaps(generatedRequest.dynamicValues, generatedRequest.meta)
|
||||
|
||||
// add known variables from metainput
|
||||
if _, ok := finalMap["ip"]; !ok && input.MetaInput.CustomIP != "" {
|
||||
finalMap["ip"] = input.MetaInput.CustomIP
|
||||
}
|
||||
|
||||
for payloadName, payloadValue := range generatedRequest.meta {
|
||||
if data, err := expressions.Evaluate(types.ToString(payloadValue), finalMap); err == nil {
|
||||
generatedRequest.meta[payloadName] = data
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
resp *http.Response
|
||||
fromCache bool
|
||||
|
||||
+134
-53
@@ -7,13 +7,14 @@ import (
|
||||
"maps"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/goja"
|
||||
"github.com/alecthomas/chroma/quick"
|
||||
"github.com/ditashi/jsbeautifier-go/jsbeautifier"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/goja"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
|
||||
@@ -24,11 +25,11 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
@@ -217,7 +218,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
|
||||
args := compiler.NewExecuteArgs()
|
||||
allVars := generators.MergeMaps(options.Variables.GetAll(), options.Options.Vars.AsMap(), request.options.Constants)
|
||||
// proceed with whatever args we have
|
||||
args.Args, _ = request.evaluateArgs(allVars, options, true)
|
||||
args.Args, _, _ = request.evaluateArgs(allVars, options, true)
|
||||
|
||||
initCompiled, err := compiler.SourceAutoMode(request.Init, false)
|
||||
if err != nil {
|
||||
@@ -333,24 +334,32 @@ func (request *Request) executeWithResults(port string, target *contextargs.Cont
|
||||
payloadValues["Port"] = port
|
||||
|
||||
hostnameVariables := protocolutils.GenerateDNSVariables(hostname)
|
||||
values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.Constants, templateCtx.GetAll())
|
||||
variablesMap := request.options.Variables.Evaluate(values)
|
||||
payloadValues = generators.MergeMaps(variablesMap, payloadValues, request.options.Constants, hostnameVariables)
|
||||
scope := request.options.NewVariablesScope(payloadValues, hostnameVariables, request.options.Constants)
|
||||
request.options.AddTemplateCtxToVariablesScope(input.MetaInput, scope)
|
||||
|
||||
var interactshURLs []string
|
||||
if request.options.Interactsh != nil {
|
||||
for payloadName, payloadValue := range payloadValues {
|
||||
var urls []string
|
||||
payloadValue, urls = request.options.Interactsh.Replace(types.ToString(payloadValue), interactshURLs)
|
||||
if len(urls) > 0 {
|
||||
interactshURLs = append(interactshURLs, urls...)
|
||||
payloadValues[payloadName] = payloadValue
|
||||
}
|
||||
}
|
||||
|
||||
evaluation := request.options.Variables.EvaluateWithInteractshScope(scope, request.options.Interactsh)
|
||||
variablesMap, interactshURLs := evaluation.Values, evaluation.InteractURLs
|
||||
templateVariables := generators.CopyMap(evaluation.TemplateValues)
|
||||
|
||||
for key := range payloadValues {
|
||||
delete(templateVariables, key)
|
||||
}
|
||||
|
||||
for key := range request.options.Constants {
|
||||
delete(templateVariables, key)
|
||||
}
|
||||
|
||||
for key := range hostnameVariables {
|
||||
delete(templateVariables, key)
|
||||
}
|
||||
|
||||
payloadValues = generators.MergeMaps(variablesMap, payloadValues, request.options.Constants, hostnameVariables)
|
||||
|
||||
// export all variables to template context
|
||||
templateCtx.Merge(payloadValues)
|
||||
templateCtx.MergeTemplateVariables(templateVariables)
|
||||
|
||||
if vardump.EnableVarDump {
|
||||
gologger.Debug().Msgf("JavaScript Protocol request variables: %s\n", vardump.DumpVariables(payloadValues))
|
||||
@@ -361,28 +370,34 @@ func (request *Request) executeWithResults(port string, target *contextargs.Cont
|
||||
|
||||
if request.options.Options.Debug || request.options.Options.DebugRequests {
|
||||
gologger.Debug().Msgf("[%s] Executing Precondition for request\n", request.TemplateID)
|
||||
|
||||
var highlightFormatter = "terminal256"
|
||||
if requestOptions.Options.NoColor {
|
||||
highlightFormatter = "text"
|
||||
}
|
||||
|
||||
var buff bytes.Buffer
|
||||
_ = quick.Highlight(&buff, beautifyJavascript(request.PreCondition), "javascript", highlightFormatter, "monokai")
|
||||
|
||||
prettyPrint(request.TemplateID, buff.String())
|
||||
}
|
||||
|
||||
argsCopy, err := request.getArgsCopy(input, payloads, requestOptions, true)
|
||||
argsCopy, argURLs, err := request.getArgsCopy(input, payloads, requestOptions, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
interactshURLs = append(interactshURLs, argURLs...)
|
||||
argsCopy.TemplateCtx = templateCtx.GetAll()
|
||||
|
||||
result, err := request.options.JsCompiler.ExecuteWithOptions(target.Context(), request.preConditionCompiled, argsCopy,
|
||||
&compiler.ExecuteOptions{
|
||||
ExecutionId: requestOptions.Options.ExecutionId,
|
||||
TimeoutVariants: requestOptions.Options.GetTimeouts(),
|
||||
Source: &request.PreCondition,
|
||||
Source: &request.PreCondition,
|
||||
},
|
||||
)
|
||||
|
||||
// if precondition was successful
|
||||
if err == nil && result.GetSuccess() {
|
||||
if request.options.Options.Debug || request.options.Options.DebugRequests {
|
||||
@@ -391,6 +406,7 @@ func (request *Request) executeWithResults(port string, target *contextargs.Cont
|
||||
}
|
||||
} else {
|
||||
var outError error
|
||||
|
||||
// if js code failed to execute
|
||||
if err != nil {
|
||||
outError = errkit.Append(errkit.New("pre-condition not satisfied skipping template execution"), err)
|
||||
@@ -398,8 +414,10 @@ func (request *Request) executeWithResults(port string, target *contextargs.Cont
|
||||
// execution successful but pre-condition returned false
|
||||
outError = errkit.New("pre-condition not satisfied skipping template execution")
|
||||
}
|
||||
|
||||
results := map[string]interface{}(result)
|
||||
results["error"] = outError.Error()
|
||||
|
||||
// generate and return failed event
|
||||
data := request.generateEventData(input, results, hostPort)
|
||||
data = generators.MergeMaps(data, payloadValues)
|
||||
@@ -408,14 +426,15 @@ func (request *Request) executeWithResults(port string, target *contextargs.Cont
|
||||
allVars = generators.MergeMaps(allVars, data)
|
||||
wrappedEvent.OperatorsResult.PayloadValues = allVars
|
||||
})
|
||||
|
||||
callback(event)
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if request.generator != nil && request.Threads > 1 {
|
||||
request.executeRequestParallel(target.Context(), hostPort, hostname, input, payloadValues, callback)
|
||||
return nil
|
||||
return request.executeRequestParallel(target.Context(), hostPort, hostname, input, payloadValues, callback, interactshURLs)
|
||||
}
|
||||
|
||||
var gotMatches bool
|
||||
@@ -434,18 +453,30 @@ func (request *Request) executeWithResults(port string, target *contextargs.Cont
|
||||
default:
|
||||
}
|
||||
|
||||
if err := request.executeRequestWithPayloads(hostPort, input, hostname, value, payloadValues, func(result *output.InternalWrappedEvent) {
|
||||
renderedValue, err := render.RenderMap(render.MapInput{
|
||||
Source: value,
|
||||
Data: payloadValues,
|
||||
Values: generators.MergeMaps(value, payloadValues),
|
||||
Interactsh: request.options.Interactsh,
|
||||
InteractURLs: interactshURLs,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not evaluate payload helper expressions")
|
||||
}
|
||||
|
||||
if err := request.executeRequestWithPayloads(hostPort, input, hostname, renderedValue.Values, nil, func(result *output.InternalWrappedEvent) {
|
||||
if result.OperatorsResult != nil && result.OperatorsResult.Matched {
|
||||
gotMatches = true
|
||||
request.options.Progress.IncrementMatched()
|
||||
}
|
||||
callback(result)
|
||||
}, requestOptions, interactshURLs); err != nil {
|
||||
}, requestOptions, renderedValue.InteractURLs); err != nil {
|
||||
if errkit.IsNetworkPermanentErr(err) {
|
||||
// gologger.Verbose().Msgf("Could not execute request: %s\n", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// If this was a match, and we want to stop at first match, skip all further requests.
|
||||
shouldStopAtFirstMatch := request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch
|
||||
if shouldStopAtFirstMatch && gotMatches {
|
||||
@@ -456,16 +487,37 @@ func (request *Request) executeWithResults(port string, target *contextargs.Cont
|
||||
return request.executeRequestWithPayloads(hostPort, input, hostname, nil, payloadValues, callback, requestOptions, interactshURLs)
|
||||
}
|
||||
|
||||
func (request *Request) executeRequestParallel(ctxParent context.Context, hostPort, hostname string, input *contextargs.Context, payloadValues map[string]interface{}, callback protocols.OutputEventCallback) {
|
||||
func (request *Request) executeRequestParallel(ctxParent context.Context, hostPort, hostname string, input *contextargs.Context, payloadValues map[string]interface{}, callback protocols.OutputEventCallback, interactshURLs []string) error {
|
||||
threads := request.Threads
|
||||
if threads == 0 {
|
||||
threads = 1
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancelCause(ctxParent)
|
||||
defer cancel(nil)
|
||||
|
||||
requestOptions := request.options
|
||||
gotmatches := &atomic.Bool{}
|
||||
|
||||
var (
|
||||
errMu sync.Mutex
|
||||
runErr error
|
||||
)
|
||||
|
||||
setRunErr := func(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
|
||||
errMu.Lock()
|
||||
if runErr == nil {
|
||||
runErr = err
|
||||
}
|
||||
errMu.Unlock()
|
||||
|
||||
cancel(err)
|
||||
}
|
||||
|
||||
// if request threads matches global payload concurrency we follow it
|
||||
shouldFollowGlobal := threads == request.options.Options.PayloadConcurrency
|
||||
|
||||
@@ -473,6 +525,7 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
|
||||
|
||||
if request.generator != nil {
|
||||
iterator := request.generator.NewIterator()
|
||||
iteratorLoop:
|
||||
for {
|
||||
value, ok := iterator.Value()
|
||||
if !ok {
|
||||
@@ -481,7 +534,9 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
|
||||
|
||||
select {
|
||||
case <-input.Context().Done():
|
||||
return
|
||||
break iteratorLoop
|
||||
case <-ctx.Done():
|
||||
break iteratorLoop
|
||||
default:
|
||||
}
|
||||
|
||||
@@ -492,38 +547,59 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
|
||||
}
|
||||
}
|
||||
|
||||
renderedValue, err := render.RenderMap(render.MapInput{
|
||||
Source: value,
|
||||
Data: payloadValues,
|
||||
Values: generators.MergeMaps(value, payloadValues),
|
||||
Interactsh: request.options.Interactsh,
|
||||
InteractURLs: interactshURLs,
|
||||
})
|
||||
if err != nil {
|
||||
setRunErr(errors.Wrap(err, "could not evaluate payload helper expressions"))
|
||||
break
|
||||
}
|
||||
|
||||
sg.Add()
|
||||
go func() {
|
||||
go func(value map[string]interface{}, urls []string) {
|
||||
defer sg.Done()
|
||||
if ctx.Err() != nil {
|
||||
// work already done exit
|
||||
return
|
||||
}
|
||||
|
||||
shouldStopAtFirstMatch := request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch
|
||||
if err := request.executeRequestWithPayloads(hostPort, input, hostname, value, payloadValues, func(result *output.InternalWrappedEvent) {
|
||||
if err := request.executeRequestWithPayloads(hostPort, input, hostname, value, nil, func(result *output.InternalWrappedEvent) {
|
||||
if result.OperatorsResult != nil && result.OperatorsResult.Matched {
|
||||
gotmatches.Store(true)
|
||||
}
|
||||
callback(result)
|
||||
}, requestOptions, []string{}); err != nil {
|
||||
}, requestOptions, urls); err != nil {
|
||||
if errkit.IsNetworkPermanentErr(err) {
|
||||
cancel(err)
|
||||
setRunErr(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
// If this was a match, and we want to stop at first match, skip all further requests.
|
||||
|
||||
// If this was a match, and we want to stop at first match, skip all further requests.
|
||||
if shouldStopAtFirstMatch && gotmatches.Load() {
|
||||
cancel(nil)
|
||||
return
|
||||
}
|
||||
}()
|
||||
}(renderedValue.Values, renderedValue.InteractURLs)
|
||||
}
|
||||
}
|
||||
|
||||
sg.Wait()
|
||||
|
||||
if runErr != nil {
|
||||
return runErr
|
||||
}
|
||||
|
||||
if gotmatches.Load() {
|
||||
request.options.Progress.IncrementMatched()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (request *Request) executeRequestWithPayloads(
|
||||
@@ -536,30 +612,21 @@ func (request *Request) executeRequestWithPayloads(
|
||||
requestOptions *protocols.ExecutorOptions,
|
||||
interactshURLs []string,
|
||||
) error {
|
||||
interactshURLs = append([]string(nil), interactshURLs...)
|
||||
payloadValues := generators.MergeMaps(payload, previous)
|
||||
argsCopy, err := request.getArgsCopy(input, payloadValues, requestOptions, false)
|
||||
|
||||
argsCopy, argURLs, err := request.getArgsCopy(input, payloadValues, requestOptions, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
interactshURLs = append(interactshURLs, argURLs...)
|
||||
if request.options.HasTemplateCtx(input.MetaInput) {
|
||||
argsCopy.TemplateCtx = request.options.GetTemplateCtx(input.MetaInput).GetAll()
|
||||
} else {
|
||||
argsCopy.TemplateCtx = map[string]interface{}{}
|
||||
}
|
||||
|
||||
if request.options.Interactsh != nil {
|
||||
if argsCopy.Args != nil {
|
||||
for k, v := range argsCopy.Args {
|
||||
var urls []string
|
||||
v, urls = request.options.Interactsh.Replace(fmt.Sprint(v), []string{})
|
||||
if len(urls) > 0 {
|
||||
interactshURLs = append(interactshURLs, urls...)
|
||||
argsCopy.Args[k] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results, err := request.options.JsCompiler.ExecuteWithOptions(input.Context(), request.scriptCompiled, argsCopy,
|
||||
&compiler.ExecuteOptions{
|
||||
ExecutionId: requestOptions.Options.ExecutionId,
|
||||
@@ -704,41 +771,55 @@ func (request *Request) generateEventData(input *contextargs.Context, values map
|
||||
return data
|
||||
}
|
||||
|
||||
func (request *Request) getArgsCopy(input *contextargs.Context, payloadValues map[string]interface{}, requestOptions *protocols.ExecutorOptions, ignoreErrors bool) (*compiler.ExecuteArgs, error) {
|
||||
func (request *Request) getArgsCopy(input *contextargs.Context, payloadValues map[string]interface{}, requestOptions *protocols.ExecutorOptions, ignoreErrors bool) (*compiler.ExecuteArgs, []string, error) {
|
||||
// Template args from payloads
|
||||
argsCopy, err := request.evaluateArgs(payloadValues, requestOptions, ignoreErrors)
|
||||
argsCopy, interactshURLs, err := request.evaluateArgs(payloadValues, requestOptions, ignoreErrors)
|
||||
if err != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
return nil, err
|
||||
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// "Port" is a special variable that is considered as network port
|
||||
// and is conditional based on input port and default port specified in input
|
||||
argsCopy["Port"] = input.Port()
|
||||
|
||||
return &compiler.ExecuteArgs{Args: argsCopy}, nil
|
||||
return &compiler.ExecuteArgs{Args: argsCopy}, interactshURLs, nil
|
||||
}
|
||||
|
||||
// evaluateArgs evaluates arguments using available payload values and returns a copy of args
|
||||
func (request *Request) evaluateArgs(payloadValues map[string]interface{}, _ *protocols.ExecutorOptions, ignoreErrors bool) (map[string]interface{}, error) {
|
||||
func (request *Request) evaluateArgs(payloadValues map[string]interface{}, _ *protocols.ExecutorOptions, ignoreErrors bool) (map[string]interface{}, []string, error) {
|
||||
argsCopy := make(map[string]interface{})
|
||||
|
||||
var interactshURLs []string
|
||||
|
||||
mainLoop:
|
||||
for k, v := range request.Args {
|
||||
if vVal, ok := v.(string); ok && strings.Contains(vVal, "{") {
|
||||
finalAddress, dataErr := expressions.Evaluate(vVal, payloadValues)
|
||||
result, dataErr := render.Render(render.Input{
|
||||
Text: vVal,
|
||||
Values: payloadValues,
|
||||
Interactsh: request.options.Interactsh,
|
||||
InteractURLs: interactshURLs,
|
||||
})
|
||||
if dataErr != nil {
|
||||
return nil, errors.Wrap(dataErr, "could not evaluate template expressions")
|
||||
return nil, nil, errors.Wrap(dataErr, "could not evaluate template expressions")
|
||||
}
|
||||
if finalAddress == vVal && ignoreErrors {
|
||||
|
||||
interactshURLs = result.InteractURLs
|
||||
|
||||
if result.Text == vVal && ignoreErrors {
|
||||
argsCopy[k] = ""
|
||||
continue mainLoop
|
||||
}
|
||||
argsCopy[k] = finalAddress
|
||||
|
||||
argsCopy[k] = result.Text
|
||||
} else {
|
||||
argsCopy[k] = v
|
||||
}
|
||||
}
|
||||
return argsCopy, nil
|
||||
return argsCopy, interactshURLs, nil
|
||||
}
|
||||
|
||||
// RequestPartDefinitions contains a mapping of request part definitions and their
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package javascript
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
)
|
||||
|
||||
func TestGeneratorPayloadInteractshMarkerRendersBeforeArgs(t *testing.T) {
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.InteractionsCoolDownPeriod = 0
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() {
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
request := &Request{
|
||||
Args: map[string]interface{}{
|
||||
"cb": "{{payload}}",
|
||||
},
|
||||
Payloads: map[string]interface{}{
|
||||
"payload": []string{"{{interactsh-url}}"},
|
||||
},
|
||||
Code: `module.exports = { success: true }`,
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: "javascript-payload-interactsh",
|
||||
})
|
||||
client, err := interactsh.New(&interactsh.Options{
|
||||
ServerURL: options.InteractshURL,
|
||||
CacheSize: options.InteractionsCacheSize,
|
||||
Eviction: time.Duration(options.InteractionsEviction) * time.Second,
|
||||
CooldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second,
|
||||
PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second,
|
||||
DisableHttpFallback: true,
|
||||
})
|
||||
require.NoError(t, err, "could not create interactsh client")
|
||||
t.Cleanup(func() {
|
||||
client.Close()
|
||||
})
|
||||
executerOpts.Interactsh = client
|
||||
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
payloadValue, ok := request.generator.NewIterator().Value()
|
||||
require.True(t, ok, "could not get generated payload value")
|
||||
|
||||
payloadValues := generators.BuildPayloadFromOptions(options)
|
||||
renderedPayload, err := render.RenderMap(render.MapInput{
|
||||
Source: payloadValue,
|
||||
Data: payloadValues,
|
||||
Values: generators.MergeMaps(payloadValue, payloadValues),
|
||||
Interactsh: request.options.Interactsh,
|
||||
})
|
||||
require.NoError(t, err, "could not render generated payload")
|
||||
require.Len(t, renderedPayload.InteractURLs, 1, "payload marker should allocate one interactsh URL")
|
||||
|
||||
args, argURLs, err := request.evaluateArgs(renderedPayload.Values, executerOpts, false)
|
||||
require.NoError(t, err, "could not evaluate javascript args")
|
||||
require.Empty(t, argURLs, "already-rendered payload value should not allocate again from args")
|
||||
require.Equal(t, renderedPayload.InteractURLs[0], args["cb"])
|
||||
require.NotContains(t, args["cb"], "{{interactsh-url}}")
|
||||
}
|
||||
@@ -10,7 +10,9 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/portutil"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
@@ -192,8 +194,13 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
|
||||
if input.Type.String() != "" {
|
||||
continue
|
||||
}
|
||||
if compiled, evalErr := expressions.Evaluate(input.Data, preCompileVars); evalErr == nil {
|
||||
input.Data = compiled
|
||||
|
||||
if interactsh.HasMarkers(input.Data) {
|
||||
continue
|
||||
}
|
||||
|
||||
if result, evalErr := render.Render(render.Input{Text: input.Data, Values: preCompileVars}); evalErr == nil && !interactsh.HasMarkers(result.Text) {
|
||||
input.Data = result.Text
|
||||
}
|
||||
}
|
||||
|
||||
@@ -282,4 +289,3 @@ func (request *Request) SetDialer(dialer *fastdialer.Dialer) {
|
||||
func (r *Request) UpdateOptions(opts *protocols.ExecutorOptions) {
|
||||
r.options.ApplyNewEngineOptions(opts)
|
||||
}
|
||||
|
||||
@@ -5,10 +5,12 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/portutil"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
)
|
||||
|
||||
func TestResolvePort(t *testing.T) {
|
||||
@@ -108,3 +110,28 @@ func TestNetworkCompileMake(t *testing.T) {
|
||||
require.True(t, request.addresses[0].tls, "could not get correct port for host")
|
||||
})
|
||||
}
|
||||
|
||||
func TestCompileDoesNotPersistValueIntroducedInteractshMarker(t *testing.T) {
|
||||
options := testutils.DefaultOptions
|
||||
|
||||
testutils.Init(options)
|
||||
templateID := "testing-network-defer-value-marker"
|
||||
request := &Request{
|
||||
ID: templateID,
|
||||
Address: []string{"{{Host}}"},
|
||||
ReadSize: 1024,
|
||||
Inputs: []*Input{{Data: "{{callback}}"}},
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: templateID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
executerOpts.Variables = variables.Variable{
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(1),
|
||||
}
|
||||
executerOpts.Variables.Set("callback", "{{interactsh-url}}")
|
||||
|
||||
err := request.Compile(executerOpts)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "{{callback}}", request.Inputs[0].Data)
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
|
||||
@@ -136,6 +137,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, metadata
|
||||
func (request *Request) executeOnTarget(input *contextargs.Context, visited mapsutil.Map[string, struct{}], metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
var address string
|
||||
var err error
|
||||
|
||||
if request.isUnresponsiveAddress(input) {
|
||||
// skip on unresponsive address no need to continue
|
||||
return nil
|
||||
@@ -146,17 +148,27 @@ func (request *Request) executeOnTarget(input *contextargs.Context, visited maps
|
||||
} else {
|
||||
address, err = getAddress(input.MetaInput.Input)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
|
||||
request.options.Progress.IncrementFailedRequestsBy(1)
|
||||
return errors.Wrap(err, "could not get address from url")
|
||||
}
|
||||
|
||||
variables := protocolutils.GenerateVariables(address, false, nil)
|
||||
scope := request.options.NewVariablesScope(variables)
|
||||
|
||||
request.options.AddTemplateCtxToVariablesScope(input.MetaInput, scope)
|
||||
scope.AddData(request.options.Constants)
|
||||
|
||||
evaluation := request.options.Variables.EvaluateWithInteractshScope(scope, request.options.Interactsh)
|
||||
variablesMap, interactshURLs := evaluation.Values, evaluation.InteractURLs
|
||||
|
||||
// add template ctx variables to varMap
|
||||
if request.options.HasTemplateCtx(input.MetaInput) {
|
||||
variables = generators.MergeMaps(variables, request.options.GetTemplateCtx(input.MetaInput).GetAll())
|
||||
}
|
||||
variablesMap := request.options.Variables.Evaluate(variables)
|
||||
|
||||
variables = generators.MergeMaps(variablesMap, variables, request.options.Constants)
|
||||
|
||||
// stop at first match if requested
|
||||
@@ -181,21 +193,26 @@ func (request *Request) executeOnTarget(input *contextargs.Context, visited maps
|
||||
if visited.Has(actualAddress) && !request.options.Options.DisableClustering {
|
||||
continue
|
||||
}
|
||||
|
||||
visited.Set(actualAddress, struct{}{})
|
||||
if err = request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, wrappedCallback); err != nil {
|
||||
|
||||
if err = request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, interactshURLs, wrappedCallback); err != nil {
|
||||
outputEvent := request.responseToDSLMap("", "", "", address, "")
|
||||
callback(&output.InternalWrappedEvent{InternalEvent: outputEvent})
|
||||
gologger.Warning().Msgf("[%v] Could not make network request for (%s) : %s\n", request.options.TemplateID, actualAddress, err)
|
||||
|
||||
callback(&output.InternalWrappedEvent{InternalEvent: outputEvent})
|
||||
}
|
||||
|
||||
if shouldStopAtFirstMatch && atomicBool.Load() {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// executeAddress executes the request for an address
|
||||
func (request *Request) executeAddress(variables map[string]interface{}, actualAddress, address string, input *contextargs.Context, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
func (request *Request) executeAddress(variables map[string]interface{}, actualAddress, address string, input *contextargs.Context, shouldUseTLS bool, previous output.InternalEvent, interactshURLs []string, callback protocols.OutputEventCallback) error {
|
||||
variables = generators.MergeMaps(variables, map[string]interface{}{"Hostname": address})
|
||||
payloads := generators.BuildPayloadFromOptions(request.options.Options)
|
||||
|
||||
@@ -245,35 +262,53 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
|
||||
return nil
|
||||
}
|
||||
|
||||
value = generators.MergeMaps(value, payloads)
|
||||
renderedValue, err := render.RenderMap(render.MapInput{
|
||||
Source: value,
|
||||
Data: payloads,
|
||||
Values: generators.MergeMaps(variables, value, payloads),
|
||||
Interactsh: request.options.Interactsh,
|
||||
InteractURLs: interactshURLs,
|
||||
})
|
||||
if err != nil {
|
||||
m.Lock()
|
||||
multiErr = multierr.Append(multiErr, errors.Wrap(err, "could not evaluate payload helper expressions"))
|
||||
m.Unlock()
|
||||
continue
|
||||
}
|
||||
|
||||
swg.Add()
|
||||
go func(vars map[string]interface{}) {
|
||||
go func(vars map[string]interface{}, urls []string) {
|
||||
defer swg.Done()
|
||||
|
||||
if request.isUnresponsiveAddress(updatedTarget) {
|
||||
// skip on unresponsive address no need to continue
|
||||
return
|
||||
}
|
||||
if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, vars, previous, callback); err != nil {
|
||||
|
||||
if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, vars, previous, urls, callback); err != nil {
|
||||
m.Lock()
|
||||
multiErr = multierr.Append(multiErr, err)
|
||||
m.Unlock()
|
||||
}
|
||||
}(value)
|
||||
}(renderedValue.Values, renderedValue.InteractURLs)
|
||||
}
|
||||
|
||||
swg.Wait()
|
||||
|
||||
if multiErr != nil {
|
||||
return multiErr
|
||||
}
|
||||
} else {
|
||||
value := maps.Clone(payloads)
|
||||
if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil {
|
||||
if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, value, previous, interactshURLs, callback); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (request *Request) executeRequestWithPayloads(variables map[string]interface{}, actualAddress, address string, input *contextargs.Context, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
func (request *Request) executeRequestWithPayloads(variables map[string]interface{}, actualAddress, address string, input *contextargs.Context, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, interactshURLs []string, callback protocols.OutputEventCallback) error {
|
||||
var (
|
||||
hostname string
|
||||
conn net.Conn
|
||||
@@ -307,7 +342,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
|
||||
}()
|
||||
_ = conn.SetDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second))
|
||||
|
||||
var interactshURLs []string
|
||||
interactshURLs = append([]string(nil), interactshURLs...)
|
||||
|
||||
var responseBuilder, reqBuilder strings.Builder
|
||||
|
||||
@@ -320,21 +355,22 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
|
||||
inputEvents := make(map[string]interface{})
|
||||
|
||||
for _, input := range request.Inputs {
|
||||
dataInBytes := []byte(input.Data)
|
||||
var err error
|
||||
|
||||
dataInBytes, err = expressions.EvaluateByte(dataInBytes, interimValues)
|
||||
result, err := render.Render(render.Input{
|
||||
Text: input.Data,
|
||||
Values: interimValues,
|
||||
Interactsh: request.options.Interactsh,
|
||||
InteractURLs: interactshURLs,
|
||||
})
|
||||
if err != nil {
|
||||
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
|
||||
request.options.Progress.IncrementFailedRequestsBy(1)
|
||||
|
||||
return errors.Wrap(err, "could not evaluate template expressions")
|
||||
}
|
||||
|
||||
data := string(dataInBytes)
|
||||
if request.options.Interactsh != nil {
|
||||
data, interactshURLs = request.options.Interactsh.Replace(data, []string{})
|
||||
dataInBytes = []byte(data)
|
||||
}
|
||||
data := result.Text
|
||||
dataInBytes := []byte(data)
|
||||
interactshURLs = result.InteractURLs
|
||||
|
||||
if err := expressions.ContainsUnresolvedVariables(data); err != nil {
|
||||
gologger.Warning().Msgf("[%s] Could not make network request for %s: %v\n", request.options.TemplateID, actualAddress, err)
|
||||
|
||||
@@ -2,15 +2,20 @@ package network
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
|
||||
@@ -18,7 +23,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
)
|
||||
|
||||
func TestNetworkExecuteWithResults(t *testing.T) {
|
||||
@@ -109,6 +114,168 @@ func TestNetworkExecuteWithResults(t *testing.T) {
|
||||
require.Equal(t, "<h1>Example Domain</h1>", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
|
||||
}
|
||||
|
||||
func captureNetworkRequest(t *testing.T, listener net.Listener) (<-chan string, <-chan error) {
|
||||
t.Helper()
|
||||
captured := make(chan string, 1)
|
||||
serverErr := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
if tcpListener, ok := listener.(*net.TCPListener); ok {
|
||||
_ = tcpListener.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
}
|
||||
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
serverErr <- err
|
||||
return
|
||||
}
|
||||
|
||||
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
buffer := make([]byte, 4096)
|
||||
n, err := conn.Read(buffer)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
if err == io.EOF {
|
||||
continue
|
||||
}
|
||||
serverErr <- err
|
||||
return
|
||||
}
|
||||
captured <- string(buffer[:n])
|
||||
_, _ = conn.Write([]byte("ok"))
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
}()
|
||||
|
||||
return captured, serverErr
|
||||
}
|
||||
|
||||
func TestNetworkGeneratorPayloadInteractshMarkerRendersBeforeInput(t *testing.T) {
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.InteractionsCoolDownPeriod = 0
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() {
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err, "could not listen")
|
||||
t.Cleanup(func() {
|
||||
_ = listener.Close()
|
||||
})
|
||||
|
||||
captured, serverErr := captureNetworkRequest(t, listener)
|
||||
|
||||
request := &Request{
|
||||
ID: "network-payload-interactsh",
|
||||
Address: []string{"{{Hostname}}"},
|
||||
ReadSize: 2,
|
||||
Threads: 1,
|
||||
Inputs: []*Input{{
|
||||
Data: "{{payload}}",
|
||||
}},
|
||||
Payloads: map[string]interface{}{
|
||||
"payload": []string{"{{interactsh-url}}"},
|
||||
},
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: "network-payload-interactsh",
|
||||
})
|
||||
client, err := interactsh.New(&interactsh.Options{
|
||||
ServerURL: options.InteractshURL,
|
||||
CacheSize: options.InteractionsCacheSize,
|
||||
Eviction: time.Duration(options.InteractionsEviction) * time.Second,
|
||||
CooldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second,
|
||||
PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second,
|
||||
DisableHttpFallback: true,
|
||||
})
|
||||
require.NoError(t, err, "could not create interactsh client")
|
||||
t.Cleanup(func() {
|
||||
client.Close()
|
||||
})
|
||||
executerOpts.Interactsh = client
|
||||
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
ctxArgs := contextargs.NewWithInput(context.Background(), listener.Addr().String())
|
||||
err = request.ExecuteWithResults(ctxArgs, nil, nil, func(*output.InternalWrappedEvent) {})
|
||||
require.NoError(t, err, "could not execute network request")
|
||||
|
||||
select {
|
||||
case got := <-captured:
|
||||
require.NotContains(t, got, "{{payload}}")
|
||||
require.NotContains(t, got, "{{interactsh-url}}")
|
||||
require.Contains(t, got, client.GetHostname())
|
||||
case err := <-serverErr:
|
||||
require.NoError(t, err, "server failed")
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for network request")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetworkCompileDefersInteractshMarkerHelpersToRuntime(t *testing.T) {
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.InteractionsCoolDownPeriod = 0
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() {
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err, "could not listen")
|
||||
t.Cleanup(func() {
|
||||
_ = listener.Close()
|
||||
})
|
||||
|
||||
captured, serverErr := captureNetworkRequest(t, listener)
|
||||
|
||||
request := &Request{
|
||||
ID: "network-runtime-interactsh-helper",
|
||||
Address: []string{"{{Hostname}}"},
|
||||
ReadSize: 2,
|
||||
Threads: 1,
|
||||
Inputs: []*Input{{
|
||||
Data: "{{md5('{{interactsh-url}}')}}",
|
||||
}},
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: "network-runtime-interactsh-helper",
|
||||
})
|
||||
client, err := interactsh.New(&interactsh.Options{
|
||||
ServerURL: options.InteractshURL,
|
||||
CacheSize: options.InteractionsCacheSize,
|
||||
Eviction: time.Duration(options.InteractionsEviction) * time.Second,
|
||||
CooldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second,
|
||||
PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second,
|
||||
DisableHttpFallback: true,
|
||||
})
|
||||
require.NoError(t, err, "could not create interactsh client")
|
||||
t.Cleanup(func() {
|
||||
client.Close()
|
||||
})
|
||||
executerOpts.Interactsh = client
|
||||
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
ctxArgs := contextargs.NewWithInput(context.Background(), listener.Addr().String())
|
||||
err = request.ExecuteWithResults(ctxArgs, nil, nil, func(*output.InternalWrappedEvent) {})
|
||||
require.NoError(t, err, "could not execute network request")
|
||||
|
||||
literalMarkerHash := fmt.Sprintf("%x", md5.Sum([]byte("{{interactsh-url}}")))
|
||||
select {
|
||||
case got := <-captured:
|
||||
require.Len(t, got, len(literalMarkerHash))
|
||||
require.NotEqual(t, literalMarkerHash, got, "compile-time evaluation must not consume the source marker before runtime Interactsh allocation")
|
||||
require.NotContains(t, got, "{{interactsh-url}}")
|
||||
case err := <-serverErr:
|
||||
require.NoError(t, err, "server failed")
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for network request")
|
||||
}
|
||||
}
|
||||
|
||||
var exampleBody = `<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
|
||||
+55
-37
@@ -221,6 +221,24 @@ func (e *ExecutorOptions) GetTemplateCtx(input *contextargs.MetaInput) *contexta
|
||||
return templateCtx
|
||||
}
|
||||
|
||||
// NewVariablesScope creates a variable evaluation scope with terminal data
|
||||
// layers applied in order.
|
||||
func (e *ExecutorOptions) NewVariablesScope(values ...map[string]interface{}) *variables.Scope {
|
||||
return variables.NewScope().AddData(values...)
|
||||
}
|
||||
|
||||
// AddTemplateCtxToVariablesScope adds template context values to a variable
|
||||
// scope while preserving which keys came from prior variable evaluation.
|
||||
func (e *ExecutorOptions) AddTemplateCtxToVariablesScope(input *contextargs.MetaInput, scope *variables.Scope) {
|
||||
if input == nil || scope == nil || !e.HasTemplateCtx(input) {
|
||||
return
|
||||
}
|
||||
|
||||
templateCtx := e.GetTemplateCtx(input)
|
||||
scope.AddData(templateCtx.GetAll())
|
||||
scope.AddTemplate(templateCtx.GetTemplateVariables())
|
||||
}
|
||||
|
||||
// AddTemplateVars adds vars to template context with given template type as prefix
|
||||
// this method is no-op if template is not multi protocol
|
||||
func (e *ExecutorOptions) AddTemplateVars(input *contextargs.MetaInput, reqType templateTypes.ProtocolType, reqID string, vars map[string]interface{}) {
|
||||
@@ -273,44 +291,44 @@ func (e *ExecutorOptions) AddTemplateVar(input *contextargs.MetaInput, templateT
|
||||
// Copy returns a copy of the executeroptions structure
|
||||
func (e *ExecutorOptions) Copy() *ExecutorOptions {
|
||||
copy := &ExecutorOptions{
|
||||
TemplateID: e.TemplateID,
|
||||
TemplatePath: e.TemplatePath,
|
||||
TemplateInfo: e.TemplateInfo,
|
||||
TemplateVerifier: e.TemplateVerifier,
|
||||
TemplateID: e.TemplateID,
|
||||
TemplatePath: e.TemplatePath,
|
||||
TemplateInfo: e.TemplateInfo,
|
||||
TemplateVerifier: e.TemplateVerifier,
|
||||
TemplateVerificationCallback: e.TemplateVerificationCallback,
|
||||
RawTemplate: e.RawTemplate,
|
||||
Output: e.Output,
|
||||
Options: e.Options,
|
||||
IssuesClient: e.IssuesClient,
|
||||
Progress: e.Progress,
|
||||
RateLimiter: e.RateLimiter,
|
||||
Catalog: e.Catalog,
|
||||
ProjectFile: e.ProjectFile,
|
||||
Browser: e.Browser,
|
||||
Interactsh: e.Interactsh,
|
||||
HostErrorsCache: e.HostErrorsCache,
|
||||
StopAtFirstMatch: e.StopAtFirstMatch,
|
||||
Variables: e.Variables,
|
||||
Constants: e.Constants,
|
||||
ExcludeMatchers: e.ExcludeMatchers,
|
||||
InputHelper: e.InputHelper,
|
||||
FuzzParamsFrequency: e.FuzzParamsFrequency,
|
||||
FuzzStatsDB: e.FuzzStatsDB,
|
||||
Operators: e.Operators,
|
||||
DoNotCache: e.DoNotCache,
|
||||
Colorizer: e.Colorizer,
|
||||
WorkflowLoader: e.WorkflowLoader,
|
||||
ResumeCfg: e.ResumeCfg,
|
||||
ProtocolType: e.ProtocolType,
|
||||
Flow: e.Flow,
|
||||
IsMultiProtocol: e.IsMultiProtocol,
|
||||
JsCompiler: e.JsCompiler,
|
||||
AuthProvider: e.AuthProvider,
|
||||
TemporaryDirectory: e.TemporaryDirectory,
|
||||
Parser: e.Parser,
|
||||
ExportReqURLPattern: e.ExportReqURLPattern,
|
||||
GlobalMatchers: e.GlobalMatchers,
|
||||
Logger: e.Logger,
|
||||
RawTemplate: e.RawTemplate,
|
||||
Output: e.Output,
|
||||
Options: e.Options,
|
||||
IssuesClient: e.IssuesClient,
|
||||
Progress: e.Progress,
|
||||
RateLimiter: e.RateLimiter,
|
||||
Catalog: e.Catalog,
|
||||
ProjectFile: e.ProjectFile,
|
||||
Browser: e.Browser,
|
||||
Interactsh: e.Interactsh,
|
||||
HostErrorsCache: e.HostErrorsCache,
|
||||
StopAtFirstMatch: e.StopAtFirstMatch,
|
||||
Variables: e.Variables,
|
||||
Constants: e.Constants,
|
||||
ExcludeMatchers: e.ExcludeMatchers,
|
||||
InputHelper: e.InputHelper,
|
||||
FuzzParamsFrequency: e.FuzzParamsFrequency,
|
||||
FuzzStatsDB: e.FuzzStatsDB,
|
||||
Operators: e.Operators,
|
||||
DoNotCache: e.DoNotCache,
|
||||
Colorizer: e.Colorizer,
|
||||
WorkflowLoader: e.WorkflowLoader,
|
||||
ResumeCfg: e.ResumeCfg,
|
||||
ProtocolType: e.ProtocolType,
|
||||
Flow: e.Flow,
|
||||
IsMultiProtocol: e.IsMultiProtocol,
|
||||
JsCompiler: e.JsCompiler,
|
||||
AuthProvider: e.AuthProvider,
|
||||
TemporaryDirectory: e.TemporaryDirectory,
|
||||
Parser: e.Parser,
|
||||
ExportReqURLPattern: e.ExportReqURLPattern,
|
||||
GlobalMatchers: e.GlobalMatchers,
|
||||
Logger: e.Logger,
|
||||
}
|
||||
copy.ClusterMappings = e.ClusterMappings.Copy()
|
||||
copy.CreateTemplateCtxStore()
|
||||
|
||||
@@ -20,10 +20,10 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
|
||||
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
@@ -215,25 +215,33 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
||||
|
||||
hostnameVariables := protocolutils.GenerateDNSVariables(hostname)
|
||||
// add template context variables to varMap
|
||||
values := generators.MergeMaps(payloadValues, hostnameVariables)
|
||||
scope := request.options.NewVariablesScope(payloadValues, hostnameVariables, previous)
|
||||
if request.options.HasTemplateCtx(input.MetaInput) {
|
||||
values = generators.MergeMaps(values, request.options.GetTemplateCtx(input.MetaInput).GetAll())
|
||||
request.options.AddTemplateCtxToVariablesScope(input.MetaInput, scope)
|
||||
}
|
||||
|
||||
variablesMap := request.options.Variables.Evaluate(values)
|
||||
payloadValues = generators.MergeMaps(variablesMap, payloadValues, request.options.Constants)
|
||||
scope.AddData(request.options.Constants)
|
||||
variablesMap := request.options.Variables.EvaluateScope(scope).Values
|
||||
payloadValues = generators.MergeMaps(variablesMap, payloadValues, previous)
|
||||
if request.options.HasTemplateCtx(input.MetaInput) {
|
||||
payloadValues = generators.MergeMaps(payloadValues, request.options.GetTemplateCtx(input.MetaInput).GetAll())
|
||||
}
|
||||
payloadValues = generators.MergeMaps(payloadValues, request.options.Constants)
|
||||
|
||||
if vardump.EnableVarDump {
|
||||
gologger.Debug().Msgf("SSL Protocol request variables: %s\n", vardump.DumpVariables(payloadValues))
|
||||
}
|
||||
|
||||
finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues)
|
||||
result, dataErr := render.Render(render.Input{Text: request.Address, Values: payloadValues})
|
||||
if dataErr != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), dataErr)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
|
||||
return errors.Wrap(dataErr, "could not evaluate template expressions")
|
||||
}
|
||||
addressToDial := string(finalAddress)
|
||||
|
||||
addressToDial := result.Text
|
||||
|
||||
host, port, err := net.SplitHostPort(addressToDial)
|
||||
if err != nil {
|
||||
return errkit.Wrap(err, "could not split input host port")
|
||||
|
||||
@@ -23,10 +23,10 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
|
||||
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
@@ -177,30 +177,45 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
|
||||
if err != nil {
|
||||
return errors.Wrap(err, parseUrlErrorMessage)
|
||||
}
|
||||
|
||||
defaultVars := protocolutils.GenerateVariables(parsed, false, nil)
|
||||
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
|
||||
// add templatecontext variables to varMap
|
||||
variables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.GetTemplateCtx(target.MetaInput).GetAll()))
|
||||
payloadValues := generators.MergeMaps(variables, defaultVars, optionVars, dynamicValues, request.options.Constants)
|
||||
scope := request.options.NewVariablesScope(defaultVars, optionVars, dynamicValues, previous)
|
||||
|
||||
request.options.AddTemplateCtxToVariablesScope(target.MetaInput, scope)
|
||||
scope.AddData(request.options.Constants)
|
||||
|
||||
variables := request.options.Variables.EvaluateScope(scope).Values
|
||||
payloadValues := generators.MergeMaps(variables, defaultVars, optionVars, dynamicValues, previous)
|
||||
if request.options.HasTemplateCtx(target.MetaInput) {
|
||||
payloadValues = generators.MergeMaps(payloadValues, request.options.GetTemplateCtx(target.MetaInput).GetAll())
|
||||
}
|
||||
payloadValues = generators.MergeMaps(payloadValues, request.options.Constants)
|
||||
|
||||
requestOptions := request.options
|
||||
for key, value := range request.Headers {
|
||||
finalData, dataErr := expressions.EvaluateByte([]byte(value), payloadValues)
|
||||
result, dataErr := render.Render(render.Input{Text: value, Values: payloadValues})
|
||||
if dataErr != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
|
||||
return errors.Wrap(dataErr, evaluateTemplateExpressionErrorMessage)
|
||||
}
|
||||
header.Set(key, string(finalData))
|
||||
|
||||
header.Set(key, result.Text)
|
||||
}
|
||||
|
||||
tlsConfig := &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
ServerName: hostname,
|
||||
MinVersion: tls.VersionTLS10,
|
||||
}
|
||||
|
||||
if requestOptions.Options.SNI != "" {
|
||||
tlsConfig.ServerName = requestOptions.Options.SNI
|
||||
}
|
||||
|
||||
websocketDialer := ws.Dialer{
|
||||
Header: ws.HandshakeHeaderHTTP(header),
|
||||
Timeout: time.Duration(requestOptions.Options.Timeout) * time.Second,
|
||||
@@ -212,31 +227,38 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
|
||||
gologger.Debug().Msgf("WebSocket Protocol request variables: %s\n", vardump.DumpVariables(payloadValues))
|
||||
}
|
||||
|
||||
finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues)
|
||||
addressResult, dataErr := render.Render(render.Input{Text: request.Address, Values: payloadValues})
|
||||
if dataErr != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
|
||||
return errors.Wrap(dataErr, evaluateTemplateExpressionErrorMessage)
|
||||
}
|
||||
|
||||
addressToDial := string(finalAddress)
|
||||
addressToDial := addressResult.Text
|
||||
|
||||
parsedAddress, err := url.Parse(addressToDial)
|
||||
if err != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
|
||||
return errors.Wrap(err, parseUrlErrorMessage)
|
||||
}
|
||||
|
||||
if parsedAddress.Path == "" || parsedAddress.Path == "/" {
|
||||
parsedAddress.Path = parsed.Path
|
||||
}
|
||||
|
||||
addressToDial = parsedAddress.String()
|
||||
|
||||
conn, readBuffer, _, err := websocketDialer.Dial(target.Context(), addressToDial)
|
||||
if err != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
|
||||
return errors.Wrap(err, "could not connect to server")
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = conn.Close()
|
||||
}()
|
||||
@@ -250,8 +272,10 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
|
||||
if err != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
|
||||
return errors.Wrap(err, "could not read write response")
|
||||
}
|
||||
|
||||
requestOptions.Progress.IncrementRequests()
|
||||
|
||||
if requestOptions.Options.Debug || requestOptions.Options.DebugRequests {
|
||||
@@ -289,6 +313,7 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
|
||||
}
|
||||
|
||||
callback(event)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -300,18 +325,20 @@ func (request *Request) readWriteInputWebsocket(conn net.Conn, payloadValues map
|
||||
for _, req := range request.Inputs {
|
||||
reqBuilder.Grow(len(req.Data))
|
||||
|
||||
finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues)
|
||||
result, dataErr := render.Render(render.Input{Text: req.Data, Values: payloadValues})
|
||||
if dataErr != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
|
||||
return nil, "", errors.Wrap(dataErr, evaluateTemplateExpressionErrorMessage)
|
||||
}
|
||||
reqBuilder.WriteString(string(finalData))
|
||||
reqBuilder.WriteString(result.Text)
|
||||
|
||||
err = wsutil.WriteClientMessage(conn, ws.OpText, finalData)
|
||||
err = wsutil.WriteClientMessage(conn, ws.OpText, []byte(result.Text))
|
||||
if err != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
|
||||
return nil, "", errors.Wrap(err, "could not write request to server")
|
||||
}
|
||||
|
||||
@@ -319,8 +346,10 @@ func (request *Request) readWriteInputWebsocket(conn net.Conn, payloadValues map
|
||||
if err != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
return nil, "", errors.Wrap(err, "could not write request to server")
|
||||
|
||||
return nil, "", errors.Wrap(err, "could not read response from server")
|
||||
}
|
||||
|
||||
// Only perform matching and writes in case we receive
|
||||
// text or binary opcode from the websocket server.
|
||||
if opCode != ws.OpText && opCode != ws.OpBinary {
|
||||
@@ -339,6 +368,7 @@ func (request *Request) readWriteInputWebsocket(conn net.Conn, payloadValues map
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return inputEvents, reqBuilder.String(), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -92,9 +92,17 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
||||
defaultVars := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil)
|
||||
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
|
||||
// add templatectx variables to varMap
|
||||
vars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.GetTemplateCtx(input.MetaInput).GetAll()))
|
||||
scope := request.options.NewVariablesScope(defaultVars, optionVars, dynamicValues, previous)
|
||||
|
||||
variables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues, request.options.Constants)
|
||||
request.options.AddTemplateCtxToVariablesScope(input.MetaInput, scope)
|
||||
scope.AddData(request.options.Constants)
|
||||
|
||||
vars := request.options.Variables.EvaluateScope(scope).Values
|
||||
variables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues, previous)
|
||||
if request.options.HasTemplateCtx(input.MetaInput) {
|
||||
variables = generators.MergeMaps(variables, request.options.GetTemplateCtx(input.MetaInput).GetAll())
|
||||
}
|
||||
variables = generators.MergeMaps(variables, request.options.Constants)
|
||||
|
||||
if vardump.EnableVarDump {
|
||||
gologger.Debug().Msgf("Whois Protocol request variables: %s\n", vardump.DumpVariables(variables))
|
||||
|
||||
@@ -111,13 +111,21 @@ func (f *FlowExecutor) Compile() error {
|
||||
if f.results == nil {
|
||||
f.results = new(atomic.Bool)
|
||||
}
|
||||
|
||||
// load all variables and evaluate with existing data
|
||||
variableMap := f.options.Variables.Evaluate(f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll())
|
||||
// cli options
|
||||
optionVars := generators.BuildPayloadFromOptions(f.options.Options)
|
||||
// constants
|
||||
constants := f.options.Constants
|
||||
scope := f.options.NewVariablesScope()
|
||||
|
||||
f.options.AddTemplateCtxToVariablesScope(f.ctx.Input.MetaInput, scope)
|
||||
scope.AddData(constants, optionVars)
|
||||
|
||||
evaluation := f.options.Variables.EvaluateScope(scope)
|
||||
variableMap := evaluation.Values
|
||||
allVars := generators.MergeMaps(variableMap, constants, optionVars)
|
||||
|
||||
// we support loading variables from files in variables , cli options and constants
|
||||
// try to load if files exist
|
||||
for k, v := range allVars {
|
||||
@@ -129,7 +137,24 @@ func (f *FlowExecutor) Compile() error {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
templateVariables := make(map[string]interface{}, len(evaluation.TemplateValues))
|
||||
for key := range evaluation.TemplateValues {
|
||||
if _, ok := constants[key]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, ok := optionVars[key]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
if value, ok := allVars[key]; ok {
|
||||
templateVariables[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Merge(allVars) // merge all variables into template context
|
||||
f.options.GetTemplateCtx(f.ctx.Input.MetaInput).MergeTemplateVariables(templateVariables)
|
||||
|
||||
// ---- define callback functions/objects----
|
||||
f.protoFunctions = map[string]func(call goja.FunctionCall, runtime *goja.Runtime) goja.Value{}
|
||||
@@ -139,6 +164,7 @@ func (f *FlowExecutor) Compile() error {
|
||||
reqMap := mapsutil.Map[string, protocols.Request]{}
|
||||
counter := 0
|
||||
proto := strings.ToLower(p) // donot use loop variables in callback functions directly
|
||||
|
||||
for index := range requests {
|
||||
counter++ // start index from 1
|
||||
request := f.allProtocols[proto][index]
|
||||
@@ -146,22 +172,26 @@ func (f *FlowExecutor) Compile() error {
|
||||
// if id is present use it
|
||||
reqMap[request.GetID()] = request
|
||||
}
|
||||
|
||||
// fallback to using index as id
|
||||
// always allow index as id as a fallback
|
||||
reqMap[strconv.Itoa(counter)] = request
|
||||
}
|
||||
|
||||
// ---define hook that allows protocol/request execution from js-----
|
||||
// --- this is the actual callback that is executed when function is invoked in js----
|
||||
f.protoFunctions[proto] = func(call goja.FunctionCall, runtime *goja.Runtime) goja.Value {
|
||||
opts := &ProtoOptions{
|
||||
protoName: proto,
|
||||
}
|
||||
|
||||
for _, v := range call.Arguments {
|
||||
switch value := v.Export().(type) {
|
||||
default:
|
||||
opts.reqIDS = append(opts.reqIDS, types.ToString(value))
|
||||
}
|
||||
}
|
||||
|
||||
// before executing any protocol function flatten tracked values
|
||||
if len(f.flattenKeys) > 0 {
|
||||
ctx := f.options.GetTemplateCtx(f.ctx.Input.MetaInput)
|
||||
@@ -171,9 +201,11 @@ func (f *FlowExecutor) Compile() error {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return runtime.ToValue(f.requestExecutor(runtime, reqMap, opts))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -17,8 +17,11 @@ import (
|
||||
func (f *FlowExecutor) requestExecutor(runtime *goja.Runtime, reqMap mapsutil.Map[string, protocols.Request], opts *ProtoOptions) bool {
|
||||
defer func() {
|
||||
// evaluate all variables after execution of each protocol
|
||||
variableMap := f.options.Variables.Evaluate(f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll())
|
||||
f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Merge(variableMap) // merge all variables into template context
|
||||
scope := f.options.NewVariablesScope()
|
||||
f.options.AddTemplateCtxToVariablesScope(f.ctx.Input.MetaInput, scope)
|
||||
evaluation := f.options.Variables.EvaluateScope(scope)
|
||||
f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Merge(evaluation.Values) // merge all variables into template context
|
||||
f.options.GetTemplateCtx(f.ctx.Input.MetaInput).MergeTemplateVariables(evaluation.TemplateValues)
|
||||
|
||||
// to avoid polling update template variables everytime we execute a protocol
|
||||
m := f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()
|
||||
|
||||
@@ -21,6 +21,7 @@ type MultiProtocol struct {
|
||||
options *protocols.ExecutorOptions
|
||||
results *atomic.Bool
|
||||
readOnlyArgs map[string]interface{} // readOnlyArgs are readonly args that are available after compilation
|
||||
readOnlyVars map[string]interface{} // readOnlyVars are the template-owned subset of readOnlyArgs.
|
||||
}
|
||||
|
||||
// NewMultiProtocol creates a new multiprotocol template engine from a list of requests
|
||||
@@ -33,15 +34,30 @@ func NewMultiProtocol(requests []protocols.Request, options *protocols.ExecutorO
|
||||
|
||||
// Compile engine specific compilation
|
||||
func (m *MultiProtocol) Compile() error {
|
||||
// load all variables and evaluate with existing data
|
||||
variableMap := m.options.Variables.GetAll()
|
||||
// cli options
|
||||
optionVars := generators.BuildPayloadFromOptions(m.options.Options)
|
||||
// constants
|
||||
constants := m.options.Constants
|
||||
allVars := generators.MergeMaps(variableMap, constants, optionVars)
|
||||
allVars = m.options.Variables.Evaluate(allVars)
|
||||
scope := m.options.NewVariablesScope(constants, optionVars)
|
||||
evaluation := m.options.Variables.EvaluateScope(scope)
|
||||
allVars := generators.MergeMaps(evaluation.Values, constants, optionVars)
|
||||
|
||||
templateVariables := make(map[string]interface{}, len(evaluation.TemplateValues))
|
||||
for key := range evaluation.TemplateValues {
|
||||
if _, ok := constants[key]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, ok := optionVars[key]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
if value, ok := allVars[key]; ok {
|
||||
templateVariables[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
m.readOnlyArgs = allVars
|
||||
m.readOnlyVars = templateVariables
|
||||
|
||||
// no need to load files since they are done at template level
|
||||
return nil
|
||||
}
|
||||
@@ -56,6 +72,7 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
|
||||
|
||||
// put all readonly args into template context
|
||||
m.options.GetTemplateCtx(ctx.Input.MetaInput).Merge(m.readOnlyArgs)
|
||||
m.options.GetTemplateCtx(ctx.Input.MetaInput).MergeTemplateVariables(m.readOnlyVars)
|
||||
|
||||
// add all input args to template context
|
||||
ctx.Input.ForEach(func(key string, value interface{}) {
|
||||
@@ -119,9 +136,14 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
|
||||
}
|
||||
|
||||
// evaluate all variables after execution of each protocol
|
||||
variableMap := m.options.Variables.Evaluate(m.options.GetTemplateCtx(ctx.Input.MetaInput).GetAll())
|
||||
m.options.GetTemplateCtx(ctx.Input.MetaInput).Merge(variableMap) // merge all variables into template context
|
||||
scope := m.options.NewVariablesScope()
|
||||
m.options.AddTemplateCtxToVariablesScope(ctx.Input.MetaInput, scope)
|
||||
evaluation := m.options.Variables.EvaluateScope(scope)
|
||||
templateCtx := m.options.GetTemplateCtx(ctx.Input.MetaInput)
|
||||
templateCtx.Merge(evaluation.Values)
|
||||
templateCtx.MergeTemplateVariables(evaluation.TemplateValues)
|
||||
})
|
||||
|
||||
// in case of fatal error skip execution of next protocols
|
||||
if err != nil {
|
||||
// always log errors
|
||||
@@ -135,6 +157,7 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package multiproto
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
)
|
||||
|
||||
func TestCompileStoresDataOnlyReadOnlyArgs(t *testing.T) {
|
||||
optionVars := goflags.RuntimeMap{}
|
||||
require.NoError(t, optionVars.Set("option_key=option-value"))
|
||||
|
||||
templateVars := variables.Variable{
|
||||
InsertionOrderedStringMap: *utils.NewEmptyInsertionOrderedStringMap(2),
|
||||
}
|
||||
templateVars.Set("template_key", "{{constant_key}}-{{option_key}}")
|
||||
templateVars.Set("shadowed", "template-value")
|
||||
|
||||
executerOptions := &protocols.ExecutorOptions{
|
||||
Options: &types.Options{
|
||||
Vars: optionVars,
|
||||
},
|
||||
Constants: map[string]interface{}{
|
||||
"constant_key": "constant-value",
|
||||
"shadowed": "constant-shadow",
|
||||
},
|
||||
Variables: templateVars,
|
||||
}
|
||||
|
||||
m := NewMultiProtocol(nil, executerOptions, nil)
|
||||
|
||||
require.NoError(t, m.Compile())
|
||||
require.Equal(t, "constant-value", m.readOnlyArgs["constant_key"])
|
||||
require.Equal(t, "option-value", m.readOnlyArgs["option_key"])
|
||||
require.Equal(t, "constant-shadow", m.readOnlyArgs["shadowed"])
|
||||
require.Equal(t, "constant-value-option-value", m.readOnlyArgs["template_key"])
|
||||
require.Equal(t, map[string]interface{}{
|
||||
"template_key": "constant-value-option-value",
|
||||
}, m.readOnlyVars)
|
||||
}
|
||||
Reference in new issue
Block a user