package core import ( "fmt" "net/http/cookiejar" "sync" "sync/atomic" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/workflows" syncutil "github.com/projectdiscovery/utils/sync" ) const workflowStepExecutionError = "[%s] Could not execute workflow step: %s\n" // executeWorkflow runs a workflow on an input and returns true or false func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) bool { results := &atomic.Bool{} // at this point we should be at the start root execution of a workflow tree, hence we create global shared instances workflowCookieJar, _ := cookiejar.New(nil) ctxArgs := contextargs.New(ctx.Context()) ctxArgs.MetaInput = ctx.Input.MetaInput ctxArgs.CookieJar = workflowCookieJar // we can know the nesting level only at runtime, so the best we can do here is increase template threads by one unit in case it's equal to 1 to allow // at least one subtemplate to go through, which it's idempotent to one in-flight template as the parent one is in an idle state templateThreads := w.Options.Options.TemplateThreads if templateThreads == 1 { templateThreads++ } swg, _ := syncutil.New(syncutil.WithSize(templateThreads)) // pending tracks every started step until it returns. swg only limits // concurrency and its slots can be released early, so completion is // tracked here instead. var pending sync.WaitGroup for _, template := range w.Workflows { newCtx := scan.NewScanContext(ctx.Context(), ctx.Input.Clone()) // Root steps do not own a waitgroup slot. if err := e.runWorkflowStep(template, newCtx, results, swg, &pending, w, nil); err != nil { gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err) } } pending.Wait() return results.Load() } // slotRelease returns a func that gives back one swg slot. It is safe to call // from several goroutines and several times; only the first call releases. func slotRelease(swg *syncutil.AdaptiveWaitGroup) func() { var once sync.Once return func() { once.Do(swg.Done) } } // runWorkflowStep runs a workflow step for the workflow. It executes the workflow // in a recursive manner running all subtemplates and matchers. // // releaseSlot, when non-nil, releases the AdaptiveWaitGroup slot held by the // goroutine that is running this step. It must be called before this step // blocks on swg.AddWithContext for child steps; otherwise every in-flight // parent can hold a slot while waiting for another, which deadlocks when the // number of matching nested steps reaches TemplateThreads (see #8074). // // pending is incremented for every child step before it starts, and each child // calls Done when it returns, so the root can wait for the whole tree. func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *syncutil.AdaptiveWaitGroup, pending *sync.WaitGroup, w *workflows.Workflow, releaseSlot func()) error { var firstMatched bool var err error var mainErr error if len(template.Matchers) == 0 { for _, executer := range template.Executers { e.syncWorkflowInputToTemplateCtx(executer, ctx.Input) executer.Options.Progress.AddToTotal(int64(executer.Executer.Requests())) // Don't print results with subtemplates, only print results on template. if len(template.Subtemplates) > 0 { ctx.OnResult = func(result *output.InternalWrappedEvent) { if result.OperatorsResult == nil { return } if len(result.Results) > 0 { firstMatched = true } if result.OperatorsResult != nil && result.OperatorsResult.Extracts != nil { for k, v := range result.OperatorsResult.Extracts { // normalize items: switch len(v) { case 0, 1: // - key:[item] => key: item ctx.Input.Set(k, v[0]) default: // - key:[item_0, ..., item_n] => key0:item_0, keyn:item_n for vIdx, vVal := range v { normalizedKIdx := fmt.Sprintf("%s%d", k, vIdx) ctx.Input.Set(normalizedKIdx, vVal) } // also add the original name with full slice ctx.Input.Set(k, v) } } } } _, err = executer.Executer.ExecuteWithResults(ctx) } else { var matched bool matched, err = executer.Executer.Execute(ctx) if matched { firstMatched = true } } if w.Options.HostErrorsCache != nil { w.Options.HostErrorsCache.MarkFailedOrRemove(w.Options.ProtocolType.String(), ctx.Input, err) } if err != nil { if len(template.Executers) == 1 { mainErr = err } else { gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err) } continue } } } if len(template.Subtemplates) == 0 { results.CompareAndSwap(false, firstMatched) } if len(template.Matchers) > 0 { for _, executer := range template.Executers { e.syncWorkflowInputToTemplateCtx(executer, ctx.Input) executer.Options.Progress.AddToTotal(int64(executer.Executer.Requests())) ctx.OnResult = func(event *output.InternalWrappedEvent) { if event.OperatorsResult == nil { return } if event.OperatorsResult.Extracts != nil { for k, v := range event.OperatorsResult.Extracts { ctx.Input.Set(k, v) } } for _, matcher := range template.Matchers { if !matcher.Match(event.OperatorsResult) { continue } // Free this step's slot before blocking on child acquires. if releaseSlot != nil { releaseSlot() } for _, subtemplate := range matcher.Subtemplates { if err := swg.AddWithContext(ctx.Context()); err != nil { return } pending.Add(1) go func(subtemplate *workflows.WorkflowTemplate) { defer pending.Done() release := slotRelease(swg) defer release() // create a new context with the same input but with unset callbacks // clone the Input so that other parallel executions won't overwrite the shared variables when subsequent templates are running subCtx := scan.NewScanContext(ctx.Context(), ctx.Input.Clone()) if err := e.runWorkflowStep(subtemplate, subCtx, results, swg, pending, w, release); err != nil { gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err) } }(subtemplate) } } } _, err := executer.Executer.ExecuteWithResults(ctx) if err != nil { if len(template.Executers) == 1 { mainErr = err } else { gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err) } continue } } return mainErr } if len(template.Subtemplates) > 0 && firstMatched { // Free this step's slot before blocking on child acquires so nested // matching steps cannot exhaust TemplateThreads and deadlock (#8074). if releaseSlot != nil { releaseSlot() } for _, subtemplate := range template.Subtemplates { if err := swg.AddWithContext(ctx.Context()); err != nil { return err } pending.Add(1) go func(template *workflows.WorkflowTemplate) { defer pending.Done() release := slotRelease(swg) defer release() // create a new context with the same input but with unset callbacks subCtx := scan.NewScanContext(ctx.Context(), ctx.Input.Clone()) if err := e.runWorkflowStep(template, subCtx, results, swg, pending, w, release); err != nil { gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err) } }(subtemplate) } } return mainErr } func (e *Engine) syncWorkflowInputToTemplateCtx(executer *workflows.ProtocolExecuterPair, input *contextargs.Context) { if executer == nil || executer.Options == nil || input == nil || !input.HasArgs() { return } workflowValues := input.GetAll() if len(workflowValues) == 0 { return } executer.Options.GetTemplateCtx(input.MetaInput).Merge(workflowValues) }