Compare commits

...
22 changed files with 577 additions and 35 deletions

No files matched your search

+1
View File
@@ -166,6 +166,7 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -164,6 +164,7 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -164,6 +164,7 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -181,6 +181,7 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -164,6 +164,7 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -164,6 +164,7 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -164,6 +164,7 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -164,6 +164,7 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -164,6 +164,7 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -297,6 +297,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", nil, "list of workflow or workflow directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
flagSet.StringVar(&options.PlanFile, "plan", "", "write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target"),
flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all templates matching current filters"),
+30 -7
View File
@@ -119,6 +119,14 @@ func ParseOptions(options *types.Options) {
options.Logger.Debug().Msgf("Store response directory specified, enabling \"store-resp\" flag automatically\n")
options.StoreResponse = true
}
// Derived before validation: -uncover-query turns uncover on, and -plan
// has to reject that before any target lookup starts.
if options.UncoverQuery != nil {
options.Uncover = true
if len(options.UncoverEngine) == 0 {
options.UncoverEngine = append(options.UncoverEngine, "shodan")
}
}
// Validate the options passed by the user and if any
// invalid options have been used, exit.
if err := ValidateOptions(options); err != nil {
@@ -138,13 +146,6 @@ func ParseOptions(options *types.Options) {
_ = os.Setenv("GITHUB_TOKEN", options.GitHubToken)
}
if options.UncoverQuery != nil {
options.Uncover = true
if len(options.UncoverEngine) == 0 {
options.UncoverEngine = append(options.UncoverEngine, "shodan")
}
}
if options.OfflineHTTP {
options.DisableHTTPProbe = true
}
@@ -162,6 +163,9 @@ func ValidateOptions(options *types.Options) error {
}
return errors.Wrap(errors.New(strings.Join(errs, ", ")), "validation failed for these fields")
}
if err := validatePlanOptions(options); err != nil {
return err
}
if options.Verbose && options.Silent {
return errors.New("both verbose and silent mode specified")
}
@@ -536,3 +540,22 @@ func getBoolEnvValue(key string) bool {
value := os.Getenv(key)
return strings.EqualFold(value, "true")
}
// validatePlanOptions rejects options that change the templates or targets
// only once scanning starts, which a plan written beforehand cannot show.
func validatePlanOptions(options *types.Options) error {
if options.PlanFile == "" {
return nil
}
switch {
case options.AutomaticScan:
return errors.New("-plan cannot be used with -automatic-scan, which picks templates while scanning")
case options.Uncover || len(options.UncoverQuery) > 0:
return errors.New("-plan cannot be used with -uncover, which finds targets over the network")
case options.PreflightPortScan:
return errors.New("-plan cannot be used with -preflight-portscan, which drops targets after scanning their ports")
case options.DASTServer || options.DASTProxy:
return errors.New("-plan cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan")
}
return nil
}
+91
View File
@@ -0,0 +1,91 @@
package runner
import (
"bufio"
"os"
"slices"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
"github.com/projectdiscovery/nuclei/v3/pkg/core"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
)
// httpRequestSeconds is the time of one http request the plan costs are
// weighted against (see core.PlanEntry.Cost).
const httpRequestSeconds = 0.015
// planSummary is the last line of a plan file, written once every entry is
// known.
type planSummary struct {
Entries int `json:"entries"`
Requests int `json:"requests"`
Cost int `json:"cost"`
// EstimatedSeconds is the largest of the time the rate limit allows, the
// cost spread over every concurrent slot, and the longest entry, whose
// requests run one after another. Hosts slower than the measured request
// time make the scan take longer.
EstimatedSeconds int `json:"estimated-seconds"`
}
// writePlan writes one JSON line per unit of work the scan would run,
// followed by the summary. Targets are listed as given: inputs that the scan
// would probe to an http url first are not probed here, since probing sends
// requests.
func (r *Runner) writePlan(store *loader.Store, engine *core.Engine) error {
if !r.options.DisableHTTPProbe && loader.IsHTTPBasedProtocolUsed(store) && r.isInputNonHTTP() {
r.Logger.Info().Msgf("Targets without a scheme are planned as given; the scan probes them for http(s) first")
}
file, err := os.Create(r.options.PlanFile)
if err != nil {
return err
}
defer func() { _ = file.Close() }()
writer := bufio.NewWriter(file)
encoder := json.NewEncoder(writer)
var summary planSummary
var headlessCost, longest int
var encodeErr error
engine.Plan(slices.Concat(store.Templates(), store.Workflows()), r.inputProvider, r.options.DisableClustering, func(entry core.PlanEntry) {
if encodeErr != nil {
return
}
encodeErr = encoder.Encode(entry)
summary.Entries++
summary.Requests += entry.Requests
summary.Cost += entry.Cost
longest = max(longest, entry.Cost)
if entry.Protocol == types.HeadlessProtocol.String() {
headlessCost += entry.Cost
}
})
if encodeErr != nil {
return encodeErr
}
summary.EstimatedSeconds = r.estimateSeconds(summary.Requests, summary.Cost-headlessCost, headlessCost, longest)
if err := encoder.Encode(map[string]planSummary{"summary": summary}); err != nil {
return err
}
if err := writer.Flush(); err != nil {
return err
}
r.Logger.Info().Msgf("Plan written to %s: %d template runs, up to %d requests, about %ds", r.options.PlanFile, summary.Entries, summary.Requests, summary.EstimatedSeconds)
return nil
}
// estimateSeconds estimates the scan duration. Headless templates run in
// their own, smaller pool, so their cost is spread over it separately.
func (r *Runner) estimateSeconds(requests, cost, headlessCost, longest int) int {
seconds := float64(longest) * httpRequestSeconds
if r.options.RateLimit > 0 && r.options.RateLimitDuration > 0 {
seconds = max(seconds, float64(requests)/(float64(r.options.RateLimit)/r.options.RateLimitDuration.Seconds()))
}
if slots := r.options.TemplateThreads * r.options.BulkSize; slots > 0 {
seconds = max(seconds, float64(cost)*httpRequestSeconds/float64(slots))
}
if slots := r.options.HeadlessTemplateThreads * r.options.HeadlessBulkSize; slots > 0 {
seconds = max(seconds, float64(headlessCost)*httpRequestSeconds/float64(slots))
}
return int(seconds + 0.5)
}
+26
View File
@@ -0,0 +1,26 @@
package runner
import (
"testing"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestPlanRejectsOptionsThatChangeTargetsLater(t *testing.T) {
for name, options := range map[string]*types.Options{
"automatic scan": {PlanFile: "p", AutomaticScan: true},
"uncover": {PlanFile: "p", Uncover: true},
"uncover query": {PlanFile: "p", UncoverQuery: goflags.StringSlice{"example"}},
"preflight": {PlanFile: "p", PreflightPortScan: true},
"dast server": {PlanFile: "p", DASTServer: true},
"dast proxy": {PlanFile: "p", DASTProxy: true},
} {
require.Error(t, validatePlanOptions(options), name)
}
require.NoError(t, validatePlanOptions(&types.Options{PlanFile: "p"}))
require.NoError(t, validatePlanOptions(&types.Options{Uncover: true}), "only -plan restricts them")
require.NoError(t, validatePlanOptions(&types.Options{UncoverQuery: goflags.StringSlice{"example"}}))
require.NoError(t, validatePlanOptions(&types.Options{DASTServer: true}))
}
+24
View File
@@ -0,0 +1,24 @@
package runner
import (
"testing"
"time"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestEstimateSecondsTakesTheSlowestBound(t *testing.T) {
r := &Runner{options: &types.Options{
RateLimit: 150, RateLimitDuration: time.Second,
TemplateThreads: 25, BulkSize: 25,
HeadlessTemplateThreads: 10, HeadlessBulkSize: 10,
}}
require.Equal(t, 20, r.estimateSeconds(3000, 3000, 0, 12), "rate limit bound")
require.Equal(t, 10, r.estimateSeconds(52, 41, 660, 660), "one long entry runs its requests in sequence")
require.Equal(t, 9, r.estimateSeconds(100, 0, 60000, 60), "headless cost spreads over the headless pool")
r.options.RateLimit = 0
require.Equal(t, 0, r.estimateSeconds(3000, 3000, 0, 1), "no rate limit, cost spread over 625 slots")
}
+29 -9
View File
@@ -146,10 +146,18 @@ func New(options *types.Options) (*Runner, error) {
// Version check by default
if config.DefaultConfig.CanCheckForUpdates() {
latestIgnoreHash, versionCheckErr := installer.NucleiVersionCheck()
if versionCheckErr != nil {
if options.Verbose || options.Debug {
runner.Logger.Error().Msgf("nuclei version check failed got: %s\n", versionCheckErr)
// -plan sends no request it can avoid, so it skips the version and
// update checks; it still installs templates on a fresh setup, which
// would otherwise plan nothing
checkUpdates := options.PlanFile == ""
var latestIgnoreHash string
if checkUpdates {
var versionCheckErr error
latestIgnoreHash, versionCheckErr = installer.NucleiVersionCheck()
if versionCheckErr != nil {
if options.Verbose || options.Debug {
runner.Logger.Error().Msgf("nuclei version check failed got: %s\n", versionCheckErr)
}
}
}
@@ -170,17 +178,19 @@ func New(options *types.Options) (*Runner, error) {
if err := tm.FreshInstallIfNotExists(); err != nil {
runner.Logger.Warning().Msgf("Failed to install nuclei templates: %s\n", err)
}
if err := tm.UpdateIfOutdated(); err != nil {
runner.Logger.Warning().Msgf("Failed to update nuclei templates: %s\n", err)
if checkUpdates {
if err := tm.UpdateIfOutdated(); err != nil {
runner.Logger.Warning().Msgf("Failed to update nuclei templates: %s\n", err)
}
}
if config.DefaultConfig.IgnoreFileNeedsUpdate(latestIgnoreHash) {
if checkUpdates && config.DefaultConfig.IgnoreFileNeedsUpdate(latestIgnoreHash) {
if err := installer.UpdateIgnoreFile(); err != nil {
runner.Logger.Warning().Msgf("Failed to update .nuclei-ignore file: %s\n", err)
}
}
if options.UpdateTemplates {
if checkUpdates && options.UpdateTemplates {
// we automatically check for updates unless explicitly disabled
// this print statement is only to inform the user that there are no updates
noUpdates := !config.DefaultConfig.NeedsTemplateUpdate()
@@ -591,8 +601,13 @@ func (r *Runner) RunEnumeration() error {
httpclientpool.ResetConnectionStats()
// If the user has asked for DAST server mode, run the live
// DAST fuzzing server.
// DAST fuzzing server. -plan never reaches this with those flags set:
// validation rejects the combination, and this check covers a caller
// that skipped it.
if r.options.DASTServer || r.options.DASTProxy {
if err := validatePlanOptions(r.options); err != nil {
return err
}
execurOpts := &server.NucleiExecutorOptions{
Options: r.options,
Output: r.output,
@@ -815,6 +830,11 @@ func (r *Runner) RunEnumeration() error {
profiles.Prepare(slices.Concat(store.Templates(), store.Workflows()))
executorOpts.TargetScope = profiles
}
// before uncover, preflight, auth prefetch and probing below, all of which
// reach the network
if r.options.PlanFile != "" {
return r.writePlan(store, executorEngine)
}
// TODO: remove below functions after v3 or update warning messages
templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
+15 -13
View File
@@ -38,19 +38,7 @@ func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templ
totalReqBeforeCluster := getRequestCount(templatesList) * int(target.Count())
// attempt to cluster templates if noCluster is false
var finalTemplates []*templates.Template
clusterCount := 0
if !noCluster {
var clusterMappings map[string][]string
finalTemplates, clusterCount, clusterMappings = templates.ClusterTemplates(templatesList, e.executerOpts)
// Store cluster mappings in executerOpts for SDK access (thread-safe)
if clusterMappings != nil {
e.executerOpts.ClusterMappings = types.NewClusterMappingsMap(clusterMappings)
}
} else {
finalTemplates = templatesList
}
finalTemplates, clusterCount := e.clusterTemplates(templatesList, noCluster)
totalReqAfterClustering := getRequestCount(finalTemplates) * int(target.Count())
@@ -186,6 +174,20 @@ func (e *Engine) executeHostSpray(ctx context.Context, templatesList []*template
return results
}
// clusterTemplates merges templates that send identical requests unless
// noCluster is set, and returns the templates to run with the number merged.
func (e *Engine) clusterTemplates(templatesList []*templates.Template, noCluster bool) ([]*templates.Template, int) {
if noCluster {
return templatesList, 0
}
finalTemplates, clusterCount, clusterMappings := templates.ClusterTemplates(templatesList, e.executerOpts)
// Store cluster mappings in executerOpts for SDK access (thread-safe)
if clusterMappings != nil {
e.executerOpts.ClusterMappings = types.NewClusterMappingsMap(clusterMappings)
}
return finalTemplates, clusterCount
}
// expectedWork returns the requests and the tasks the target-bound templates
// will run. It counts only the template and target pairs that per-target
// profiles select, so progress reflects the work that will be done. A task is
+15 -6
View File
@@ -382,6 +382,19 @@ func newOriginAttempt() *originAttempt {
return attempt
}
// originScopeKey returns the key an origin-scoped template runs once per, and
// false when the template runs on every target.
func originScopeKey(template *templates.Template, value *contextargs.MetaInput) (string, bool) {
if template == nil || value == nil || !template.IsOriginScoped() {
return "", false
}
origin := normalize.Origin(value.Input)
if origin == "" {
return "", false
}
return template.ID + "\x00" + origin + "\x00" + value.CustomIP, true
}
// claimOriginScoped reports whether this caller should execute an origin-scoped
// template. finish(true) records a successful run. finish(false) releases the
// claim so a later target, including one that waited, can try again. Templates
@@ -389,14 +402,10 @@ func newOriginAttempt() *originAttempt {
// key so each address selected for one URL is tested on its own.
func (e *Engine) claimOriginScoped(template *templates.Template, value *contextargs.MetaInput) (finish func(bool), run bool) {
noop := func(bool) {}
if template == nil || value == nil || !template.IsOriginScoped() {
key, scoped := originScopeKey(template, value)
if !scoped {
return noop, true
}
origin := normalize.Origin(value.Input)
if origin == "" {
return noop, true
}
key := template.ID + "\x00" + origin + "\x00" + value.CustomIP
actual, _ := e.originScoped.LoadOrStore(key, newOriginAttempt())
slot := actual.(*originAttempt)
+159
View File
@@ -0,0 +1,159 @@
package core
import (
"crypto/sha256"
"encoding/hex"
"slices"
"strings"
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
)
// PlanEntry is one unit of work a scan runs: a template, a cluster of
// templates sharing one request, or a workflow, on one target.
type PlanEntry struct {
// ID identifies the entry across runs of the same plan.
ID string `json:"id"`
// Target is empty for self-contained templates, which run once.
Target string `json:"target,omitempty"`
// IP is the address the target is pinned to, when it is.
IP string `json:"ip,omitempty"`
Templates []string `json:"templates"`
Protocol string `json:"protocol"`
// Requests is what the templates declare, the count -stats starts from.
// The scan sends fewer when a request needs a value the target lacks,
// such as {{DN}} on an ip.
Requests int `json:"requests"`
// Cost is Requests weighted by how long one request of the protocol
// takes, in http requests.
Cost int `json:"cost"`
// Dynamic marks entries whose requests depend on responses: workflows
// (only the root is counted), flows and fuzzing.
Dynamic bool `json:"dynamic,omitempty"`
// Fallbacks are later targets of the same origin, in scan order. The
// scan tries the next one only when this entry's target fails, so a plan
// keeps them instead of dropping the retry.
Fallbacks []string `json:"fallbacks,omitempty"`
}
// Plan reports the work ExecuteScanWithOpts would do with the same arguments,
// without sending anything. It applies the same clustering, per-target
// profiles and once-per-origin rule, so a pair is listed only if the scan
// would run it. Later targets of an origin-scoped template are kept on the
// first entry as fallbacks: the scan runs the next one when the earlier
// target fails.
func (e *Engine) Plan(templatesList []*templates.Template, target provider.InputProvider, noCluster bool, emit func(PlanEntry)) {
finalTemplates, _ := e.clusterTemplates(templatesList, noCluster)
// clustering does not keep an order; a plan is diffed and edited, so it does
finalTemplates = slices.SortedFunc(slices.Values(finalTemplates), func(a, b *templates.Template) int {
return strings.Compare(a.ID, b.ID)
})
var targetBound []*templates.Template
for _, template := range finalTemplates {
if template.SelfContained {
emit(withPlanID(planEntry(template, &contextargs.MetaInput{}, nil)))
} else {
targetBound = append(targetBound, template)
}
}
// Buffered so a later target of the same origin can be recorded on the
// entry before its id is derived.
var planned []PlanEntry
originIndex := make(map[string]int)
target.Iterate(func(value *contextargs.MetaInput) bool {
selection := e.selectionFor(value)
for _, template := range targetBound {
if selection != nil && !template.SelectedBy(selection) {
continue
}
if key, scoped := originScopeKey(template, value); scoped {
if idx, seen := originIndex[key]; seen {
planned[idx].Fallbacks = append(planned[idx].Fallbacks, value.Input)
continue
}
originIndex[key] = len(planned)
}
planned = append(planned, planEntry(template, value, selection))
}
return true
})
for _, entry := range planned {
emit(withPlanID(entry))
}
}
func withPlanID(entry PlanEntry) PlanEntry {
entry.ID = planEntryID(entry)
return entry
}
// protocolCost is how many http requests one request of a protocol is worth
// in time. Measured per call against a CDN-fronted site, where an http
// request took about 15ms: a javascript call adds about 150ms of runtime, a
// headless page load with waitload took about 1s. Unlisted protocols cost
// about one round trip, like http.
var protocolCost = map[types.ProtocolType]int{
types.JavascriptProtocol: 10,
types.HeadlessProtocol: 60,
}
func planEntry(template *templates.Template, value *contextargs.MetaInput, selection protocols.TemplateSelection) PlanEntry {
entry := PlanEntry{
Target: value.Input,
IP: value.CustomIP,
Templates: []string{template.ID},
Protocol: template.Type().String(),
Requests: template.TotalRequests,
Dynamic: template.Flow != "" || template.IsFuzzableRequest(),
}
weight := func(protocol types.ProtocolType) int { return max(protocolCost[protocol], 1) }
switch {
case template.CompiledWorkflow != nil:
// only the root of a workflow is certain to run; subtemplates depend
// on what it matches
entry.Requests = 0
for _, workflow := range template.CompiledWorkflow.Workflows {
for _, executer := range workflow.Executers {
requests := executer.Executer.Requests()
entry.Requests += requests
entry.Cost += requests * weight(executer.TemplateType)
}
}
entry.Dynamic = true
case isCluster(template):
cluster := template.Executer.(*templates.ClusterExecuter)
entry.Templates = slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
// TotalRequests of a cluster counts only its http and dns requests,
// so an ssl cluster would show none
entry.Requests = cluster.Requests()
entry.Cost = entry.Requests * weight(template.Type())
case len(template.RequestsQueue) > 0:
// each block is weighted by its own protocol, so a template mixing
// http with javascript is not priced as all http
for _, request := range template.RequestsQueue {
entry.Cost += request.Requests() * weight(request.Type())
}
default:
entry.Cost = entry.Requests * weight(template.Type())
}
return entry
}
func isCluster(template *templates.Template) bool {
_, ok := template.Executer.(*templates.ClusterExecuter)
return ok
}
// planEntryID derives the id from what the entry runs, so planning the same
// scan again yields the same ids.
func planEntryID(entry PlanEntry) string {
sum := sha256.Sum256([]byte(entry.Target + "\x00" + entry.IP + "\x00" + strings.Join(entry.Templates, "\x00") + "\x00" + strings.Join(entry.Fallbacks, "\x00")))
return hex.EncodeToString(sum[:8])
}
+155
View File
@@ -0,0 +1,155 @@
package core
import (
"context"
"sync"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
tmpltypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestPlanEntryCostWeightsByProtocol(t *testing.T) {
headless := &templates.Template{ID: "h", RequestsHeadless: []*headless.Request{{}}, TotalRequests: 2}
require.Equal(t, 120, planEntry(headless, &contextargs.MetaInput{Input: "t"}, nil).Cost)
plain := &templates.Template{ID: "p", TotalRequests: 2}
require.Equal(t, 2, planEntry(plain, &contextargs.MetaInput{Input: "t"}, nil).Cost, "unlisted protocols cost one request each")
}
// countedRequest is a request block with a fixed protocol and count.
type countedRequest struct {
protocols.Request
protocol tmpltypes.ProtocolType
requests int
}
func (r countedRequest) Type() tmpltypes.ProtocolType { return r.protocol }
func (r countedRequest) Requests() int { return r.requests }
func TestPlanEntryWeightsEachProtocolBlock(t *testing.T) {
// one http request and ten javascript calls: Type() reports http for the
// whole template, which would price it as 11
mixed := &templates.Template{
ID: "mixed",
RequestsHTTP: []*http.Request{{}},
TotalRequests: 11,
RequestsQueue: []protocols.Request{
countedRequest{protocol: tmpltypes.HTTPProtocol, requests: 1},
countedRequest{protocol: tmpltypes.JavascriptProtocol, requests: 10},
},
}
entry := planEntry(mixed, &contextargs.MetaInput{Input: "t"}, nil)
require.Equal(t, 11, entry.Requests)
require.Equal(t, 101, entry.Cost)
}
func TestPlanListsThePairsTheScanRuns(t *testing.T) {
options := &types.Options{BulkSize: 2, TemplateThreads: 2, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
engine := New(options)
engine.SetExecuterOptions(&protocols.ExecutorOptions{
Logger: engine.Logger,
Options: options,
ResumeCfg: types.NewResumeCfg(),
ProtocolType: tmpltypes.HTTPProtocol,
Progress: &progressRecorder{},
TargetScope: scopeByInput{
"php": {paths: []string{"thinkphp.yaml"}},
"none": {},
},
})
var (
mu sync.Mutex
runs []string
)
var templatesList []*templates.Template
for _, path := range []string{"thinkphp.yaml", "generic.yaml"} {
templatesList = append(templatesList, &templates.Template{
ID: path,
Path: path,
TotalRequests: 3,
Executer: &recordingExecuter{path: path, mu: &mu, runs: &runs},
})
}
selfContained := &templates.Template{ID: "self.yaml", Path: "self.yaml", SelfContained: true, TotalRequests: 1}
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "php"}, {Input: "none"}, {Input: "plain"}}}
var plan []PlanEntry
engine.Plan(append(templatesList, selfContained), targets, true, func(entry PlanEntry) { plan = append(plan, entry) })
require.Empty(t, plan[0].Target, "a self-contained template runs once, with no target")
require.Equal(t, []string{"self.yaml"}, plan[0].Templates)
var planned []string
ids := make(map[string]struct{})
for _, entry := range plan[1:] {
require.Equal(t, 3, entry.Requests)
planned = append(planned, entry.Templates[0]+"@"+entry.Target)
ids[entry.ID] = struct{}{}
}
require.Len(t, ids, len(plan)-1, "every entry has its own id")
var again []PlanEntry
engine.Plan(append(templatesList, selfContained), targets, true, func(entry PlanEntry) { again = append(again, entry) })
require.Equal(t, plan, again, "planning the same scan again yields the same entries and ids")
engine.ExecuteScanWithOpts(context.Background(), templatesList, targets, true)
require.ElementsMatch(t, runs, planned, "the plan lists exactly the pairs the scan runs")
require.ElementsMatch(t, []string{"thinkphp.yaml@php", "thinkphp.yaml@plain", "generic.yaml@plain"}, planned)
}
func TestPlanKeepsOriginFallbacks(t *testing.T) {
options := &types.Options{BulkSize: 1, TemplateThreads: 1, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
engine := New(options)
engine.SetExecuterOptions(&protocols.ExecutorOptions{
Logger: engine.Logger,
Options: options,
ResumeCfg: types.NewResumeCfg(),
ProtocolType: tmpltypes.HTTPProtocol,
Progress: &progressRecorder{},
})
root := &templates.Template{
ID: "root.yaml",
RequestsHTTP: []*http.Request{{Path: []string{"{{RootURL}}/health"}}},
TotalRequests: 1,
}
require.True(t, root.IsOriginScoped())
generic := &templates.Template{ID: "generic.yaml", TotalRequests: 1}
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{
{Input: "https://example.com/a"},
{Input: "https://example.com/b"},
{Input: "https://other.example/a"},
{Input: "https://example.com/a", CustomIP: "203.0.113.5"},
}}
var plan []PlanEntry
engine.Plan([]*templates.Template{generic, root}, targets, true, func(entry PlanEntry) {
plan = append(plan, entry)
})
var roots []PlanEntry
for _, entry := range plan {
if entry.Templates[0] == "root.yaml" {
roots = append(roots, entry)
}
}
require.Equal(t, []PlanEntry{
{ID: roots[0].ID, Target: "https://example.com/a", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1, Fallbacks: []string{"https://example.com/b"}},
{ID: roots[1].ID, Target: "https://other.example/a", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
{ID: roots[2].ID, Target: "https://example.com/a", IP: "203.0.113.5", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
}, roots)
require.NotEqual(t, roots[0].ID, roots[2].ID, "a pinned address is its own attempt")
var again []PlanEntry
engine.Plan([]*templates.Template{generic, root}, targets, true, func(entry PlanEntry) {
again = append(again, entry)
})
require.Equal(t, plan, again)
}
+12
View File
@@ -289,6 +289,18 @@ func (template *Template) SelectedBy(selection protocols.TemplateSelection) bool
return selection.Allows(template.Path)
}
// TemplateIDs returns the IDs of the members selection selects; a nil
// selection selects every member.
func (e *ClusterExecuter) TemplateIDs(selection protocols.TemplateSelection) []string {
ids := make([]string, 0, len(e.operators))
for _, operator := range e.operators {
if selection == nil || selection.Allows(operator.templatePath) {
ids = append(ids, operator.templateID)
}
}
return ids
}
// operatorsFor returns the members selected for input by per-target profiles.
func (e *ClusterExecuter) operatorsFor(input *contextargs.MetaInput) []*clusteredOperator {
if e.options.TargetScope == nil {
+4
View File
@@ -251,6 +251,9 @@ type Options struct {
Silent bool
// Validate validates the templates passed to nuclei.
Validate bool
// PlanFile is where the work a scan would do is written as JSON lines;
// nuclei exits after writing it, without sending requests.
PlanFile string
// NoStrictSyntax disables strict syntax check on nuclei templates (allows custom key-value pairs).
NoStrictSyntax bool
// Verbose flag indicates whether to show verbose output or not
@@ -707,6 +710,7 @@ func (options *Options) Copy() *Options {
AutomaticScan: options.AutomaticScan,
Silent: options.Silent,
Validate: options.Validate,
PlanFile: options.PlanFile,
NoStrictSyntax: options.NoStrictSyntax,
Verbose: options.Verbose,
VerboseVerbose: options.VerboseVerbose,
+7
View File
@@ -17,6 +17,13 @@ func TestCopyPreservesPayloadRandomOrder(t *testing.T) {
}
}
func TestCopyPreservesPlanFile(t *testing.T) {
options := &Options{PlanFile: "plan.jsonl"}
if options.Copy().PlanFile != "plan.jsonl" {
t.Fatal("Copy must preserve PlanFile, or an embedded caller loses plan mode")
}
}
func TestToJSONMapRejectsKeysSharingAStringForm(t *testing.T) {
metadata := map[string]interface{}{
"labels": []interface{}{map[interface{}]interface{}{1: "int", 1.0: "float64"}},