reject flags

README.md

@@ -167,6 +167,7 @@ TEMPLATES:
    -wurl, -workflow-url string[]          workflow url or list containing workflow urls to run (comma-separated, file)
    -validate                              validate the passed templates to nuclei
    -plan string                           write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
+   -run-plan string                       run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content
    -tl                                    list all templates matching current filters

README_CN.md

@@ -165,6 +165,7 @@ TEMPLATES:
    -wurl, -workflow-url string[]          workflow url or list containing workflow urls to run (comma-separated, file)
    -validate                              validate the passed templates to nuclei
    -plan string                           write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
+   -run-plan string                       run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content
    -tl                                    list all templates matching current filters

README_ES.md

@@ -165,6 +165,7 @@ TEMPLATES:
    -wurl, -workflow-url string[]          workflow url or list containing workflow urls to run (comma-separated, file)
    -validate                              validate the passed templates to nuclei
    -plan string                           write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
+   -run-plan string                       run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content
    -tl                                    list all templates matching current filters

README_FA.md

@@ -182,6 +182,7 @@ TEMPLATES:
    -wurl, -workflow-url string[]          workflow url or list containing workflow urls to run (comma-separated, file)
    -validate                              validate the passed templates to nuclei
    -plan string                           write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
+   -run-plan string                       run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content
    -tl                                    list all templates matching current filters

README_ID.md

@@ -165,6 +165,7 @@ TEMPLATES:
    -wurl, -workflow-url string[]          workflow url or list containing workflow urls to run (comma-separated, file)
    -validate                              validate the passed templates to nuclei
    -plan string                           write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
+   -run-plan string                       run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content
    -tl                                    list all templates matching current filters

README_JP.md

@@ -165,6 +165,7 @@ TEMPLATES:
    -wurl, -workflow-url string[]          workflow url or list containing workflow urls to run (comma-separated, file)
    -validate                              validate the passed templates to nuclei
    -plan string                           write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
+   -run-plan string                       run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content
    -tl                                    list all templates matching current filters

README_KR.md

@@ -165,6 +165,7 @@ TEMPLATES:
    -wurl, -workflow-url string[]          workflow url or list containing workflow urls to run (comma-separated, file)
    -validate                              validate the passed templates to nuclei
    -plan string                           write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
+   -run-plan string                       run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content
    -tl                                    list all templates matching current filters

README_PT-BR.md

@@ -165,6 +165,7 @@ TEMPLATES:
    -wurl, -workflow-url string[]          workflow url or list containing workflow urls to run (comma-separated, file)
    -validate                              validate the passed templates to nuclei
    -plan string                           write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
+   -run-plan string                       run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content
    -tl                                    list all templates matching current filters

README_TR.md

@@ -165,6 +165,7 @@ TEMPLATES:
    -wurl, -workflow-url string[]          workflow url or list containing workflow urls to run (comma-separated, file)
    -validate                              validate the passed templates to nuclei
    -plan string                           write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
+   -run-plan string                       run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content
    -tl                                    list all templates matching current filters

cmd/nuclei/main.go

@@ -314,6 +314,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
 		flagSet.StringVar(&options.PlanFile, "plan", "", "write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target"),
+		flagSet.StringVar(&options.RunPlanFile, "run-plan", "", "run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes"),
 		flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
 		flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
 		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all templates matching current filters"),

internal/runner/options.go

@@ -557,18 +557,31 @@ func getBoolEnvValue(key string) bool {
 // validatePlanOptions rejects options that change the templates or targets
 // only once scanning starts, which a plan written beforehand cannot show.
 func validatePlanOptions(options *types.Options) error {
-	if options.PlanFile == "" {
+	if options.RunPlanFile != "" && options.PlanFile != "" {
+		return errors.New("-plan cannot be used with -run-plan")
+	}
+	if options.RunPlanFile != "" && options.Resume != "" {
+		return errors.New("-run-plan cannot be used with -resume, the plan's done log resumes it")
+	}
+	if options.PlanFile == "" && options.RunPlanFile == "" {
 		return nil
 	}
+	// -automatic-scan builds its own engine, so a plan set on the main engine
+	// would never be consulted. The other flags change the targets only after
+	// a plan has been written.
+	flag := "-plan"
+	if options.PlanFile == "" {
+		flag = "-run-plan"
+	}
 	switch {
 	case options.AutomaticScan:
-		return errors.New("-plan cannot be used with -automatic-scan, which picks templates while scanning")
+		return fmt.Errorf("%s cannot be used with -automatic-scan, which picks templates while scanning", flag)
 	case options.Uncover || len(options.UncoverQuery) > 0:
-		return errors.New("-plan cannot be used with -uncover, which finds targets over the network")
+		return fmt.Errorf("%s cannot be used with -uncover, which finds targets over the network", flag)
 	case options.PreflightPortScan:
-		return errors.New("-plan cannot be used with -preflight-portscan, which drops targets after scanning their ports")
+		return fmt.Errorf("%s cannot be used with -preflight-portscan, which drops targets after scanning their ports", flag)
 	case options.DASTServer || options.DASTProxy:
-		return errors.New("-plan cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan")
+		return fmt.Errorf("%s cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan", flag)
 	}
 	return nil
 }

internal/runner/plan_options_test.go

@@ -23,4 +23,19 @@ func TestPlanRejectsOptionsThatChangeTargetsLater(t *testing.T) {
 	require.NoError(t, validatePlanOptions(&types.Options{Uncover: true}), "only -plan restricts them")
 	require.NoError(t, validatePlanOptions(&types.Options{UncoverQuery: goflags.StringSlice{"example"}}))
 	require.NoError(t, validatePlanOptions(&types.Options{DASTServer: true}))
+
+	require.Error(t, validatePlanOptions(&types.Options{RunPlanFile: "p", PlanFile: "q"}))
+	require.Error(t, validatePlanOptions(&types.Options{RunPlanFile: "p", Resume: "r"}))
+	require.NoError(t, validatePlanOptions(&types.Options{RunPlanFile: "p"}))
+
+	for name, options := range map[string]*types.Options{
+		"run automatic scan": {RunPlanFile: "p", AutomaticScan: true},
+		"run uncover":        {RunPlanFile: "p", Uncover: true},
+		"run uncover query":  {RunPlanFile: "p", UncoverQuery: goflags.StringSlice{"example"}},
+		"run preflight":      {RunPlanFile: "p", PreflightPortScan: true},
+		"run dast server":    {RunPlanFile: "p", DASTServer: true},
+		"run dast proxy":     {RunPlanFile: "p", DASTProxy: true},
+	} {
+		require.Error(t, validatePlanOptions(options), name)
+	}
 }

internal/runner/run_plan.go

@@ -0,0 +1,215 @@
+package runner
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"sync"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
+)
+
+// planRun limits a scan to the entries of a plan file and appends the id of
+// each finished entry to the plan's done log, so running the plan again skips
+// them. Ids are derived from what an entry runs, not from its position, so the
+// log still applies after the plan is written again for changed targets or
+// templates.
+type planRun struct {
+	mu      sync.Mutex
+	entries map[uint64]plannedEntry
+	// done holds finished origin groups by the id of their first entry; an
+	// entry outside a group is its own group
+	done   map[uint64]struct{}
+	log    *os.File
+	logErr error
+	// resumed counts the entries finished before this run
+	resumed int
+}
+
+type plannedEntry struct {
+	group uint64
+	seen  bool
+}
+
+func planDoneLog(planFile string) string {
+	return planFile + ".done"
+}
+
+func openPlanRun(planFile string) (*planRun, error) {
+	run := &planRun{entries: make(map[uint64]plannedEntry), done: make(map[uint64]struct{})}
+	if err := run.readPlan(planFile); err != nil {
+		return nil, err
+	}
+	if err := run.readDoneLog(planDoneLog(planFile)); err != nil {
+		return nil, err
+	}
+	log, err := os.OpenFile(planDoneLog(planFile), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o644)
+	if err != nil {
+		return nil, err
+	}
+	// an interrupted write leaves a partial line, which the next id must not
+	// extend
+	if info, err := log.Stat(); err == nil && info.Size() > 0 {
+		last := make([]byte, 1)
+		if _, err := log.ReadAt(last, info.Size()-1); err == nil && last[0] != '\n' {
+			_, _ = log.WriteString("\n")
+		}
+	}
+	run.log = log
+	return run, nil
+}
+
+func (run *planRun) readPlan(planFile string) error {
+	file, err := os.Open(planFile)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = file.Close() }()
+	reader := bufio.NewReader(file)
+	for number := 1; ; number++ {
+		line, readErr := reader.ReadBytes('\n')
+		if line = bytes.TrimSpace(line); len(line) > 0 {
+			var entry struct {
+				ID            string `json:"id"`
+				AlternativeTo string `json:"alternative-to"`
+			}
+			if err := json.Unmarshal(line, &entry); err != nil {
+				return fmt.Errorf("%s:%d: %w", planFile, number, err)
+			}
+			// the summary line has no id
+			if entry.ID != "" {
+				id, ok := parsePlanID(entry.ID)
+				if !ok {
+					return fmt.Errorf("%s:%d: invalid id %q", planFile, number, entry.ID)
+				}
+				group := id
+				if entry.AlternativeTo != "" {
+					if group, ok = parsePlanID(entry.AlternativeTo); !ok {
+						return fmt.Errorf("%s:%d: invalid alternative-to %q", planFile, number, entry.AlternativeTo)
+					}
+				}
+				run.entries[id] = plannedEntry{group: group}
+			}
+		}
+		if readErr == io.EOF {
+			return nil
+		}
+		if readErr != nil {
+			return readErr
+		}
+	}
+}
+
+func (run *planRun) readDoneLog(path string) error {
+	file, err := os.Open(path)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	defer func() { _ = file.Close() }()
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		// a partial line from an interrupted write does not parse and is
+		// run again
+		id, ok := parsePlanID(scanner.Text())
+		if !ok {
+			continue
+		}
+		if entry, planned := run.entries[id]; planned {
+			run.done[entry.group] = struct{}{}
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+	for _, entry := range run.entries {
+		if _, done := run.done[entry.group]; done {
+			run.resumed++
+		}
+	}
+	return nil
+}
+
+// parsePlanID keeps ids as integers, which a plan of millions of entries
+// holds in a fraction of the memory of strings.
+func parsePlanID(id string) (uint64, bool) {
+	if len(id) != 16 {
+		return 0, false
+	}
+	value, err := strconv.ParseUint(id, 16, 64)
+	return value, err == nil
+}
+
+func (run *planRun) Skip(id string) bool {
+	key, ok := parsePlanID(id)
+	if !ok {
+		return true
+	}
+	run.mu.Lock()
+	defer run.mu.Unlock()
+	entry, planned := run.entries[key]
+	if !planned {
+		return true
+	}
+	if !entry.seen {
+		entry.seen = true
+		run.entries[key] = entry
+	}
+	_, done := run.done[entry.group]
+	return done
+}
+
+func (run *planRun) Finished(id string) {
+	key, ok := parsePlanID(id)
+	if !ok {
+		return
+	}
+	run.mu.Lock()
+	defer run.mu.Unlock()
+	entry, planned := run.entries[key]
+	if !planned {
+		return
+	}
+	run.done[entry.group] = struct{}{}
+	// written straight to the file: ctrl+c exits without running defers
+	if _, err := run.log.WriteString(id + "\n"); err != nil && run.logErr == nil {
+		run.logErr = err
+	}
+}
+
+// unreached counts unfinished entries the scan did not get to: those of a
+// host it found unresponsive, and those it does not have because it runs
+// with other templates, targets or options than the plan.
+func (run *planRun) unreached() int {
+	run.mu.Lock()
+	defer run.mu.Unlock()
+	count := 0
+	for _, entry := range run.entries {
+		if _, done := run.done[entry.group]; !done && !entry.seen {
+			count++
+		}
+	}
+	return count
+}
+
+func (run *planRun) close() error {
+	if err := run.log.Close(); err != nil && run.logErr == nil {
+		run.logErr = err
+	}
+	return run.logErr
+}
+
+// finishPlanRun closes the done log and reports the entries left to run.
+func (r *Runner) finishPlanRun(run *planRun) {
+	if err := run.close(); err != nil {
+		r.Logger.Error().Msgf("Could not write %s: %s", planDoneLog(r.options.RunPlanFile), err)
+	}
+	if unreached := run.unreached(); unreached > 0 {
+		r.Logger.Info().Msgf("%d plan entries did not run; running the plan again retries them. Entries run only when the scan has the templates, targets and options the plan was written with", unreached)
+	}
+}

internal/runner/run_plan_test.go

@@ -0,0 +1,52 @@
+package runner
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPlanRunResumesFromItsDoneLog(t *testing.T) {
+	planFile := filepath.Join(t.TempDir(), "plan.jsonl")
+	require.NoError(t, os.WriteFile(planFile, []byte(`{"id":"00000000000000a1","target":"https://a.example/x","templates":["root"]}
+{"id":"00000000000000a2","target":"https://a.example/y","templates":["root"],"alternative-to":"00000000000000a1"}
+{"id":"00000000000000b1","target":"https://b.example","templates":["generic"]}
+{"id":"00000000000000c1","target":"https://c.example","templates":["generic"]}
+{"summary":{"entries":3,"alternatives":1}}
+`), 0o644))
+	// the last line was cut off by an interrupted write
+	require.NoError(t, os.WriteFile(planDoneLog(planFile), []byte("00000000000000b1\n00000000"), 0o644))
+
+	run, err := openPlanRun(planFile)
+	require.NoError(t, err)
+	require.Len(t, run.entries, 4)
+	require.Equal(t, 1, run.resumed)
+
+	require.True(t, run.Skip("00000000000000b1"), "finished in the previous run")
+	require.True(t, run.Skip("00000000000000ff"), "not in the plan")
+	require.False(t, run.Skip("00000000000000a1"))
+	run.Finished("00000000000000a1")
+	require.True(t, run.Skip("00000000000000a2"), "its origin is covered")
+	require.Equal(t, 1, run.unreached(), "c1 never ran")
+	require.NoError(t, run.close())
+
+	log, err := os.ReadFile(planDoneLog(planFile))
+	require.NoError(t, err)
+	require.Equal(t, "00000000000000b1\n00000000\n00000000000000a1\n", string(log), "the partial line is not extended")
+
+	again, err := openPlanRun(planFile)
+	require.NoError(t, err)
+	require.Equal(t, 3, again.resumed, "b1, and a1 with its alternative")
+	require.True(t, again.Skip("00000000000000a2"), "a finished origin stays covered")
+	require.False(t, again.Skip("00000000000000c1"))
+	require.NoError(t, again.close())
+}
+
+func TestPlanRunRejectsAMalformedPlan(t *testing.T) {
+	planFile := filepath.Join(t.TempDir(), "plan.jsonl")
+	require.NoError(t, os.WriteFile(planFile, []byte("{\"id\":\"00000000000000a1\"}\nnot json\n"), 0o644))
+	_, err := openPlanRun(planFile)
+	require.ErrorContains(t, err, "plan.jsonl:2")
+}

internal/runner/runner.go

@@ -837,6 +837,15 @@ func (r *Runner) RunEnumeration() error {
 	if r.options.PlanFile != "" {
 		return r.writePlan(store, executorEngine)
 	}
+	if r.options.RunPlanFile != "" {
+		run, err := openPlanRun(r.options.RunPlanFile)
+		if err != nil {
+			return errors.Wrap(err, "could not read plan")
+		}
+		r.Logger.Info().Msgf("Running %d plan entries from %s, %d already finished", len(run.entries), r.options.RunPlanFile, run.resumed)
+		executorEngine.SetPlanRun(run)
+		defer r.finishPlanRun(run)
+	}
 	// TODO: remove below functions after v3 or update warning messages
 	templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
 

pkg/core/engine.go

@@ -51,6 +51,7 @@ type Engine struct {
 	executerOpts              *protocols.ExecutorOptions
 	Callback                  func(*output.ResultEvent) // Executed on results
 	templateExecutionCallback TemplateExecutionCallback
+	planRun                   PlanRun
 	Logger                    *gologger.Logger
 }
 

pkg/core/executors.go

@@ -36,9 +36,13 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
 			if sharedBudget {
 				defer e.options.ReleaseTemplateThread()
 			}
+			defer e.tasksDone(1)
+			planned, skip := e.claimPlanned(template, &contextargs.MetaInput{})
+			if skip {
+				return
+			}
 			finished := e.templateExecutionStarted(template, "")
 			defer func() { finished(ctx.Err()) }()
-			defer e.tasksDone(1)
 			var err error
 			var match bool
 			ctx := scan.NewScanContext(ctx, contextargs.New(ctx))
@@ -55,6 +59,8 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
 			}
 			if err != nil {
 				e.options.Logger.Warning().Msgf("[%s] Could not execute step (self-contained): %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
+			} else if ctx.Context().Err() == nil {
+				planned()
 			}
 			results.CompareAndSwap(false, match)
 		}(v, usesSharedTemplateBudget)
@@ -330,6 +336,16 @@ func (e *Engine) tasksDone(count int64) {
 func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates.Template, value *contextargs.MetaInput) (matched bool, err error) {
 	// a pair skipped as a repeat of its origin is done as well
 	defer e.tasksDone(1)
+	planned, skip := e.claimPlanned(template, value)
+	if skip {
+		return false, nil
+	}
+	// an origin a sibling already covered is settled too, so it is recorded
+	defer func() {
+		if err == nil && ctx.Err() == nil {
+			planned()
+		}
+	}()
 	// A template fixed to the target's origin makes the same requests for every
 	// target that shares one, which a crawled list produces by the hundred.
 	// Running it once per origin sends exactly the same traffic, minus the

pkg/core/plan.go

@@ -104,7 +104,7 @@ func planEntry(template *templates.Template, value *contextargs.MetaInput, selec
 	entry := PlanEntry{
 		Target:    value.Input,
 		IP:        value.CustomIP,
-		Templates: []string{template.ID},
+		Templates: planTemplates(template, selection),
 		Protocol:  template.Type().String(),
 		Requests:  template.TotalRequests,
 		Dynamic:   template.Flow != "" || template.IsFuzzableRequest(),
@@ -126,7 +126,6 @@ func planEntry(template *templates.Template, value *contextargs.MetaInput, selec
 		entry.Dynamic = true
 	case isCluster(template):
 		cluster := template.Executer.(*templates.ClusterExecuter)
-		entry.Templates = slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
 		// TotalRequests of a cluster counts only its http and dns requests,
 		// so an ssl cluster would show none
 		entry.Requests = cluster.Requests()
@@ -143,6 +142,15 @@ func planEntry(template *templates.Template, value *contextargs.MetaInput, selec
 	return entry
 }
 
+// planTemplates lists the templates an entry runs, sorted so the entry id does
+// not depend on clustering order.
+func planTemplates(template *templates.Template, selection protocols.TemplateSelection) []string {
+	if cluster, ok := template.Executer.(*templates.ClusterExecuter); ok {
+		return slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
+	}
+	return []string{template.ID}
+}
+
 func isCluster(template *templates.Template) bool {
 	_, ok := template.Executer.(*templates.ClusterExecuter)
 	return ok
@@ -154,3 +162,36 @@ func planEntryID(entry PlanEntry) string {
 	sum := sha256.Sum256([]byte(entry.Target + "\x00" + entry.IP + "\x00" + strings.Join(entry.Templates, "\x00")))
 	return hex.EncodeToString(sum[:8])
 }
+
+// PlanRun limits a scan to the entries of a plan and records the ones that
+// finish, so running the plan again resumes it.
+type PlanRun interface {
+	// Skip reports whether the entry is not in the plan or already finished.
+	Skip(id string) bool
+	// Finished records an entry that ran without error.
+	Finished(id string)
+}
+
+// SetPlanRun makes the scan run only the entries run allows.
+func (e *Engine) SetPlanRun(run PlanRun) {
+	e.planRun = run
+}
+
+// claimPlanned reports whether the pair is skipped under a plan run, and
+// otherwise returns the func that records it finished. Without a plan run
+// every pair runs.
+func (e *Engine) claimPlanned(template *templates.Template, value *contextargs.MetaInput) (finished func(), skip bool) {
+	if e.planRun == nil {
+		return func() {}, false
+	}
+	// as in Plan, a self-contained template is not narrowed by any target
+	var selection protocols.TemplateSelection
+	if !template.SelfContained {
+		selection = e.selectionFor(value)
+	}
+	id := planEntryID(PlanEntry{Target: value.Input, IP: value.CustomIP, Templates: planTemplates(template, selection)})
+	if e.planRun.Skip(id) {
+		return nil, true
+	}
+	return func() { e.planRun.Finished(id) }, false
+}

pkg/core/plan_test.go

@@ -154,3 +154,62 @@ func TestPlanListsOriginAlternatives(t *testing.T) {
 	})
 	require.Equal(t, plan, again)
 }
+
+// fixedPlanRun is a plan run over fixed ids that records the finished ones.
+type fixedPlanRun struct {
+	mu       sync.Mutex
+	allowed  map[string]bool
+	finished []string
+}
+
+func (a *fixedPlanRun) Skip(id string) bool { return !a.allowed[id] }
+
+func (a *fixedPlanRun) Finished(id string) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	a.finished = append(a.finished, id)
+}
+
+func TestPlanRunRunsOnlyThePlannedPairs(t *testing.T) {
+	options := &types.Options{BulkSize: 2, TemplateThreads: 2, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
+	engine := New(options)
+	engine.SetExecuterOptions(&protocols.ExecutorOptions{
+		Logger:       engine.Logger,
+		Options:      options,
+		ResumeCfg:    types.NewResumeCfg(),
+		ProtocolType: tmpltypes.HTTPProtocol,
+		Progress:     &progressRecorder{},
+		TargetScope:  scopeByInput{"php": {paths: []string{"thinkphp.yaml"}}},
+	})
+
+	var (
+		mu   sync.Mutex
+		runs []string
+	)
+	var templatesList []*templates.Template
+	for _, path := range []string{"thinkphp.yaml", "generic.yaml"} {
+		templatesList = append(templatesList, &templates.Template{
+			ID:            path,
+			Path:          path,
+			TotalRequests: 1,
+			Executer:      &recordingExecuter{path: path, mu: &mu, runs: &runs},
+		})
+	}
+	targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "php"}, {Input: "plain"}}}
+
+	var plan []PlanEntry
+	engine.Plan(templatesList, targets, true, func(entry PlanEntry) { plan = append(plan, entry) })
+	require.Len(t, plan, 3)
+
+	// the first entry is left out of the plan, as if an agent removed it
+	run := &fixedPlanRun{allowed: map[string]bool{plan[1].ID: true, plan[2].ID: true}}
+	engine.SetPlanRun(run)
+	engine.ExecuteScanWithOpts(context.Background(), templatesList, targets, true)
+
+	var want []string
+	for _, entry := range plan[1:] {
+		want = append(want, entry.Templates[0]+"@"+entry.Target)
+	}
+	require.ElementsMatch(t, want, runs, "only the planned pairs run")
+	require.ElementsMatch(t, []string{plan[1].ID, plan[2].ID}, run.finished, "the scan derives the ids the plan wrote")
+}

pkg/js/devtools/tsgen/parser.go

@@ -38,11 +38,14 @@ func NewEntityParser(dir string) (*EntityParser, error) {
 			return parser.ParseFile(fset, filename, src, parser.ParseComments)
 		},
 	}
-
-	pkg, err := loadPackage(cfg, ".")
+	pkgs, err := packages.Load(cfg, ".")
 	if err != nil {
 		return nil, err
 	}
+	if len(pkgs) == 0 {
+		return nil, errors.New("no packages found")
+	}
+	pkg := pkgs[0]
 
 	return &EntityParser{
 		syntax:      pkg.Syntax,
@@ -284,8 +287,8 @@ func (p *EntityParser) extractClassProperties(node *ast.StructType) []Property {
 					Description: field.Doc.Text(),
 				}
 				if strings.Contains(property.Type, ".") {
-					// external type found
-					property.Type = p.handleExternalType(property.Type)
+					// external struct found
+					property.Type = p.handleExternalStruct(property.Type)
 				}
 				properties = append(properties, property)
 			}
@@ -349,7 +352,7 @@ func (p *EntityParser) extractReturnType(fn *ast.FuncDecl) (out string) {
 		for _, result := range fn.Type.Results.List {
 			tmp := exprToString(result.Type)
 			if strings.Contains(tmp, ".") && !strings.HasPrefix(tmp, "goja.") {
-				tmp = p.handleExternalType(tmp) + " | null" // external object interfaces can always return null
+				tmp = p.handleExternalStruct(tmp) + " | null" // external object interfaces can always return null
 			}
 			returns = append(returns, tmp)
 		}
@@ -374,10 +377,8 @@ func (p *EntityParser) extractReturnType(fn *ast.FuncDecl) (out string) {
 			}
 		}
 		if sliceutil.Contains(returns, "error") {
-			// external types already record that they can be null
-			if !strings.HasSuffix(out, "| null") {
-				out += " | null"
-			}
+			// add | null to the return type
+			out = out + " | null"
 			return
 		}
 	}
@@ -415,18 +416,10 @@ func (p *EntityParser) extractParameters(fn *ast.FuncDecl) []Parameter {
 	return parameters
 }
 
-// handleExternalType returns the TypeScript type for typeName, a type from
-// another package such as ssh.ClientConfig. A struct is queued for
-// scrapeAndCreate, which describes its fields in an interface of the same name.
-// Any other type, such as the net.Conn interface, has no fields to describe,
-// so it becomes any. A typeName that does not resolve is queued too, so that
-// scrapeAndCreate reports it.
-func (p *EntityParser) handleExternalType(typeName string) string {
-	obj, err := p.lookupType(typeName)
-	if err == nil && !isStruct(obj.Type()) {
-		return "any"
-	}
-
+// typeName is in format ssh.ClientConfig
+// it first fetches all fields from the struct and creates a new object
+// with that name and returns name of that object as type
+func (p *EntityParser) handleExternalStruct(typeName string) string {
 	baseType := typeName[strings.LastIndex(typeName, ".")+1:]
 	p.newObjects[typeName] = &Entity{
 		Name:        baseType,
@@ -520,7 +513,6 @@ func (p *EntityParser) extractVarsNConstants() {
 
 // loadImportedPackages loads all imported packages
 func (p *EntityParser) loadImportedPackages() error {
-	cfg := &packages.Config{Mode: packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesInfo}
 	// get all import statements
 	// iterate over all imports
 	for _, file := range p.syntax {
@@ -530,7 +522,7 @@ func (p *EntityParser) loadImportedPackages() error {
 			// remove the quotes from the path
 			path = path[1 : len(path)-1]
 			// load the package
-			pkg, err := loadPackage(cfg, path)
+			pkg, err := loadPackage(path)
 			if err != nil {
 				return err
 			}
@@ -551,31 +543,18 @@ func (p *EntityParser) loadImportedPackages() error {
 	return nil
 }
 
-// loadPackage loads the package that pattern names.
-//
-// packages.Load reports parse, type, and list errors in Package.Errors
-// rather than in its returned error, so loadPackage converts them into an
-// error. The parser assumes a complete syntax tree and type information.
-func loadPackage(cfg *packages.Config, pattern string) (*packages.Package, error) {
-	pkgs, err := packages.Load(cfg, pattern)
+// Load the package containing the type definition
+// TODO: we don't support named imports yet
+func loadPackage(pkgPath string) (*packages.Package, error) {
+	cfg := &packages.Config{Mode: packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesInfo}
+	pkgs, err := packages.Load(cfg, pkgPath)
 	if err != nil {
 		return nil, err
 	}
 	if len(pkgs) == 0 {
 		return nil, errors.New("no packages found")
 	}
-
-	pkg := pkgs[0]
-	if len(pkg.Errors) > 0 {
-		errs := make([]error, len(pkg.Errors))
-		for i, e := range pkg.Errors {
-			errs[i] = e
-		}
-
-		return nil, fmt.Errorf("package %s: %w", pkg.ID, errors.Join(errs...))
-	}
-
-	return pkg, nil
+	return pkgs[0], nil
 }
 
 // exprToString converts an expression to a string

pkg/js/devtools/tsgen/parser_external_test.go

@@ -1,33 +0,0 @@
-package tsgen_test
-
-import (
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/js/devtools/tsgen"
-)
-
-func TestParseExternalTypes(t *testing.T) {
-	t.Parallel()
-
-	ep, err := tsgen.NewEntityParser(filepath.Join("testdata", "external"))
-	require.NoError(t, err)
-	require.NoError(t, ep.Parse())
-
-	entities := make(map[string]tsgen.Entity)
-	for _, e := range ep.GetEntities() {
-		entities[e.Name] = e
-	}
-
-	// net.Conn is an interface, so it has no fields to describe.
-	require.Equal(t, "any | null", entities["Dial"].Function.Returns)
-	require.Equal(t, []tsgen.Property{{Name: "Conn", Type: "any", Description: ""}}, entities["Session"].Object.Properties)
-	require.NotContains(t, entities, "Conn")
-
-	// url.URL is a struct, so its fields become an interface.
-	require.Equal(t, "URL | null", entities["Parse"].Function.Returns)
-	require.Equal(t, "interface", entities["URL"].Type)
-	require.NotEmpty(t, entities["URL"].Object.Properties)
-}

pkg/js/devtools/tsgen/parser_test.go

@@ -1,86 +0,0 @@
-package tsgen
-
-import (
-	"go/ast"
-	"go/parser"
-	"go/token"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-	"golang.org/x/tools/go/packages"
-)
-
-// writeModule writes src as the only package of a new Go module and
-// returns the module directory.
-func writeModule(t *testing.T, src string) string {
-	t.Helper()
-
-	dir := t.TempDir()
-	files := map[string]string{
-		"go.mod":     "module example.com/tsgentest\n\ngo 1.21\n",
-		"example.go": src,
-	}
-
-	for name, content := range files {
-		require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600))
-	}
-
-	return dir
-}
-
-func TestNewEntityParser(t *testing.T) {
-	const src = `package example
-
-import "strings"
-
-// Upper returns s in upper case.
-func Upper(s string) string { return strings.ToUpper(s) }
-`
-
-	p, err := NewEntityParser(writeModule(t, src))
-	require.NoError(t, err)
-	require.NoError(t, p.Parse())
-
-	entities := p.GetEntities()
-
-	names := make([]string, 0, len(entities))
-	for _, e := range entities {
-		names = append(names, e.Name)
-	}
-
-	require.Contains(t, names, "Upper")
-}
-
-func TestNewEntityParserPackageErrors(t *testing.T) {
-	tests := []struct {
-		name string
-		src  string
-	}{
-		{name: "syntax error", src: "package example\n\nfunc Broken( {\n\treturn 42\n}\n"},
-		{name: "type error", src: "package example\n\nfunc Broken() int { return undefined }\n"},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			_, err := NewEntityParser(writeModule(t, tt.src))
-			require.ErrorContains(t, err, "example.go:")
-		})
-	}
-}
-
-func TestLoadImportedPackagesMissing(t *testing.T) {
-	const src = `package example; import "example.com/tsgentest/missing"`
-
-	file, err := parser.ParseFile(token.NewFileSet(), "example.go", src, parser.ImportsOnly)
-	require.NoError(t, err)
-	p := &EntityParser{
-		syntax:  []*ast.File{file},
-		imports: map[string]*packages.Package{},
-	}
-
-	var pkgErr packages.Error
-
-	require.ErrorAs(t, p.loadImportedPackages(), &pkgErr)
-	require.Contains(t, pkgErr.Msg, "example.com/tsgentest/missing")
-}

pkg/js/devtools/tsgen/scrape.go

@@ -1,15 +1,13 @@
 package tsgen
 
 import (
-	"errors"
 	"fmt"
 	"go/types"
 	"regexp"
 	"strings"
-)
 
-// errUnresolvedType is returned by lookupType for a name it cannot resolve.
-var errUnresolvedType = errors.New("unresolved type")
+	"github.com/projectdiscovery/utils/errkit"
+)
 
 // scrape.go scrapes all information of exported type from different package
 
@@ -17,13 +15,24 @@ func (p *EntityParser) scrapeAndCreate(typeName string) error {
 	if p.newObjects[typeName] == nil {
 		return nil
 	}
-
-	typeNameObj, err := p.lookupType(typeName)
-	if err != nil {
-		return err
+	// get package name
+	pkgName := strings.Split(typeName, ".")[0]
+	baseTypeName := strings.Split(typeName, ".")[1]
+	// get package
+	pkg, ok := p.imports[pkgName]
+	if !ok {
+		return errkit.Newf("package %v for type %v not found", pkgName, typeName)
+	}
+	// get type
+	obj := pkg.Types.Scope().Lookup(baseTypeName)
+	if obj == nil {
+		return errkit.Newf("type %v not found in package %+v", typeName, pkg)
+	}
+	// Ensure the object is a type name
+	typeNameObj, ok := obj.(*types.TypeName)
+	if !ok {
+		return errkit.Newf("%v is not a type name", typeName)
 	}
-
-	baseTypeName := typeNameObj.Name()
 	// Ensure the type is a named struct type
 	namedStruct, ok := typeNameObj.Type().Underlying().(*types.Struct)
 	if !ok {
@@ -63,36 +72,6 @@ func (p *EntityParser) scrapeAndCreate(typeName string) error {
 	return nil
 }
 
-// lookupType resolves typeName, such as net.Conn, in the packages imported by
-// the parsed files.
-func (p *EntityParser) lookupType(typeName string) (*types.TypeName, error) {
-	pkgName, baseTypeName, _ := strings.Cut(typeName, ".")
-
-	pkg := p.imports[pkgName]
-	if pkg == nil {
-		return nil, fmt.Errorf("%w: package %v for type %v not found", errUnresolvedType, pkgName, typeName)
-	}
-
-	obj := pkg.Types.Scope().Lookup(baseTypeName)
-	if obj == nil {
-		return nil, fmt.Errorf("%w: type %v not found in package %+v", errUnresolvedType, typeName, pkg)
-	}
-
-	typeNameObj, ok := obj.(*types.TypeName)
-	if !ok {
-		return nil, fmt.Errorf("%w: %v is not a type name", errUnresolvedType, typeName)
-	}
-
-	return typeNameObj, nil
-}
-
-// isStruct reports whether t is a struct type.
-func isStruct(t types.Type) bool {
-	_, ok := t.Underlying().(*types.Struct)
-
-	return ok
-}
-
 type ExtObject struct {
 	builtIn map[string]string
 	nested  map[string]map[string]*ExtObject // Changed to map of field names to ExtObject

pkg/js/devtools/tsgen/testdata/external/external.go

@@ -1,32 +0,0 @@
-// Package external is a tsgen fixture that references types from other
-// packages, both structs and non-structs.
-package external
-
-import (
-	"context"
-	"net"
-	"net/url"
-)
-
-// Dial returns an external interface type.
-func Dial(network, address string) (net.Conn, error) {
-	return net.Dial(network, address)
-}
-
-// Parse returns an external struct type.
-func Parse(raw string) (*url.URL, error) {
-	return url.Parse(raw)
-}
-
-// Session holds an external interface type in an exported field.
-type Session struct {
-	Conn net.Conn
-}
-
-type dialer struct{}
-
-// DialContext is exported, but its receiver is not.
-func (dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
-	var d net.Dialer
-	return d.DialContext(ctx, network, address)
-}

pkg/js/libs/gptransport/.nuclei-jsgen-ignore

@@ -1 +0,0 @@
-shared adapter package; not a public JavaScript module

pkg/js/libs/smbsession/.nuclei-jsgen-ignore

@@ -1 +0,0 @@
-shared adapter package; not a public JavaScript module

pkg/types/types.go

@@ -254,6 +254,9 @@ type Options struct {
 	// PlanFile is where the work a scan would do is written as JSON lines;
 	// nuclei exits after writing it, without sending requests.
 	PlanFile string
+	// RunPlanFile is a plan whose entries the scan runs, skipping those its
+	// done log lists as finished.
+	RunPlanFile string
 	// NoStrictSyntax disables strict syntax check on nuclei templates (allows custom key-value pairs).
 	NoStrictSyntax bool
 	// Verbose flag indicates whether to show verbose output or not
@@ -715,6 +718,7 @@ func (options *Options) Copy() *Options {
 		Silent:                         options.Silent,
 		Validate:                       options.Validate,
 		PlanFile:                       options.PlanFile,
+		RunPlanFile:                    options.RunPlanFile,
 		NoStrictSyntax:                 options.NoStrictSyntax,
 		Verbose:                        options.Verbose,
 		VerboseVerbose:                 options.VerboseVerbose,

pkg/types/types_test.go

@@ -18,10 +18,13 @@ func TestCopyPreservesPayloadRandomOrder(t *testing.T) {
 }
 
 func TestCopyPreservesPlanFile(t *testing.T) {
-	options := &Options{PlanFile: "plan.jsonl"}
+	options := &Options{PlanFile: "plan.jsonl", RunPlanFile: "run.jsonl"}
 	if options.Copy().PlanFile != "plan.jsonl" {
 		t.Fatal("Copy must preserve PlanFile, or an embedded caller loses plan mode")
 	}
+	if options.Copy().RunPlanFile != "run.jsonl" {
+		t.Fatal("Copy must preserve RunPlanFile, or an embedded caller runs the whole scan")
+	}
 }
 
 func TestToJSONMapRejectsKeysSharingAStringForm(t *testing.T) {