Compare commits

..
Author SHA1 Message Date
Mzack9999 380a561729 unwrap sharing 2026-11-10 23:11:12 +01:00
Mzack9999 0596c77011 Merge dev 2026-11-10 23:08:40 +01:00
Mzack9999 fe24f8638b review fixes 2026-11-04 05:09:15 +01:00
Mzack9999 8b7cbf44e6 all trackers 2026-11-04 03:26:07 +01:00
Mzack9999 59ae4cac04 tls dialer 2026-11-04 03:00:58 +01:00
Mzack9999 64965ba776 tls verify 2026-11-04 00:54:48 +01:00
Mzack9999 15271926ad tracker client 2026-11-03 23:28:48 +01:00
32 changed files with 451 additions and 455 deletions

No files matched your search

-1
View File
@@ -167,7 +167,6 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file) -wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei -validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target -plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates -nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content -td, -template-display displays the templates content
-tl list all templates matching current filters -tl list all templates matching current filters
-1
View File
@@ -165,7 +165,6 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file) -wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei -validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target -plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates -nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content -td, -template-display displays the templates content
-tl list all templates matching current filters -tl list all templates matching current filters
-1
View File
@@ -165,7 +165,6 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file) -wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei -validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target -plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates -nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content -td, -template-display displays the templates content
-tl list all templates matching current filters -tl list all templates matching current filters
-1
View File
@@ -182,7 +182,6 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file) -wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei -validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target -plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates -nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content -td, -template-display displays the templates content
-tl list all templates matching current filters -tl list all templates matching current filters
-1
View File
@@ -165,7 +165,6 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file) -wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei -validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target -plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates -nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content -td, -template-display displays the templates content
-tl list all templates matching current filters -tl list all templates matching current filters
-1
View File
@@ -165,7 +165,6 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file) -wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei -validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target -plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates -nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content -td, -template-display displays the templates content
-tl list all templates matching current filters -tl list all templates matching current filters
-1
View File
@@ -165,7 +165,6 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file) -wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei -validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target -plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates -nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content -td, -template-display displays the templates content
-tl list all templates matching current filters -tl list all templates matching current filters
-1
View File
@@ -165,7 +165,6 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file) -wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei -validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target -plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates -nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content -td, -template-display displays the templates content
-tl list all templates matching current filters -tl list all templates matching current filters
-1
View File
@@ -165,7 +165,6 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file) -wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei -validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target -plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates -nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content -td, -template-display displays the templates content
-tl list all templates matching current filters -tl list all templates matching current filters
-1
View File
@@ -314,7 +314,6 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"), flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
flagSet.StringVar(&options.PlanFile, "plan", "", "write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target"), flagSet.StringVar(&options.PlanFile, "plan", "", "write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target"),
flagSet.StringVar(&options.RunPlanFile, "run-plan", "", "run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes"),
flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"), flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"), flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all templates matching current filters"), flagSet.BoolVar(&options.TemplateList, "tl", false, "list all templates matching current filters"),
+5 -18
View File
@@ -558,31 +558,18 @@ func getBoolEnvValue(key string) bool {
// validatePlanOptions rejects options that change the templates or targets // validatePlanOptions rejects options that change the templates or targets
// only once scanning starts, which a plan written beforehand cannot show. // only once scanning starts, which a plan written beforehand cannot show.
func validatePlanOptions(options *types.Options) error { func validatePlanOptions(options *types.Options) error {
if options.RunPlanFile != "" && options.PlanFile != "" {
return errors.New("-plan cannot be used with -run-plan")
}
if options.RunPlanFile != "" && options.Resume != "" {
return errors.New("-run-plan cannot be used with -resume, the plan's done log resumes it")
}
if options.PlanFile == "" && options.RunPlanFile == "" {
return nil
}
// -automatic-scan builds its own engine, so a plan set on the main engine
// would never be consulted. The other flags change the targets only after
// a plan has been written.
flag := "-plan"
if options.PlanFile == "" { if options.PlanFile == "" {
flag = "-run-plan" return nil
} }
switch { switch {
case options.AutomaticScan: case options.AutomaticScan:
return fmt.Errorf("%s cannot be used with -automatic-scan, which picks templates while scanning", flag) return errors.New("-plan cannot be used with -automatic-scan, which picks templates while scanning")
case options.Uncover || len(options.UncoverQuery) > 0: case options.Uncover || len(options.UncoverQuery) > 0:
return fmt.Errorf("%s cannot be used with -uncover, which finds targets over the network", flag) return errors.New("-plan cannot be used with -uncover, which finds targets over the network")
case options.PreflightPortScan: case options.PreflightPortScan:
return fmt.Errorf("%s cannot be used with -preflight-portscan, which drops targets after scanning their ports", flag) return errors.New("-plan cannot be used with -preflight-portscan, which drops targets after scanning their ports")
case options.DASTServer || options.DASTProxy: case options.DASTServer || options.DASTProxy:
return fmt.Errorf("%s cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan", flag) return errors.New("-plan cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan")
} }
return nil return nil
} }
-15
View File
@@ -23,19 +23,4 @@ func TestPlanRejectsOptionsThatChangeTargetsLater(t *testing.T) {
require.NoError(t, validatePlanOptions(&types.Options{Uncover: true}), "only -plan restricts them") require.NoError(t, validatePlanOptions(&types.Options{Uncover: true}), "only -plan restricts them")
require.NoError(t, validatePlanOptions(&types.Options{UncoverQuery: goflags.StringSlice{"example"}})) require.NoError(t, validatePlanOptions(&types.Options{UncoverQuery: goflags.StringSlice{"example"}}))
require.NoError(t, validatePlanOptions(&types.Options{DASTServer: true})) require.NoError(t, validatePlanOptions(&types.Options{DASTServer: true}))
require.Error(t, validatePlanOptions(&types.Options{RunPlanFile: "p", PlanFile: "q"}))
require.Error(t, validatePlanOptions(&types.Options{RunPlanFile: "p", Resume: "r"}))
require.NoError(t, validatePlanOptions(&types.Options{RunPlanFile: "p"}))
for name, options := range map[string]*types.Options{
"run automatic scan": {RunPlanFile: "p", AutomaticScan: true},
"run uncover": {RunPlanFile: "p", Uncover: true},
"run uncover query": {RunPlanFile: "p", UncoverQuery: goflags.StringSlice{"example"}},
"run preflight": {RunPlanFile: "p", PreflightPortScan: true},
"run dast server": {RunPlanFile: "p", DASTServer: true},
"run dast proxy": {RunPlanFile: "p", DASTProxy: true},
} {
require.Error(t, validatePlanOptions(options), name)
}
} }
-215
View File
@@ -1,215 +0,0 @@
package runner
import (
"bufio"
"bytes"
"fmt"
"io"
"os"
"strconv"
"sync"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
)
// planRun limits a scan to the entries of a plan file and appends the id of
// each finished entry to the plan's done log, so running the plan again skips
// them. Ids are derived from what an entry runs, not from its position, so the
// log still applies after the plan is written again for changed targets or
// templates.
type planRun struct {
mu sync.Mutex
entries map[uint64]plannedEntry
// done holds finished origin groups by the id of their first entry; an
// entry outside a group is its own group
done map[uint64]struct{}
log *os.File
logErr error
// resumed counts the entries finished before this run
resumed int
}
type plannedEntry struct {
group uint64
seen bool
}
func planDoneLog(planFile string) string {
return planFile + ".done"
}
func openPlanRun(planFile string) (*planRun, error) {
run := &planRun{entries: make(map[uint64]plannedEntry), done: make(map[uint64]struct{})}
if err := run.readPlan(planFile); err != nil {
return nil, err
}
if err := run.readDoneLog(planDoneLog(planFile)); err != nil {
return nil, err
}
log, err := os.OpenFile(planDoneLog(planFile), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o644)
if err != nil {
return nil, err
}
// an interrupted write leaves a partial line, which the next id must not
// extend
if info, err := log.Stat(); err == nil && info.Size() > 0 {
last := make([]byte, 1)
if _, err := log.ReadAt(last, info.Size()-1); err == nil && last[0] != '\n' {
_, _ = log.WriteString("\n")
}
}
run.log = log
return run, nil
}
func (run *planRun) readPlan(planFile string) error {
file, err := os.Open(planFile)
if err != nil {
return err
}
defer func() { _ = file.Close() }()
reader := bufio.NewReader(file)
for number := 1; ; number++ {
line, readErr := reader.ReadBytes('\n')
if line = bytes.TrimSpace(line); len(line) > 0 {
var entry struct {
ID string `json:"id"`
AlternativeTo string `json:"alternative-to"`
}
if err := json.Unmarshal(line, &entry); err != nil {
return fmt.Errorf("%s:%d: %w", planFile, number, err)
}
// the summary line has no id
if entry.ID != "" {
id, ok := parsePlanID(entry.ID)
if !ok {
return fmt.Errorf("%s:%d: invalid id %q", planFile, number, entry.ID)
}
group := id
if entry.AlternativeTo != "" {
if group, ok = parsePlanID(entry.AlternativeTo); !ok {
return fmt.Errorf("%s:%d: invalid alternative-to %q", planFile, number, entry.AlternativeTo)
}
}
run.entries[id] = plannedEntry{group: group}
}
}
if readErr == io.EOF {
return nil
}
if readErr != nil {
return readErr
}
}
}
func (run *planRun) readDoneLog(path string) error {
file, err := os.Open(path)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
defer func() { _ = file.Close() }()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
// a partial line from an interrupted write does not parse and is
// run again
id, ok := parsePlanID(scanner.Text())
if !ok {
continue
}
if entry, planned := run.entries[id]; planned {
run.done[entry.group] = struct{}{}
}
}
if err := scanner.Err(); err != nil {
return err
}
for _, entry := range run.entries {
if _, done := run.done[entry.group]; done {
run.resumed++
}
}
return nil
}
// parsePlanID keeps ids as integers, which a plan of millions of entries
// holds in a fraction of the memory of strings.
func parsePlanID(id string) (uint64, bool) {
if len(id) != 16 {
return 0, false
}
value, err := strconv.ParseUint(id, 16, 64)
return value, err == nil
}
func (run *planRun) Skip(id string) bool {
key, ok := parsePlanID(id)
if !ok {
return true
}
run.mu.Lock()
defer run.mu.Unlock()
entry, planned := run.entries[key]
if !planned {
return true
}
if !entry.seen {
entry.seen = true
run.entries[key] = entry
}
_, done := run.done[entry.group]
return done
}
func (run *planRun) Finished(id string) {
key, ok := parsePlanID(id)
if !ok {
return
}
run.mu.Lock()
defer run.mu.Unlock()
entry, planned := run.entries[key]
if !planned {
return
}
run.done[entry.group] = struct{}{}
// written straight to the file: ctrl+c exits without running defers
if _, err := run.log.WriteString(id + "\n"); err != nil && run.logErr == nil {
run.logErr = err
}
}
// unreached counts unfinished entries the scan did not get to: those of a
// host it found unresponsive, and those it does not have because it runs
// with other templates, targets or options than the plan.
func (run *planRun) unreached() int {
run.mu.Lock()
defer run.mu.Unlock()
count := 0
for _, entry := range run.entries {
if _, done := run.done[entry.group]; !done && !entry.seen {
count++
}
}
return count
}
func (run *planRun) close() error {
if err := run.log.Close(); err != nil && run.logErr == nil {
run.logErr = err
}
return run.logErr
}
// finishPlanRun closes the done log and reports the entries left to run.
func (r *Runner) finishPlanRun(run *planRun) {
if err := run.close(); err != nil {
r.Logger.Error().Msgf("Could not write %s: %s", planDoneLog(r.options.RunPlanFile), err)
}
if unreached := run.unreached(); unreached > 0 {
r.Logger.Info().Msgf("%d plan entries did not run; running the plan again retries them. Entries run only when the scan has the templates, targets and options the plan was written with", unreached)
}
}
-52
View File
@@ -1,52 +0,0 @@
package runner
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
)
func TestPlanRunResumesFromItsDoneLog(t *testing.T) {
planFile := filepath.Join(t.TempDir(), "plan.jsonl")
require.NoError(t, os.WriteFile(planFile, []byte(`{"id":"00000000000000a1","target":"https://a.example/x","templates":["root"]}
{"id":"00000000000000a2","target":"https://a.example/y","templates":["root"],"alternative-to":"00000000000000a1"}
{"id":"00000000000000b1","target":"https://b.example","templates":["generic"]}
{"id":"00000000000000c1","target":"https://c.example","templates":["generic"]}
{"summary":{"entries":3,"alternatives":1}}
`), 0o644))
// the last line was cut off by an interrupted write
require.NoError(t, os.WriteFile(planDoneLog(planFile), []byte("00000000000000b1\n00000000"), 0o644))
run, err := openPlanRun(planFile)
require.NoError(t, err)
require.Len(t, run.entries, 4)
require.Equal(t, 1, run.resumed)
require.True(t, run.Skip("00000000000000b1"), "finished in the previous run")
require.True(t, run.Skip("00000000000000ff"), "not in the plan")
require.False(t, run.Skip("00000000000000a1"))
run.Finished("00000000000000a1")
require.True(t, run.Skip("00000000000000a2"), "its origin is covered")
require.Equal(t, 1, run.unreached(), "c1 never ran")
require.NoError(t, run.close())
log, err := os.ReadFile(planDoneLog(planFile))
require.NoError(t, err)
require.Equal(t, "00000000000000b1\n00000000\n00000000000000a1\n", string(log), "the partial line is not extended")
again, err := openPlanRun(planFile)
require.NoError(t, err)
require.Equal(t, 3, again.resumed, "b1, and a1 with its alternative")
require.True(t, again.Skip("00000000000000a2"), "a finished origin stays covered")
require.False(t, again.Skip("00000000000000c1"))
require.NoError(t, again.close())
}
func TestPlanRunRejectsAMalformedPlan(t *testing.T) {
planFile := filepath.Join(t.TempDir(), "plan.jsonl")
require.NoError(t, os.WriteFile(planFile, []byte("{\"id\":\"00000000000000a1\"}\nnot json\n"), 0o644))
_, err := openPlanRun(planFile)
require.ErrorContains(t, err, "plan.jsonl:2")
}
-9
View File
@@ -855,15 +855,6 @@ func (r *Runner) RunEnumeration() error {
} }
}() }()
} }
if r.options.RunPlanFile != "" {
run, err := openPlanRun(r.options.RunPlanFile)
if err != nil {
return errors.Wrap(err, "could not read plan")
}
r.Logger.Info().Msgf("Running %d plan entries from %s, %d already finished", len(run.entries), r.options.RunPlanFile, run.resumed)
executorEngine.SetPlanRun(run)
defer r.finishPlanRun(run)
}
// TODO: remove below functions after v3 or update warning messages // TODO: remove below functions after v3 or update warning messages
templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose) templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
-1
View File
@@ -51,7 +51,6 @@ type Engine struct {
executerOpts *protocols.ExecutorOptions executerOpts *protocols.ExecutorOptions
Callback func(*output.ResultEvent) // Executed on results Callback func(*output.ResultEvent) // Executed on results
templateExecutionCallback TemplateExecutionCallback templateExecutionCallback TemplateExecutionCallback
planRun PlanRun
Logger *gologger.Logger Logger *gologger.Logger
} }
+1 -17
View File
@@ -36,13 +36,9 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
if sharedBudget { if sharedBudget {
defer e.options.ReleaseTemplateThread() defer e.options.ReleaseTemplateThread()
} }
defer e.tasksDone(1)
planned, skip := e.claimPlanned(template, &contextargs.MetaInput{})
if skip {
return
}
finished := e.templateExecutionStarted(template, "") finished := e.templateExecutionStarted(template, "")
defer func() { finished(ctx.Err()) }() defer func() { finished(ctx.Err()) }()
defer e.tasksDone(1)
var err error var err error
var match bool var match bool
ctx := scan.NewScanContext(ctx, contextargs.New(ctx)) ctx := scan.NewScanContext(ctx, contextargs.New(ctx))
@@ -59,8 +55,6 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
} }
if err != nil { if err != nil {
e.options.Logger.Warning().Msgf("[%s] Could not execute step (self-contained): %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err) e.options.Logger.Warning().Msgf("[%s] Could not execute step (self-contained): %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
} else if ctx.Context().Err() == nil {
planned()
} }
results.CompareAndSwap(false, match) results.CompareAndSwap(false, match)
}(v, usesSharedTemplateBudget) }(v, usesSharedTemplateBudget)
@@ -336,16 +330,6 @@ func (e *Engine) tasksDone(count int64) {
func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates.Template, value *contextargs.MetaInput) (matched bool, err error) { func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates.Template, value *contextargs.MetaInput) (matched bool, err error) {
// a pair skipped as a repeat of its origin is done as well // a pair skipped as a repeat of its origin is done as well
defer e.tasksDone(1) defer e.tasksDone(1)
planned, skip := e.claimPlanned(template, value)
if skip {
return false, nil
}
// an origin a sibling already covered is settled too, so it is recorded
defer func() {
if err == nil && ctx.Err() == nil {
planned()
}
}()
// A template fixed to the target's origin makes the same requests for every // A template fixed to the target's origin makes the same requests for every
// target that shares one, which a crawled list produces by the hundred. // target that shares one, which a crawled list produces by the hundred.
// Running it once per origin sends exactly the same traffic, minus the // Running it once per origin sends exactly the same traffic, minus the
+2 -43
View File
@@ -104,7 +104,7 @@ func planEntry(template *templates.Template, value *contextargs.MetaInput, selec
entry := PlanEntry{ entry := PlanEntry{
Target: value.Input, Target: value.Input,
IP: value.CustomIP, IP: value.CustomIP,
Templates: planTemplates(template, selection), Templates: []string{template.ID},
Protocol: template.Type().String(), Protocol: template.Type().String(),
Requests: template.TotalRequests, Requests: template.TotalRequests,
Dynamic: template.Flow != "" || template.IsFuzzableRequest(), Dynamic: template.Flow != "" || template.IsFuzzableRequest(),
@@ -126,6 +126,7 @@ func planEntry(template *templates.Template, value *contextargs.MetaInput, selec
entry.Dynamic = true entry.Dynamic = true
case isCluster(template): case isCluster(template):
cluster := template.Executer.(*templates.ClusterExecuter) cluster := template.Executer.(*templates.ClusterExecuter)
entry.Templates = slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
// TotalRequests of a cluster counts only its http and dns requests, // TotalRequests of a cluster counts only its http and dns requests,
// so an ssl cluster would show none // so an ssl cluster would show none
entry.Requests = cluster.Requests() entry.Requests = cluster.Requests()
@@ -142,15 +143,6 @@ func planEntry(template *templates.Template, value *contextargs.MetaInput, selec
return entry return entry
} }
// planTemplates lists the templates an entry runs, sorted so the entry id does
// not depend on clustering order.
func planTemplates(template *templates.Template, selection protocols.TemplateSelection) []string {
if cluster, ok := template.Executer.(*templates.ClusterExecuter); ok {
return slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
}
return []string{template.ID}
}
func isCluster(template *templates.Template) bool { func isCluster(template *templates.Template) bool {
_, ok := template.Executer.(*templates.ClusterExecuter) _, ok := template.Executer.(*templates.ClusterExecuter)
return ok return ok
@@ -162,36 +154,3 @@ func planEntryID(entry PlanEntry) string {
sum := sha256.Sum256([]byte(entry.Target + "\x00" + entry.IP + "\x00" + strings.Join(entry.Templates, "\x00"))) sum := sha256.Sum256([]byte(entry.Target + "\x00" + entry.IP + "\x00" + strings.Join(entry.Templates, "\x00")))
return hex.EncodeToString(sum[:8]) return hex.EncodeToString(sum[:8])
} }
// PlanRun limits a scan to the entries of a plan and records the ones that
// finish, so running the plan again resumes it.
type PlanRun interface {
// Skip reports whether the entry is not in the plan or already finished.
Skip(id string) bool
// Finished records an entry that ran without error.
Finished(id string)
}
// SetPlanRun makes the scan run only the entries run allows.
func (e *Engine) SetPlanRun(run PlanRun) {
e.planRun = run
}
// claimPlanned reports whether the pair is skipped under a plan run, and
// otherwise returns the func that records it finished. Without a plan run
// every pair runs.
func (e *Engine) claimPlanned(template *templates.Template, value *contextargs.MetaInput) (finished func(), skip bool) {
if e.planRun == nil {
return func() {}, false
}
// as in Plan, a self-contained template is not narrowed by any target
var selection protocols.TemplateSelection
if !template.SelfContained {
selection = e.selectionFor(value)
}
id := planEntryID(PlanEntry{Target: value.Input, IP: value.CustomIP, Templates: planTemplates(template, selection)})
if e.planRun.Skip(id) {
return nil, true
}
return func() { e.planRun.Finished(id) }, false
}
-59
View File
@@ -154,62 +154,3 @@ func TestPlanListsOriginAlternatives(t *testing.T) {
}) })
require.Equal(t, plan, again) require.Equal(t, plan, again)
} }
// fixedPlanRun is a plan run over fixed ids that records the finished ones.
type fixedPlanRun struct {
mu sync.Mutex
allowed map[string]bool
finished []string
}
func (a *fixedPlanRun) Skip(id string) bool { return !a.allowed[id] }
func (a *fixedPlanRun) Finished(id string) {
a.mu.Lock()
defer a.mu.Unlock()
a.finished = append(a.finished, id)
}
func TestPlanRunRunsOnlyThePlannedPairs(t *testing.T) {
options := &types.Options{BulkSize: 2, TemplateThreads: 2, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
engine := New(options)
engine.SetExecuterOptions(&protocols.ExecutorOptions{
Logger: engine.Logger,
Options: options,
ResumeCfg: types.NewResumeCfg(),
ProtocolType: tmpltypes.HTTPProtocol,
Progress: &progressRecorder{},
TargetScope: scopeByInput{"php": {paths: []string{"thinkphp.yaml"}}},
})
var (
mu sync.Mutex
runs []string
)
var templatesList []*templates.Template
for _, path := range []string{"thinkphp.yaml", "generic.yaml"} {
templatesList = append(templatesList, &templates.Template{
ID: path,
Path: path,
TotalRequests: 1,
Executer: &recordingExecuter{path: path, mu: &mu, runs: &runs},
})
}
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "php"}, {Input: "plain"}}}
var plan []PlanEntry
engine.Plan(templatesList, targets, true, func(entry PlanEntry) { plan = append(plan, entry) })
require.Len(t, plan, 3)
// the first entry is left out of the plan, as if an agent removed it
run := &fixedPlanRun{allowed: map[string]bool{plan[1].ID: true, plan[2].ID: true}}
engine.SetPlanRun(run)
engine.ExecuteScanWithOpts(context.Background(), templatesList, targets, true)
var want []string
for _, entry := range plan[1:] {
want = append(want, entry.Templates[0]+"@"+entry.Target)
}
require.ElementsMatch(t, want, runs, "only the planned pairs run")
require.ElementsMatch(t, []string{plan[1].ID, plan[2].ID}, run.finished, "the scan derives the ids the plan wrote")
}
+12 -1
View File
@@ -94,10 +94,21 @@ func NewTransportWrapper() func(http.RoundTripper) http.RoundTripper {
}, },
DisableWarningHeader: true, DisableWarningHeader: true,
} }
return transport return &unwrappingTransport{Transport: transport}
} }
} }
// unwrappingTransport lets code that must not use the cache, such as reporting
// clients that need TLS verification, reach the wrapped transport.
type unwrappingTransport struct {
*httpcache.Transport
}
// Unwrap returns the transport the cache wraps.
func (t *unwrappingTransport) Unwrap() http.RoundTripper {
return t.Transport.Transport
}
func openDiskCache(dir string, maxBytes int64) (httpcache.Cache, *leveldb.DB, error) { func openDiskCache(dir string, maxBytes int64) (httpcache.Cache, *leveldb.DB, error) {
if err := os.MkdirAll(dir, 0o700); err != nil { if err := os.MkdirAll(dir, 0o700); err != nil {
return nil, nil, fmt.Errorf("create cache dir: %w", err) return nil, nil, fmt.Errorf("create cache dir: %w", err)
@@ -1,6 +1,7 @@
package httpcache package httpcache
import ( import (
"crypto/tls"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
@@ -10,6 +11,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
libcache "github.com/sandrolain/httpcache" libcache "github.com/sandrolain/httpcache"
) )
@@ -178,3 +180,29 @@ func TestDiskCacheRespectsSizeCap(t *testing.T) {
t.Fatal("stored a response over a 1-byte directory cap") t.Fatal("stored a response over a 1-byte directory cap")
} }
} }
// Reporting clients send tokens, so tlsverify must reach the scan transport
// under the cache wrapper and turn certificate verification on.
func TestTransportWrapperUnwrapsForTLSVerification(t *testing.T) {
cacheMu.Lock()
saved := cache
cache = libcache.NewMemoryCache()
cacheMu.Unlock()
t.Cleanup(func() {
cacheMu.Lock()
cache = saved
cacheMu.Unlock()
})
base := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
wrapped := NewTransportWrapper()(base)
u, ok := wrapped.(interface{ Unwrap() http.RoundTripper })
if !ok || u.Unwrap() != base {
t.Fatal("the cache wrapper does not unwrap to the scan transport")
}
got, ok := tlsverify.Client(&http.Client{Transport: wrapped}).Transport.(*http.Transport)
if !ok || got.TLSClientConfig.InsecureSkipVerify {
t.Fatal("tlsverify did not turn verification on under the cache wrapper")
}
}
@@ -166,6 +166,11 @@ func (t *connTrackingTransport) RoundTrip(req *http.Request) (*http.Response, er
return resp, err return resp, err
} }
// Unwrap returns the underlying transport.
func (t *connTrackingTransport) Unwrap() http.RoundTripper {
return t.base
}
func (t *connTrackingTransport) CloseIdleConnections() { func (t *connTrackingTransport) CloseIdleConnections() {
type closeIdler interface{ CloseIdleConnections() } type closeIdler interface{ CloseIdleConnections() }
if ci, ok := t.base.(closeIdler); ok { if ci, ok := t.base.(closeIdler); ok {
@@ -124,6 +124,12 @@ func (t *sharingTransport) RoundTrip(req *http.Request) (*http.Response, error)
return entry.response.copyFor(req), nil return entry.response.copyFor(req), nil
} }
// Unwrap returns the underlying transport, so reporting clients that need TLS
// verification reach the scan transport and never share responses.
func (t *sharingTransport) Unwrap() http.RoundTripper {
return t.base
}
func (t *sharingTransport) CloseIdleConnections() { func (t *sharingTransport) CloseIdleConnections() {
type closeIdler interface{ CloseIdleConnections() } type closeIdler interface{ CloseIdleConnections() }
if ci, ok := t.base.(closeIdler); ok { if ci, ok := t.base.(closeIdler); ok {
@@ -3,6 +3,7 @@ package httpclientpool
import ( import (
"container/list" "container/list"
"context" "context"
"crypto/tls"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@@ -14,6 +15,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
"github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/useragent" "github.com/projectdiscovery/useragent"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -306,3 +308,14 @@ func TestSharedResponsesDoNotKeepShortReads(t *testing.T) {
_ = second.Body.Close() _ = second.Body.Close()
require.EqualValues(t, 2, base.hits.Load(), "a short read is not reused") require.EqualValues(t, 2, base.hits.Load(), "a short read is not reused")
} }
// Reporting clients send tokens, so tlsverify must reach the scan transport
// under the sharing wrapper and turn certificate verification on.
func TestSharingTransportUnwrapsForTLSVerification(t *testing.T) {
base := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
wrapped := newSharingTransport(&connTrackingTransport{base: base})
got, ok := tlsverify.Client(&http.Client{Transport: wrapped}).Transport.(*http.Transport)
require.True(t, ok, "tlsverify did not reach the scan transport under the sharing wrapper")
require.False(t, got.TLSClientConfig.InsecureSkipVerify)
}
+13
View File
@@ -21,6 +21,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/pdf" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/pdf"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
@@ -28,6 +29,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml" "github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/utils/errkit" "github.com/projectdiscovery/utils/errkit"
fileutil "github.com/projectdiscovery/utils/file" fileutil "github.com/projectdiscovery/utils/file"
"go.uber.org/multierr" "go.uber.org/multierr"
@@ -77,6 +79,17 @@ type IssueTrackerStats struct {
func New(options *Options, db string, doNotDedupe bool) (Client, error) { func New(options *Options, db string, doNotDedupe bool) (Client, error) {
client := &ReportingClient{options: options} client := &ReportingClient{options: options}
// The caller can pass the scan client, which skips TLS verification;
// trackers and exporters send tokens and passwords, so they get a copy
// that verifies certificates.
if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
if hc := tlsverify.Client(options.HttpClient.HTTPClient); hc != options.HttpClient.HTTPClient {
opts := retryablehttp.DefaultOptionsSingle
opts.Timeout = hc.Timeout // keep the caller's timeout, also when it is 0
options.HttpClient = retryablehttp.NewWithHTTPClient(hc, opts)
}
}
if options.GitHub != nil { if options.GitHub != nil {
options.GitHub.HttpClient = options.HttpClient options.GitHub.HttpClient = options.HttpClient
options.GitHub.OmitRaw = options.OmitRaw options.GitHub.OmitRaw = options.OmitRaw
+98
View File
@@ -0,0 +1,98 @@
package reporting_test
import (
"io"
"net"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/es"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitlab"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
// connectProxy tunnels every CONNECT to target, whatever host the client asked
// for, so no name is resolved and nothing leaves 127.0.0.1.
func connectProxy(t *testing.T, target string) *httptest.Server {
p := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodConnect {
http.Error(w, "connect only", http.StatusMethodNotAllowed)
return
}
up, err := net.Dial("tcp", target)
if err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
c, _, err := w.(http.Hijacker).Hijack()
if err != nil {
_ = up.Close()
return
}
_, _ = c.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n"))
go func() { _, _ = io.Copy(up, c); _ = up.Close() }()
go func() { _, _ = io.Copy(c, up); _ = c.Close() }()
}))
t.Cleanup(p.Close)
return p
}
// With -pi and a proxy, the runner gives reporting the scan client, which skips
// TLS verification. No tracker or exporter may send its secret to a server with
// an untrusted certificate.
func TestNewVerifiesTLSWithScanClient(t *testing.T) {
var hits atomic.Int32
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.WriteHeader(http.StatusUnauthorized)
}))
t.Cleanup(srv.Close)
proxy := connectProxy(t, srv.Listener.Addr().String())
opts := types.DefaultOptions()
opts.SetExecutionID("reporting-tls-test")
opts.ProxyInternal = true
opts.AliveHttpProxy = proxy.URL
require.NoError(t, protocolstate.Init(opts))
t.Cleanup(func() { protocolstate.Close(opts.ExecutionId) })
scanClient, err := httpclientpool.Get(opts, &httpclientpool.Configuration{}, "")
require.NoError(t, err)
event := &output.ResultEvent{
TemplateID: "test-template",
Host: "https://target.example",
Matched: "https://target.example/",
Info: model.Info{Name: "test", SeverityHolder: severity.Holder{Severity: severity.High}},
}
configs := map[string]*reporting.Options{
"github": {GitHub: &github.Options{BaseURL: "https://github.victim.example/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token"}},
"gitlab": {GitLab: &gitlab.Options{BaseURL: "https://gitlab.victim.example/", Username: "u", Token: "test-token", ProjectName: "1"}},
"gitea": {Gitea: &gitea.Options{BaseURL: "https://gitea.victim.example/", Token: "test-token", ProjectOwner: "o", ProjectName: "r"}},
"jira": {Jira: &jira.Options{Cloud: true, URL: "https://jira.victim.example", AccountID: "a", Email: "[email protected]", Token: "test-token", ProjectName: "P", IssueType: "Bug"}},
"elasticsearch": {ElasticsearchExporter: &es.Options{Host: "es.victim.example", Port: 443, SSL: true, Username: "u", Password: "test-password", IndexName: "n"}},
"splunk": {SplunkExporter: &splunk.Options{Host: "splunk.victim.example", Port: 443, SSL: true, Token: "test-token", IndexName: "n"}},
}
for name, ro := range configs {
ro.HttpClient = scanClient
ro.ExecutionId = opts.ExecutionId
client, err := reporting.New(ro, "", true)
if err == nil {
_ = client.CreateIssue(event)
client.Close()
}
require.Zero(t, hits.Load(), "%s sent a request to a server with an untrusted certificate", name)
}
}
+68
View File
@@ -0,0 +1,68 @@
// Package tlsverify gives trackers and exporters an HTTP client that verifies
// TLS certificates, since they send tokens and passwords.
package tlsverify
import (
"crypto/tls"
"errors"
"net/http"
)
// errTLSDialerOnly is returned when a transport dials HTTPS only through a
// custom TLS dialer: dropping that dialer would also drop its route or address
// policy, so the copy refuses requests instead.
var errTLSDialerOnly = errors.New("tlsverify: the transport has a custom TLS dialer and no DialContext, so certificate verification cannot be enforced")
type refusingTransport struct{}
func (refusingTransport) RoundTrip(*http.Request) (*http.Response, error) {
return nil, errTLSDialerOnly
}
// Client returns c, or a copy of c whose transport verifies TLS certificates
// when c can skip verification (the scan client can). The copy keeps the proxy
// and the dialer, so the address policy still applies, but it drops a custom
// TLS dialer, because that dialer can use the scan TLS config. So the token
// never goes over an unverified connection.
//
// A transport with an Unwrap() http.RoundTripper method declares that its
// RoundTrip only observes or caches requests and enforces no policy; the copy
// uses the unwrapped transport. Other transports are returned unchanged.
func Client(c *http.Client) *http.Client {
rt := c.Transport
for {
u, ok := rt.(interface{ Unwrap() http.RoundTripper })
if !ok {
break
}
rt = u.Unwrap()
}
t, ok := rt.(*http.Transport)
if !ok {
return c
}
insecure := t.TLSClientConfig != nil && t.TLSClientConfig.InsecureSkipVerify
tlsDialer := t.DialTLSContext != nil || t.DialTLS != nil //nolint:staticcheck // DialTLS is deprecated but still used
if !insecure && !tlsDialer {
return c
}
copied := *c
if tlsDialer && t.DialContext == nil && t.Dial == nil { //nolint:staticcheck // Dial is deprecated but still used
copied.Transport = refusingTransport{}
return &copied
}
verifying := t.Clone()
verifying.DialTLSContext = nil
verifying.DialTLS = nil //nolint:staticcheck // DialTLS is deprecated but still used
if insecure {
// Keep the caller's roots, client certificates and checks; the server
// name of the scan config belongs to a scan target, not to this host.
verifying.TLSClientConfig.InsecureSkipVerify = false
verifying.TLSClientConfig.ServerName = ""
if verifying.TLSClientConfig.MinVersion < tls.VersionTLS12 {
verifying.TLSClientConfig.MinVersion = tls.VersionTLS12
}
}
copied.Transport = verifying
return &copied
}
+67
View File
@@ -0,0 +1,67 @@
package tlsverify
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"net"
"net/http"
"testing"
"github.com/stretchr/testify/require"
)
// The scan client can skip TLS verification; the tracker must not send the
// token over such a connection, and must not change the caller's client.
func TestClientTurnsOnTLSVerification(t *testing.T) {
insecure := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
caller := &http.Client{Transport: insecure}
got := Client(caller)
gotTransport, ok := got.Transport.(*http.Transport)
require.True(t, ok)
require.False(t, gotTransport.TLSClientConfig.InsecureSkipVerify)
require.True(t, insecure.TLSClientConfig.InsecureSkipVerify, "the caller's transport changed")
secure := &http.Client{Transport: &http.Transport{}}
require.Same(t, secure, Client(secure))
}
func TestClientKeepsCallerTLSSettings(t *testing.T) {
pinned := func(tls.ConnectionState) error { return nil }
insecure := &http.Transport{
DialTLSContext: func(context.Context, string, string) (net.Conn, error) { return nil, errors.New("scan dialer") },
DialContext: (&net.Dialer{}).DialContext,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
ServerName: "scan-target.example",
MinVersion: tls.VersionTLS10,
RootCAs: x509.NewCertPool(),
VerifyConnection: pinned,
},
}
got := Client(&http.Client{Transport: insecure}).Transport.(*http.Transport)
require.Nil(t, got.DialTLSContext)
require.NotNil(t, got.DialContext)
cfg := got.TLSClientConfig
require.False(t, cfg.InsecureSkipVerify)
require.Empty(t, cfg.ServerName)
require.Equal(t, uint16(tls.VersionTLS12), cfg.MinVersion)
require.Same(t, insecure.TLSClientConfig.RootCAs, cfg.RootCAs)
require.NotNil(t, cfg.VerifyConnection)
require.True(t, insecure.TLSClientConfig.InsecureSkipVerify, "the caller's config changed")
}
// A custom TLS dialer may carry the only route or address policy, so a
// transport without DialContext cannot be rebuilt and must refuse requests.
func TestClientRefusesTLSDialerWithoutDialContext(t *testing.T) {
var dialed bool
tlsOnly := &http.Transport{DialTLSContext: func(context.Context, string, string) (net.Conn, error) {
dialed = true
return nil, errors.New("policy dialer")
}}
_, err := Client(&http.Client{Transport: tlsOnly}).Get("https://tracker.example/")
require.ErrorIs(t, err, errTLSDialerOnly)
require.False(t, dialed)
}
+6 -7
View File
@@ -4,7 +4,6 @@ import (
"context" "context"
"fmt" "fmt"
"io" "io"
"net/http"
"strconv" "strconv"
"github.com/google/go-github/v92/github" "github.com/google/go-github/v92/github"
@@ -12,6 +11,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/format"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
"github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/retryablehttp-go"
"golang.org/x/oauth2" "golang.org/x/oauth2"
@@ -55,17 +55,16 @@ type Options struct {
// New creates a new issue tracker integration client based on options. // New creates a new issue tracker integration client based on options.
func New(options *Options) (*Integration, error) { func New(options *Options) (*Integration, error) {
ctx := context.Background() ctx := context.Background()
// Build the oauth2 client on top of the caller's client, so its transport
// (proxy, address policy) and redirect rules apply to every request.
if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
ctx = context.WithValue(ctx, oauth2.HTTPClient, tlsverify.Client(options.HttpClient.HTTPClient))
}
ts := oauth2.StaticTokenSource( ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: options.Token}, &oauth2.Token{AccessToken: options.Token},
) )
tc := oauth2.NewClient(ctx, ts) tc := oauth2.NewClient(ctx, ts)
if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
if tcTransport, ok := tc.Transport.(*http.Transport); ok {
tcTransport.Proxy = options.HttpClient.HTTPClient.Transport.(*http.Transport).Proxy
}
}
clientOptions := []github.ClientOptionsFunc{github.WithHTTPClient(tc)} clientOptions := []github.ClientOptionsFunc{github.WithHTTPClient(tc)}
if options.BaseURL != "" { if options.BaseURL != "" {
clientOptions = append(clientOptions, github.WithURLs(&options.BaseURL, nil)) clientOptions = append(clientOptions, github.WithURLs(&options.BaseURL, nil))
@@ -0,0 +1,126 @@
package github
import (
"context"
"crypto/tls"
"errors"
"net"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/stretchr/testify/require"
)
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func testEvent() *output.ResultEvent {
return &output.ResultEvent{
TemplateID: "test-template",
Host: "https://target.example",
Info: model.Info{Name: "test", SeverityHolder: severity.Holder{Severity: severity.High}},
}
}
func clientWithTransport(rt http.RoundTripper) *retryablehttp.Client {
c := retryablehttp.NewClient(retryablehttp.DefaultOptionsSingle)
c.HTTPClient = &http.Client{Transport: rt}
return c
}
// The tracker must send every request through the caller's client, so a
// caller's address policy also covers the GitHub API base URL.
func TestNewUsesCallerHTTPClient(t *testing.T) {
var serverHits atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
serverHits.Add(1)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusCreated)
_, _ = w.Write([]byte(`{"number":1,"html_url":"https://github.example/o/r/issues/1"}`))
}))
defer srv.Close()
t.Run("requests go through the caller's transport", func(t *testing.T) {
var seen atomic.Int32
rt := roundTripFunc(func(r *http.Request) (*http.Response, error) {
seen.Add(1)
require.Equal(t, "Bearer test-token", r.Header.Get("Authorization"))
return http.DefaultTransport.RoundTrip(r)
})
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(rt)})
require.NoError(t, err)
_, err = tracker.CreateIssue(testEvent())
require.NoError(t, err)
require.Positive(t, seen.Load())
})
t.Run("a refusing transport stops the request", func(t *testing.T) {
before := serverHits.Load()
denied := errors.New("denied by address policy")
rt := roundTripFunc(func(*http.Request) (*http.Response, error) { return nil, denied })
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(rt)})
require.NoError(t, err)
_, err = tracker.CreateIssue(testEvent())
require.ErrorContains(t, err, "denied by address policy")
require.Equal(t, before, serverHits.Load(), "the request reached the server")
})
}
func TestNewRefusesUntrustedCertificateFromInsecureCallerClient(t *testing.T) {
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusCreated)
_, _ = w.Write([]byte(`{"number":1}`))
}))
defer srv.Close()
insecure := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(insecure)})
require.NoError(t, err)
_, err = tracker.CreateIssue(testEvent())
require.ErrorContains(t, err, "certificate")
}
type wrappingTransport struct{ base http.RoundTripper }
func (w *wrappingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
return w.base.RoundTrip(r)
}
func (w *wrappingTransport) Unwrap() http.RoundTripper { return w.base }
// The scan client wraps its transport and, with a SOCKS proxy, does TLS in its
// own dialer with the scan TLS config, which skips verification.
func TestNewRefusesUntrustedCertificateFromInsecureTLSDialer(t *testing.T) {
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusCreated)
_, _ = w.Write([]byte(`{"number":1}`))
}))
defer srv.Close()
var dialed atomic.Int32
var dialer net.Dialer
base := &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
dialed.Add(1)
return dialer.DialContext(ctx, network, addr)
},
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
conn, err := dialer.DialContext(ctx, network, addr)
if err != nil {
return nil, err
}
return tls.Client(conn, &tls.Config{InsecureSkipVerify: true}), nil
},
}
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(&wrappingTransport{base: base})})
require.NoError(t, err)
_, err = tracker.CreateIssue(testEvent())
require.ErrorContains(t, err, "certificate")
require.Positive(t, dialed.Load(), "the caller's dialer was not used")
}
-4
View File
@@ -254,9 +254,6 @@ type Options struct {
// PlanFile is where the work a scan would do is written as JSON lines; // PlanFile is where the work a scan would do is written as JSON lines;
// nuclei exits after writing it, without sending requests. // nuclei exits after writing it, without sending requests.
PlanFile string PlanFile string
// RunPlanFile is a plan whose entries the scan runs, skipping those its
// done log lists as finished.
RunPlanFile string
// StoreEvents is where the scan writes the event of every request, as // StoreEvents is where the scan writes the event of every request, as
// JSON lines, for -replay. // JSON lines, for -replay.
StoreEvents string StoreEvents string
@@ -724,7 +721,6 @@ func (options *Options) Copy() *Options {
Silent: options.Silent, Silent: options.Silent,
Validate: options.Validate, Validate: options.Validate,
PlanFile: options.PlanFile, PlanFile: options.PlanFile,
RunPlanFile: options.RunPlanFile,
StoreEvents: options.StoreEvents, StoreEvents: options.StoreEvents,
ReplayFile: options.ReplayFile, ReplayFile: options.ReplayFile,
NoStrictSyntax: options.NoStrictSyntax, NoStrictSyntax: options.NoStrictSyntax,
+1 -4
View File
@@ -18,13 +18,10 @@ func TestCopyPreservesPayloadRandomOrder(t *testing.T) {
} }
func TestCopyPreservesPlanFile(t *testing.T) { func TestCopyPreservesPlanFile(t *testing.T) {
options := &Options{PlanFile: "plan.jsonl", RunPlanFile: "run.jsonl"} options := &Options{PlanFile: "plan.jsonl"}
if options.Copy().PlanFile != "plan.jsonl" { if options.Copy().PlanFile != "plan.jsonl" {
t.Fatal("Copy must preserve PlanFile, or an embedded caller loses plan mode") t.Fatal("Copy must preserve PlanFile, or an embedded caller loses plan mode")
} }
if options.Copy().RunPlanFile != "run.jsonl" {
t.Fatal("Copy must preserve RunPlanFile, or an embedded caller runs the whole scan")
}
} }
func TestToJSONMapRejectsKeysSharingAStringForm(t *testing.T) { func TestToJSONMapRejectsKeysSharingAStringForm(t *testing.T) {