Compare commits

..
Author SHA1 Message Date
Mzack9999 8903942700 bump rod 2026-11-13 16:03:50 +01:00
Mzack9999 aec1420bdb rod module 2026-11-13 14:41:54 +01:00
Dogan Can Bakir b2ea1489fd Merge pull request #8101 from projectdiscovery/feat/run-plan
add -run-plan to run a plan file and resume it
2026-11-11 07:27:33 +03:00
Dogan Can Bakir e6ce143015 Merge remote-tracking branch 'origin/dev' into feat/run-plan
# Conflicts:
#	internal/runner/runner.go
#	pkg/types/types.go
2026-11-11 07:05:20 +03:00
Mzack9999 d3d84b2be8 Merge pull request #8107 from projectdiscovery/feat/shared-responses
send identical requests once per scan
2026-11-10 20:44:27 +01:00
Mzack9999 67cda5044b fix test 2026-11-10 20:18:00 +01:00
Mzack9999 a335de0be0 keep mismatch 2026-11-10 16:06:38 +01:00
Mzack9999 df5b8b367c fix reuse 2026-11-10 15:40:16 +01:00
Dogan Can Bakir f9192b96ae keep responses from requests whose operators read the duration, count entry overhead toward the cap 2026-11-10 02:00:00 +03:00
Dogan Can Bakir 448ee69710 start shared responses empty at each scan 2026-11-10 01:23:19 +03:00
Dogan Can Bakir cb1f0757d4 send identical requests once per scan and share the response across templates 2026-11-10 00:44:19 +03:00
Mzack9999 894f90dd11 reject flags 2026-11-09 20:31:19 +01:00
Dogan Can Bakir a351263c86 add -run-plan to run a plan file and resume it from its done log 2026-11-09 20:18:42 +03:00
47 changed files with 1135 additions and 95 deletions

No files matched your search

+1
View File
@@ -167,6 +167,7 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -165,6 +165,7 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -165,6 +165,7 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -182,6 +182,7 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -165,6 +165,7 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -165,6 +165,7 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -165,6 +165,7 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -165,6 +165,7 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -165,6 +165,7 @@ TEMPLATES:
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
+1
View File
@@ -314,6 +314,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
flagSet.StringVar(&options.PlanFile, "plan", "", "write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target"),
flagSet.StringVar(&options.RunPlanFile, "run-plan", "", "run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes"),
flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all templates matching current filters"),
+4 -8
View File
@@ -1,6 +1,6 @@
module github.com/projectdiscovery/nuclei/v3
go 1.27.3
go 1.27.4
require (
carvel.dev/ytt v0.52.0
@@ -46,7 +46,6 @@ require (
github.com/go-pdf/fpdf v0.9.0
github.com/go-pg/pg/v10 v10.15.0
github.com/go-playground/validator/v10 v10.26.0
github.com/go-rod/rod v0.116.2
github.com/go-sql-driver/mysql v1.10.0
github.com/gobwas/ws v1.4.0
github.com/goccy/go-json v0.10.5
@@ -85,7 +84,7 @@ require (
github.com/projectdiscovery/gcache v0.0.0-20241015120333-12546c6e3f4c
github.com/projectdiscovery/goflags v0.2.1
github.com/projectdiscovery/goimpacket v0.1.1-0.20261007205806-6042a8213eeb
github.com/projectdiscovery/goja v0.0.0-20261105204359-4285f3c977ed
github.com/projectdiscovery/goja v0.0.0-20261012075649-9b78f2e05b9c
github.com/projectdiscovery/goja_nodejs v0.0.0-20260618132410-8519f75f703d
github.com/projectdiscovery/gologger v1.1.73
github.com/projectdiscovery/gostruct v0.0.2
@@ -102,6 +101,7 @@ require (
github.com/projectdiscovery/rdap v0.9.0
github.com/projectdiscovery/retryabledns v1.0.116
github.com/projectdiscovery/retryablehttp-go v1.3.31
github.com/projectdiscovery/rod v0.116.3-0.20261113150229-d775599ff53d
github.com/projectdiscovery/sarif v0.1.0
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75
github.com/projectdiscovery/uncover v1.2.1
@@ -237,7 +237,7 @@ require (
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/djherbis/times v1.6.0 // indirect
github.com/dlclark/regexp2/v2 v2.11.0 // indirect
github.com/dlclark/regexp2/v2 v2.2.1 // indirect
github.com/docker/cli v29.2.0+incompatible // indirect
github.com/docker/go-connections v0.7.0 // indirect
github.com/dop251/goja v0.0.0-20260311135729-065cd970411c // indirect
@@ -427,11 +427,7 @@ require (
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
github.com/ysmood/fetchup v0.2.3 // indirect
github.com/ysmood/goob v0.4.0 // indirect
github.com/ysmood/got v0.40.0 // indirect
github.com/ysmood/gson v0.7.3 // indirect
github.com/ysmood/leakless v0.9.0 // indirect
github.com/yuin/goldmark v1.7.17 // indirect
github.com/yuin/goldmark-emoji v1.0.6 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
+10 -16
View File
@@ -358,8 +358,8 @@ github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2/v2 v2.11.0 h1:z5uRPHav+eTVpoffDSuUP2KPS4f1ONZaPDK73S8HA0w=
github.com/dlclark/regexp2/v2 v2.11.0/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
github.com/dlclark/regexp2/v2 v2.2.1 h1:mf4KkFUj0gJuarK8P+LgiS+Lit7m9N1yAwEfPbee7R0=
github.com/dlclark/regexp2/v2 v2.2.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
github.com/docker/cli v29.2.0+incompatible h1:9oBd9+YM7rxjZLfyMGxjraKBKE4/nVyvVfN4qNl9XRM=
github.com/docker/cli v29.2.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c=
@@ -493,8 +493,6 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k=
github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA=
github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg=
github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
@@ -945,8 +943,8 @@ github.com/projectdiscovery/goflags v0.2.1 h1:XnSYvsCjUwHbZaaJUZ3YznCJPoXRk/IhTF
github.com/projectdiscovery/goflags v0.2.1/go.mod h1:7nAP1r2Dqgn/rwmOE3EWbZWUCEJKNIhVSBGpuzJAIns=
github.com/projectdiscovery/goimpacket v0.1.1-0.20261007205806-6042a8213eeb h1:+L1yW2PblRHSSHQRTmvxLkGDPZnM52N62oeBd2xHKF0=
github.com/projectdiscovery/goimpacket v0.1.1-0.20261007205806-6042a8213eeb/go.mod h1:bfgULaYf+6pjjcLCekpXIYElnLJbmhvTttxMAbQJvhY=
github.com/projectdiscovery/goja v0.0.0-20261105204359-4285f3c977ed h1:h6aBYgIrjnapch77HWnpCPPz+bTU2OMfWOrr1fr0knA=
github.com/projectdiscovery/goja v0.0.0-20261105204359-4285f3c977ed/go.mod h1:sjYPrVMMCZNujHasGmCZGs3Hx8DymbJA2plsJjlC3LA=
github.com/projectdiscovery/goja v0.0.0-20261012075649-9b78f2e05b9c h1:4H/CfVNlue9PIwsCM1IGxOynDI7cir+emXuDvGJTLJs=
github.com/projectdiscovery/goja v0.0.0-20261012075649-9b78f2e05b9c/go.mod h1:SO0AP+uKfYeeoR6jyVH/PKRYJE/f5gJrPMAM00iGEMc=
github.com/projectdiscovery/goja_nodejs v0.0.0-20260618132410-8519f75f703d h1:fqqH9LHpN2WDz9QuxFrhKNxXSRtzk+Sa6jAhbB7tXcQ=
github.com/projectdiscovery/goja_nodejs v0.0.0-20260618132410-8519f75f703d/go.mod h1:Ezmbgdaw4EunGGBU4MQViLoGMJc37LA3ip55YV3KeRI=
github.com/projectdiscovery/gologger v1.1.73 h1:WX2nzq29cL7HdvOHqIttacoid/5lMwapyM7ukGU1jFY=
@@ -985,6 +983,8 @@ github.com/projectdiscovery/retryabledns v1.0.116 h1:tK28PJ4tLO7bgM89RpIFjkfTPMS
github.com/projectdiscovery/retryabledns v1.0.116/go.mod h1:r3vH0AKYtJ9fculcqRFrdVYe2ABcHwZeR4C6j3rqCbY=
github.com/projectdiscovery/retryablehttp-go v1.3.31 h1:0hSEZ5+jj6Oo1IL8qrytD36TK/qCxfe543jEaRqSTjs=
github.com/projectdiscovery/retryablehttp-go v1.3.31/go.mod h1:/WEa7YejxTCXEMLCGfGYaC3JsgXgzuqw8rv8cb4buEc=
github.com/projectdiscovery/rod v0.116.3-0.20261113150229-d775599ff53d h1:6mwNBdlYWe0hu1/QLLhIrZiz+G1JYsU7OIVmCOaF+q0=
github.com/projectdiscovery/rod v0.116.3-0.20261113150229-d775599ff53d/go.mod h1:FLVLMgXw6OFcOU5nd4Il8cdSWgDfAlDeRZMol3drmGs=
github.com/projectdiscovery/sarif v0.1.0 h1:O541T+a448nSJsmIMnXXSOeDQEzpnCAYvRfe0eG5h74=
github.com/projectdiscovery/sarif v0.1.0/go.mod h1:LBC+reM3bkI3qIIhE0rZaINaYX6VG+En6u2hHa5mA7E=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -1212,20 +1212,14 @@ github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+Seva
github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ=
github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
github.com/ysmood/gop v0.2.0 h1:+tFrG0TWPxT6p9ZaZs+VY+opCvHU8/3Fk6BaNv6kqKg=
github.com/ysmood/gop v0.2.0/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
github.com/ysmood/got v0.40.0 h1:ZQk1B55zIvS7zflRrkGfPDrPG3d7+JOza1ZkNxcc74Q=
github.com/ysmood/got v0.40.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg=
github.com/ysmood/gop v0.4.2 h1:KtW1/RugapWs1iW9ynQdk52BK6TWdP9zC0qDOhW8EDE=
github.com/ysmood/gop v0.4.2/go.mod h1:K4LAB/BdLArknXrVSy+DyXcfb8P85JwAtwYM8zlb0mU=
github.com/ysmood/got v0.43.0 h1:bcHlKg+qnhFz5JxNZ+j4fIfxxB8WWjTh6qdqeS8FfZw=
github.com/ysmood/got v0.43.0/go.mod h1:rsfH2KJ5rcS/5n9pmybIGodoHlT/NTYxjgh3ob7UdSU=
github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU=
github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+18 -5
View File
@@ -558,18 +558,31 @@ func getBoolEnvValue(key string) bool {
// validatePlanOptions rejects options that change the templates or targets
// only once scanning starts, which a plan written beforehand cannot show.
func validatePlanOptions(options *types.Options) error {
if options.PlanFile == "" {
if options.RunPlanFile != "" && options.PlanFile != "" {
return errors.New("-plan cannot be used with -run-plan")
}
if options.RunPlanFile != "" && options.Resume != "" {
return errors.New("-run-plan cannot be used with -resume, the plan's done log resumes it")
}
if options.PlanFile == "" && options.RunPlanFile == "" {
return nil
}
// -automatic-scan builds its own engine, so a plan set on the main engine
// would never be consulted. The other flags change the targets only after
// a plan has been written.
flag := "-plan"
if options.PlanFile == "" {
flag = "-run-plan"
}
switch {
case options.AutomaticScan:
return errors.New("-plan cannot be used with -automatic-scan, which picks templates while scanning")
return fmt.Errorf("%s cannot be used with -automatic-scan, which picks templates while scanning", flag)
case options.Uncover || len(options.UncoverQuery) > 0:
return errors.New("-plan cannot be used with -uncover, which finds targets over the network")
return fmt.Errorf("%s cannot be used with -uncover, which finds targets over the network", flag)
case options.PreflightPortScan:
return errors.New("-plan cannot be used with -preflight-portscan, which drops targets after scanning their ports")
return fmt.Errorf("%s cannot be used with -preflight-portscan, which drops targets after scanning their ports", flag)
case options.DASTServer || options.DASTProxy:
return errors.New("-plan cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan")
return fmt.Errorf("%s cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan", flag)
}
return nil
}
+15
View File
@@ -23,4 +23,19 @@ func TestPlanRejectsOptionsThatChangeTargetsLater(t *testing.T) {
require.NoError(t, validatePlanOptions(&types.Options{Uncover: true}), "only -plan restricts them")
require.NoError(t, validatePlanOptions(&types.Options{UncoverQuery: goflags.StringSlice{"example"}}))
require.NoError(t, validatePlanOptions(&types.Options{DASTServer: true}))
require.Error(t, validatePlanOptions(&types.Options{RunPlanFile: "p", PlanFile: "q"}))
require.Error(t, validatePlanOptions(&types.Options{RunPlanFile: "p", Resume: "r"}))
require.NoError(t, validatePlanOptions(&types.Options{RunPlanFile: "p"}))
for name, options := range map[string]*types.Options{
"run automatic scan": {RunPlanFile: "p", AutomaticScan: true},
"run uncover": {RunPlanFile: "p", Uncover: true},
"run uncover query": {RunPlanFile: "p", UncoverQuery: goflags.StringSlice{"example"}},
"run preflight": {RunPlanFile: "p", PreflightPortScan: true},
"run dast server": {RunPlanFile: "p", DASTServer: true},
"run dast proxy": {RunPlanFile: "p", DASTProxy: true},
} {
require.Error(t, validatePlanOptions(options), name)
}
}
+215
View File
@@ -0,0 +1,215 @@
package runner
import (
"bufio"
"bytes"
"fmt"
"io"
"os"
"strconv"
"sync"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
)
// planRun limits a scan to the entries of a plan file and appends the id of
// each finished entry to the plan's done log, so running the plan again skips
// them. Ids are derived from what an entry runs, not from its position, so the
// log still applies after the plan is written again for changed targets or
// templates.
type planRun struct {
mu sync.Mutex
entries map[uint64]plannedEntry
// done holds finished origin groups by the id of their first entry; an
// entry outside a group is its own group
done map[uint64]struct{}
log *os.File
logErr error
// resumed counts the entries finished before this run
resumed int
}
type plannedEntry struct {
group uint64
seen bool
}
func planDoneLog(planFile string) string {
return planFile + ".done"
}
func openPlanRun(planFile string) (*planRun, error) {
run := &planRun{entries: make(map[uint64]plannedEntry), done: make(map[uint64]struct{})}
if err := run.readPlan(planFile); err != nil {
return nil, err
}
if err := run.readDoneLog(planDoneLog(planFile)); err != nil {
return nil, err
}
log, err := os.OpenFile(planDoneLog(planFile), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o644)
if err != nil {
return nil, err
}
// an interrupted write leaves a partial line, which the next id must not
// extend
if info, err := log.Stat(); err == nil && info.Size() > 0 {
last := make([]byte, 1)
if _, err := log.ReadAt(last, info.Size()-1); err == nil && last[0] != '\n' {
_, _ = log.WriteString("\n")
}
}
run.log = log
return run, nil
}
func (run *planRun) readPlan(planFile string) error {
file, err := os.Open(planFile)
if err != nil {
return err
}
defer func() { _ = file.Close() }()
reader := bufio.NewReader(file)
for number := 1; ; number++ {
line, readErr := reader.ReadBytes('\n')
if line = bytes.TrimSpace(line); len(line) > 0 {
var entry struct {
ID string `json:"id"`
AlternativeTo string `json:"alternative-to"`
}
if err := json.Unmarshal(line, &entry); err != nil {
return fmt.Errorf("%s:%d: %w", planFile, number, err)
}
// the summary line has no id
if entry.ID != "" {
id, ok := parsePlanID(entry.ID)
if !ok {
return fmt.Errorf("%s:%d: invalid id %q", planFile, number, entry.ID)
}
group := id
if entry.AlternativeTo != "" {
if group, ok = parsePlanID(entry.AlternativeTo); !ok {
return fmt.Errorf("%s:%d: invalid alternative-to %q", planFile, number, entry.AlternativeTo)
}
}
run.entries[id] = plannedEntry{group: group}
}
}
if readErr == io.EOF {
return nil
}
if readErr != nil {
return readErr
}
}
}
func (run *planRun) readDoneLog(path string) error {
file, err := os.Open(path)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
defer func() { _ = file.Close() }()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
// a partial line from an interrupted write does not parse and is
// run again
id, ok := parsePlanID(scanner.Text())
if !ok {
continue
}
if entry, planned := run.entries[id]; planned {
run.done[entry.group] = struct{}{}
}
}
if err := scanner.Err(); err != nil {
return err
}
for _, entry := range run.entries {
if _, done := run.done[entry.group]; done {
run.resumed++
}
}
return nil
}
// parsePlanID keeps ids as integers, which a plan of millions of entries
// holds in a fraction of the memory of strings.
func parsePlanID(id string) (uint64, bool) {
if len(id) != 16 {
return 0, false
}
value, err := strconv.ParseUint(id, 16, 64)
return value, err == nil
}
func (run *planRun) Skip(id string) bool {
key, ok := parsePlanID(id)
if !ok {
return true
}
run.mu.Lock()
defer run.mu.Unlock()
entry, planned := run.entries[key]
if !planned {
return true
}
if !entry.seen {
entry.seen = true
run.entries[key] = entry
}
_, done := run.done[entry.group]
return done
}
func (run *planRun) Finished(id string) {
key, ok := parsePlanID(id)
if !ok {
return
}
run.mu.Lock()
defer run.mu.Unlock()
entry, planned := run.entries[key]
if !planned {
return
}
run.done[entry.group] = struct{}{}
// written straight to the file: ctrl+c exits without running defers
if _, err := run.log.WriteString(id + "\n"); err != nil && run.logErr == nil {
run.logErr = err
}
}
// unreached counts unfinished entries the scan did not get to: those of a
// host it found unresponsive, and those it does not have because it runs
// with other templates, targets or options than the plan.
func (run *planRun) unreached() int {
run.mu.Lock()
defer run.mu.Unlock()
count := 0
for _, entry := range run.entries {
if _, done := run.done[entry.group]; !done && !entry.seen {
count++
}
}
return count
}
func (run *planRun) close() error {
if err := run.log.Close(); err != nil && run.logErr == nil {
run.logErr = err
}
return run.logErr
}
// finishPlanRun closes the done log and reports the entries left to run.
func (r *Runner) finishPlanRun(run *planRun) {
if err := run.close(); err != nil {
r.Logger.Error().Msgf("Could not write %s: %s", planDoneLog(r.options.RunPlanFile), err)
}
if unreached := run.unreached(); unreached > 0 {
r.Logger.Info().Msgf("%d plan entries did not run; running the plan again retries them. Entries run only when the scan has the templates, targets and options the plan was written with", unreached)
}
}
+52
View File
@@ -0,0 +1,52 @@
package runner
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
)
func TestPlanRunResumesFromItsDoneLog(t *testing.T) {
planFile := filepath.Join(t.TempDir(), "plan.jsonl")
require.NoError(t, os.WriteFile(planFile, []byte(`{"id":"00000000000000a1","target":"https://a.example/x","templates":["root"]}
{"id":"00000000000000a2","target":"https://a.example/y","templates":["root"],"alternative-to":"00000000000000a1"}
{"id":"00000000000000b1","target":"https://b.example","templates":["generic"]}
{"id":"00000000000000c1","target":"https://c.example","templates":["generic"]}
{"summary":{"entries":3,"alternatives":1}}
`), 0o644))
// the last line was cut off by an interrupted write
require.NoError(t, os.WriteFile(planDoneLog(planFile), []byte("00000000000000b1\n00000000"), 0o644))
run, err := openPlanRun(planFile)
require.NoError(t, err)
require.Len(t, run.entries, 4)
require.Equal(t, 1, run.resumed)
require.True(t, run.Skip("00000000000000b1"), "finished in the previous run")
require.True(t, run.Skip("00000000000000ff"), "not in the plan")
require.False(t, run.Skip("00000000000000a1"))
run.Finished("00000000000000a1")
require.True(t, run.Skip("00000000000000a2"), "its origin is covered")
require.Equal(t, 1, run.unreached(), "c1 never ran")
require.NoError(t, run.close())
log, err := os.ReadFile(planDoneLog(planFile))
require.NoError(t, err)
require.Equal(t, "00000000000000b1\n00000000\n00000000000000a1\n", string(log), "the partial line is not extended")
again, err := openPlanRun(planFile)
require.NoError(t, err)
require.Equal(t, 3, again.resumed, "b1, and a1 with its alternative")
require.True(t, again.Skip("00000000000000a2"), "a finished origin stays covered")
require.False(t, again.Skip("00000000000000c1"))
require.NoError(t, again.close())
}
func TestPlanRunRejectsAMalformedPlan(t *testing.T) {
planFile := filepath.Join(t.TempDir(), "plan.jsonl")
require.NoError(t, os.WriteFile(planFile, []byte("{\"id\":\"00000000000000a1\"}\nnot json\n"), 0o644))
_, err := openPlanRun(planFile)
require.ErrorContains(t, err, "plan.jsonl:2")
}
+9
View File
@@ -855,6 +855,15 @@ func (r *Runner) RunEnumeration() error {
}
}()
}
if r.options.RunPlanFile != "" {
run, err := openPlanRun(r.options.RunPlanFile)
if err != nil {
return errors.Wrap(err, "could not read plan")
}
r.Logger.Info().Msgf("Running %d plan entries from %s, %d already finished", len(run.entries), r.options.RunPlanFile, run.resumed)
executorEngine.SetPlanRun(run)
defer r.finishPlanRun(run)
}
// TODO: remove below functions after v3 or update warning messages
templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
+1
View File
@@ -51,6 +51,7 @@ type Engine struct {
executerOpts *protocols.ExecutorOptions
Callback func(*output.ResultEvent) // Executed on results
templateExecutionCallback TemplateExecutionCallback
planRun PlanRun
Logger *gologger.Logger
}
+2
View File
@@ -9,6 +9,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
@@ -35,6 +36,7 @@ func (e *Engine) ExecuteWithResults(ctx context.Context, templatesList []*templa
func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider, noCluster bool) *atomic.Bool {
results := &atomic.Bool{}
selfcontainedWg := &sync.WaitGroup{}
httpclientpool.ResetSharedResponses(e.options)
totalReqBeforeCluster := getRequestCount(templatesList) * int(target.Count())
+17 -1
View File
@@ -36,9 +36,13 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
if sharedBudget {
defer e.options.ReleaseTemplateThread()
}
defer e.tasksDone(1)
planned, skip := e.claimPlanned(template, &contextargs.MetaInput{})
if skip {
return
}
finished := e.templateExecutionStarted(template, "")
defer func() { finished(ctx.Err()) }()
defer e.tasksDone(1)
var err error
var match bool
ctx := scan.NewScanContext(ctx, contextargs.New(ctx))
@@ -55,6 +59,8 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
}
if err != nil {
e.options.Logger.Warning().Msgf("[%s] Could not execute step (self-contained): %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
} else if ctx.Context().Err() == nil {
planned()
}
results.CompareAndSwap(false, match)
}(v, usesSharedTemplateBudget)
@@ -330,6 +336,16 @@ func (e *Engine) tasksDone(count int64) {
func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates.Template, value *contextargs.MetaInput) (matched bool, err error) {
// a pair skipped as a repeat of its origin is done as well
defer e.tasksDone(1)
planned, skip := e.claimPlanned(template, value)
if skip {
return false, nil
}
// an origin a sibling already covered is settled too, so it is recorded
defer func() {
if err == nil && ctx.Err() == nil {
planned()
}
}()
// A template fixed to the target's origin makes the same requests for every
// target that shares one, which a crawled list produces by the hundred.
// Running it once per origin sends exactly the same traffic, minus the
+43 -2
View File
@@ -104,7 +104,7 @@ func planEntry(template *templates.Template, value *contextargs.MetaInput, selec
entry := PlanEntry{
Target: value.Input,
IP: value.CustomIP,
Templates: []string{template.ID},
Templates: planTemplates(template, selection),
Protocol: template.Type().String(),
Requests: template.TotalRequests,
Dynamic: template.Flow != "" || template.IsFuzzableRequest(),
@@ -126,7 +126,6 @@ func planEntry(template *templates.Template, value *contextargs.MetaInput, selec
entry.Dynamic = true
case isCluster(template):
cluster := template.Executer.(*templates.ClusterExecuter)
entry.Templates = slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
// TotalRequests of a cluster counts only its http and dns requests,
// so an ssl cluster would show none
entry.Requests = cluster.Requests()
@@ -143,6 +142,15 @@ func planEntry(template *templates.Template, value *contextargs.MetaInput, selec
return entry
}
// planTemplates lists the templates an entry runs, sorted so the entry id does
// not depend on clustering order.
func planTemplates(template *templates.Template, selection protocols.TemplateSelection) []string {
if cluster, ok := template.Executer.(*templates.ClusterExecuter); ok {
return slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
}
return []string{template.ID}
}
func isCluster(template *templates.Template) bool {
_, ok := template.Executer.(*templates.ClusterExecuter)
return ok
@@ -154,3 +162,36 @@ func planEntryID(entry PlanEntry) string {
sum := sha256.Sum256([]byte(entry.Target + "\x00" + entry.IP + "\x00" + strings.Join(entry.Templates, "\x00")))
return hex.EncodeToString(sum[:8])
}
// PlanRun limits a scan to the entries of a plan and records the ones that
// finish, so running the plan again resumes it.
type PlanRun interface {
// Skip reports whether the entry is not in the plan or already finished.
Skip(id string) bool
// Finished records an entry that ran without error.
Finished(id string)
}
// SetPlanRun makes the scan run only the entries run allows.
func (e *Engine) SetPlanRun(run PlanRun) {
e.planRun = run
}
// claimPlanned reports whether the pair is skipped under a plan run, and
// otherwise returns the func that records it finished. Without a plan run
// every pair runs.
func (e *Engine) claimPlanned(template *templates.Template, value *contextargs.MetaInput) (finished func(), skip bool) {
if e.planRun == nil {
return func() {}, false
}
// as in Plan, a self-contained template is not narrowed by any target
var selection protocols.TemplateSelection
if !template.SelfContained {
selection = e.selectionFor(value)
}
id := planEntryID(PlanEntry{Target: value.Input, IP: value.CustomIP, Templates: planTemplates(template, selection)})
if e.planRun.Skip(id) {
return nil, true
}
return func() { e.planRun.Finished(id) }, false
}
+59
View File
@@ -154,3 +154,62 @@ func TestPlanListsOriginAlternatives(t *testing.T) {
})
require.Equal(t, plan, again)
}
// fixedPlanRun is a plan run over fixed ids that records the finished ones.
type fixedPlanRun struct {
mu sync.Mutex
allowed map[string]bool
finished []string
}
func (a *fixedPlanRun) Skip(id string) bool { return !a.allowed[id] }
func (a *fixedPlanRun) Finished(id string) {
a.mu.Lock()
defer a.mu.Unlock()
a.finished = append(a.finished, id)
}
func TestPlanRunRunsOnlyThePlannedPairs(t *testing.T) {
options := &types.Options{BulkSize: 2, TemplateThreads: 2, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
engine := New(options)
engine.SetExecuterOptions(&protocols.ExecutorOptions{
Logger: engine.Logger,
Options: options,
ResumeCfg: types.NewResumeCfg(),
ProtocolType: tmpltypes.HTTPProtocol,
Progress: &progressRecorder{},
TargetScope: scopeByInput{"php": {paths: []string{"thinkphp.yaml"}}},
})
var (
mu sync.Mutex
runs []string
)
var templatesList []*templates.Template
for _, path := range []string{"thinkphp.yaml", "generic.yaml"} {
templatesList = append(templatesList, &templates.Template{
ID: path,
Path: path,
TotalRequests: 1,
Executer: &recordingExecuter{path: path, mu: &mu, runs: &runs},
})
}
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "php"}, {Input: "plain"}}}
var plan []PlanEntry
engine.Plan(templatesList, targets, true, func(entry PlanEntry) { plan = append(plan, entry) })
require.Len(t, plan, 3)
// the first entry is left out of the plan, as if an agent removed it
run := &fixedPlanRun{allowed: map[string]bool{plan[1].ID: true, plan[2].ID: true}}
engine.SetPlanRun(run)
engine.ExecuteScanWithOpts(context.Background(), templatesList, targets, true)
var want []string
for _, entry := range plan[1:] {
want = append(want, entry.Templates[0]+"@"+entry.Target)
}
require.ElementsMatch(t, want, runs, "only the planned pairs run")
require.ElementsMatch(t, []string{plan[1].ID, plan[2].ID}, run.finished, "the scan derives the ids the plan wrote")
}
-30
View File
@@ -1,30 +0,0 @@
package compiler
import (
"strings"
"testing"
"github.com/stretchr/testify/require"
)
// A Set that contains itself used to overflow the Go stack on export and kill the process.
func TestExecuteSelfReferencingSetResult(t *testing.T) {
result, err := executeScript(t, t.Name(), false, `var s = new Set(); s.add(s); s`)
require.NoError(t, err)
require.NotNil(t, result["response"])
}
// Deeply nested source used to overflow the Go stack in the parser.
func TestCompileDeepNesting(t *testing.T) {
const n = 1000000
_, err := SourceAutoMode(strings.Repeat("(", n)+"a"+strings.Repeat(")", n), false)
require.ErrorContains(t, err, "Maximum nesting depth exceeded")
}
// A context-first Go function called with fewer arguments used to panic in reflect.
func TestContextFunctionMissingArguments(t *testing.T) {
result, _ := executeScript(t, t.Name(), false, `require("nuclei/redis").GetServerInfo("127.0.0.1")`)
errText, _ := result["error"].(string)
require.NotContains(t, errText, "reflect")
require.NotContains(t, errText, "panic")
}
@@ -16,6 +16,7 @@ type Dialers struct {
HTTPClientPool *HTTPPool
PerHostRateLimitPool any // *hostratelimit.PerHostRateLimitPool
HTTPToHTTPSPortTracker any // *httpclientpool.HTTPToHTTPSPortTracker
SharedResponses any // *httpclientpool.sharedResponses
HTTPDesyncHosts *ExpiringSet
NetworkPolicy *networkpolicy.NetworkPolicy
LocalFileAccessAllowed bool
@@ -7,10 +7,10 @@ import (
"strings"
"sync"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/networkpolicy"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/proto"
"github.com/projectdiscovery/utils/errkit"
stringsutil "github.com/projectdiscovery/utils/strings"
urlutil "github.com/projectdiscovery/utils/url"
@@ -15,7 +15,7 @@ func TestMemGuardianGoroutineLeak(t *testing.T) {
defer goleak.VerifyNone(t,
goleak.IgnoreAnyContainingPkg("go.opencensus.io/stats/view"),
goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"),
goleak.IgnoreAnyContainingPkg("github.com/go-rod/rod"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/rod"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/server"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/ratelimit"),
// expirable LRU cache creates a background goroutine for TTL expiration that persists
@@ -104,7 +104,7 @@ func TestMemGuardianReset(t *testing.T) {
defer goleak.VerifyNone(t,
goleak.IgnoreAnyContainingPkg("go.opencensus.io/stats/view"),
goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"),
goleak.IgnoreAnyContainingPkg("github.com/go-rod/rod"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/rod"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/server"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/ratelimit"),
// expirable LRU cache creates a background goroutine for TTL expiration that persists
+3 -3
View File
@@ -7,10 +7,10 @@ import (
"strings"
"sync"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher"
"github.com/go-rod/rod/lib/launcher/flags"
"github.com/pkg/errors"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/launcher"
"github.com/projectdiscovery/rod/lib/launcher/flags"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
+2 -2
View File
@@ -5,8 +5,8 @@ package engine
import (
"encoding/base64"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/proto"
)
// NewHijack create hijack from page.
+2 -2
View File
@@ -5,11 +5,11 @@ import (
"errors"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/utils"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/utils"
)
// Instance is an isolated browser instance opened for doing operations with it.
@@ -5,9 +5,9 @@ import (
"testing"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/cdp"
"github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/cdp"
"github.com/projectdiscovery/rod/lib/proto"
"github.com/stretchr/testify/require"
)
+2 -2
View File
@@ -11,8 +11,6 @@ import (
"sync"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
@@ -21,6 +19,8 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/proto"
"github.com/projectdiscovery/utils/errkit"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -11,10 +11,6 @@ import (
"sync"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/input"
"github.com/go-rod/rod/lib/proto"
"github.com/go-rod/rod/lib/utils"
"github.com/kitabisa/go-ci"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
@@ -24,6 +20,10 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
filepathutil "github.com/projectdiscovery/nuclei/v3/pkg/utils/filepath"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/input"
"github.com/projectdiscovery/rod/lib/proto"
"github.com/projectdiscovery/rod/lib/utils"
contextutil "github.com/projectdiscovery/utils/context"
"github.com/projectdiscovery/utils/errkit"
fileutil "github.com/projectdiscovery/utils/file"
+3 -3
View File
@@ -11,12 +11,12 @@ import (
"testing"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/cdp"
"github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/cdp"
"github.com/projectdiscovery/rod/lib/proto"
"github.com/stretchr/testify/require"
)
+3 -3
View File
@@ -10,12 +10,12 @@ import (
"testing"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher"
"github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/networkpolicy"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/launcher"
"github.com/projectdiscovery/rod/lib/proto"
"github.com/stretchr/testify/require"
)
+2 -2
View File
@@ -6,10 +6,10 @@ import (
"net/http/httputil"
"strings"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/proto"
)
// routingRuleHandler handles proxy rule for actions related to request/response modification
@@ -6,7 +6,7 @@ import (
"runtime"
"testing"
"github.com/go-rod/rod/lib/launcher/flags"
"github.com/projectdiscovery/rod/lib/launcher/flags"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
+18
View File
@@ -148,6 +148,9 @@ type Request struct {
replayClient *http.Client // set when -replay-proxy is given
dialer *fastdialer.Dialer
bodySize *bodySizeEstimate
// sharesResponse lets other templates reuse a response this request gets,
// and this request reuse theirs (see httpclientpool.WithSharedResponse)
sharesResponse bool
// description: |
// SelfContained specifies if the request is self-contained.
@@ -587,9 +590,24 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
if shouldDisableKeepAliveForHTTPProxy(request, options) {
request.connConfiguration.Connection.DisableKeepAlive = true
}
request.sharesResponse = canShareResponse(request, options.Flow)
return nil
}
// canShareResponse reports whether a response another template got for the
// same request is as good as sending it again. A shared response arrives at
// once, so nothing that measures time may take one: race requests, timing
// analyzers, and a request whose operators or flow read the duration. The
// operators are searched whole, so a new field that can read it, as LLM
// matcher inputs do, is covered without being listed.
func canShareResponse(request *Request, flow string) bool {
if request.Race || request.Analyzer != nil || strings.Contains(flow, "duration") {
return false
}
operators, err := json.Marshal(request.Operators)
return err == nil && !bytes.Contains(operators, []byte("duration"))
}
// RebuildGenerator rebuilds the generator for the request
func (request *Request) RebuildGenerator() error {
generator, err := generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options)
@@ -514,7 +514,11 @@ func wrappedGet(options *types.Options, configuration *Configuration, host strin
}
}
return &connTrackingTransport{base: transport}, nil
tracking := &connTrackingTransport{base: transport}
if shared := getSharedResponses(options); shared != nil {
return &sharingTransport{base: tracking, shared: shared}, nil
}
return tracking, nil
}
redirectFlow := configuration.RedirectFlow
maxRedirects := configuration.MaxRedirects
@@ -175,7 +175,9 @@ func TestIdleConnPoolSizeIsWiredToTransport(t *testing.T) {
client, err := Get(opts, &Configuration{}, "example.com")
require.NoError(t, err)
tracking, ok := client.HTTPClient.Transport.(*connTrackingTransport)
sharing, ok := client.HTTPClient.Transport.(*sharingTransport)
require.True(t, ok, "expected the shared-response transport wrapper")
tracking, ok := sharing.base.(*connTrackingTransport)
require.True(t, ok, "expected the connection-tracking transport wrapper")
transport, ok := tracking.base.(*http.Transport)
require.True(t, ok, "expected an *http.Transport underneath")
@@ -0,0 +1,281 @@
package httpclientpool
import (
"bytes"
"container/list"
"context"
"io"
"maps"
"net/http"
"slices"
"strings"
"sync"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/useragent"
)
// Templates send many requests another template already sent to the same
// target, byte for byte: clustering merges only single-request templates. A
// request marked with WithSharedResponse is sent once per scan and every
// other template sending the same bytes gets a copy of the response. The key
// is the request as it goes on the wire, cookies included, and the client
// still processes each copy, so redirects and cookies behave as if it was
// sent again.
const (
// sharedResponseMaxBody keeps large downloads out of the store; they are
// sent each time
sharedResponseMaxBody = 1 << 20
sharedResponsesMaxBytes = 64 << 20
)
type sharedResponseKey struct{}
// WithSharedResponse marks the request of templateID as one whose response
// other templates may reuse. A template sending the same request again gets
// it sent again: a repeat within one template is intended.
func WithSharedResponse(ctx context.Context, templateID string) context.Context {
return context.WithValue(ctx, sharedResponseKey{}, templateID)
}
type sharedResponses struct {
mu sync.Mutex
entries map[string]*sharedEntry
// order holds stored entries oldest first, for eviction
order *list.List
bytes int
maxBytes int
}
type sharedEntry struct {
key string
templateID string
ready chan struct{}
response *storedResponse
}
type storedResponse struct {
response *http.Response
body []byte
}
func getSharedResponses(options *types.Options) *sharedResponses {
dialers := protocolstate.GetDialersWithId(options.ExecutionId)
if dialers == nil {
return nil
}
dialers.Lock()
defer dialers.Unlock()
if dialers.SharedResponses == nil {
dialers.SharedResponses = &sharedResponses{entries: make(map[string]*sharedEntry), order: list.New(), maxBytes: sharedResponsesMaxBytes}
}
shared, _ := dialers.SharedResponses.(*sharedResponses)
return shared
}
// ResetSharedResponses forgets the responses of earlier scans, so a scan
// that reuses the execution, as an SDK engine does, sends its requests again.
func ResetSharedResponses(options *types.Options) {
shared := getSharedResponses(options)
if shared == nil {
return
}
shared.mu.Lock()
defer shared.mu.Unlock()
clear(shared.entries)
shared.order.Init()
shared.bytes = 0
}
type sharingTransport struct {
base http.RoundTripper
shared *sharedResponses
}
func (t *sharingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
templateID, marked := req.Context().Value(sharedResponseKey{}).(string)
if !marked || (req.Method != http.MethodGet && req.Method != http.MethodHead) || hasBody(req) {
return t.base.RoundTrip(req)
}
key := sharedRequestKey(req)
t.shared.mu.Lock()
entry, found := t.shared.entries[key]
if !found {
entry = &sharedEntry{key: key, templateID: templateID, ready: make(chan struct{})}
t.shared.entries[key] = entry
t.shared.mu.Unlock()
return t.send(req, entry)
}
t.shared.mu.Unlock()
if entry.templateID == templateID {
return t.base.RoundTrip(req)
}
select {
case <-entry.ready:
case <-req.Context().Done():
return nil, req.Context().Err()
}
if entry.response == nil {
// the first send failed or was too large to keep
return t.base.RoundTrip(req)
}
return entry.response.copyFor(req), nil
}
func (t *sharingTransport) CloseIdleConnections() {
type closeIdler interface{ CloseIdleConnections() }
if ci, ok := t.base.(closeIdler); ok {
ci.CloseIdleConnections()
}
}
// send sends the first request of key and stores its response for the rest.
func (t *sharingTransport) send(req *http.Request, entry *sharedEntry) (*http.Response, error) {
defer close(entry.ready)
resp, err := t.base.RoundTrip(req)
if err != nil {
t.forget(entry)
return resp, err
}
body, err := io.ReadAll(io.LimitReader(resp.Body, sharedResponseMaxBody+1))
if err != nil || len(body) > sharedResponseMaxBody || !shareableStatus(resp.StatusCode) {
t.forget(entry)
// The bytes already taken off the body have to go back in front of it.
// A short read is still this caller's response: a Content-Length that
// is longer than the body is what some matchers look for. It is not
// stored, and neither is a body too large to keep or a refusal.
resp.Body = struct {
io.Reader
io.Closer
}{io.MultiReader(bytes.NewReader(body), resp.Body), resp.Body}
return resp, nil
}
_ = resp.Body.Close()
entry.response = &storedResponse{response: resp, body: body}
t.store(entry)
return entry.response.copyFor(req), nil
}
// shareableStatus reports whether code is an answer a later template can
// reuse. 404 and 500 are answers templates match on. 429 and the gateway
// errors mean the server did not answer, so the next template must send.
func shareableStatus(code int) bool {
switch code {
case http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
return false
default:
return true
}
}
// hasBody reports whether req sends a body. retryablehttp gives every
// request one, empty for a GET, so its content is what counts.
func hasBody(req *http.Request) bool {
if req.Body == nil || req.Body == http.NoBody {
return false
}
if req.ContentLength > 0 || req.GetBody == nil {
return true
}
body, err := req.GetBody()
if err != nil {
return true
}
defer func() { _ = body.Close() }()
var first [1]byte
n, _ := body.Read(first[:])
return n > 0
}
func (t *sharingTransport) forget(entry *sharedEntry) {
t.shared.mu.Lock()
defer t.shared.mu.Unlock()
if t.shared.entries[entry.key] == entry {
delete(t.shared.entries, entry.key)
}
}
func (t *sharingTransport) store(entry *sharedEntry) {
t.shared.mu.Lock()
defer t.shared.mu.Unlock()
// a reset while it was in flight dropped it
if t.shared.entries[entry.key] != entry {
return
}
t.shared.order.PushBack(entry)
t.shared.bytes += entry.size()
for t.shared.bytes > t.shared.maxBytes && t.shared.order.Len() > 1 {
oldest := t.shared.order.Remove(t.shared.order.Front()).(*sharedEntry)
t.shared.bytes -= oldest.size()
if t.shared.entries[oldest.key] == oldest {
delete(t.shared.entries, oldest.key)
}
}
}
// sharedEntryOverhead stands for the response struct and bookkeeping, so
// bodyless responses, as every HEAD gets, still count toward the cap.
const sharedEntryOverhead = 512
func (e *sharedEntry) size() int {
size := sharedEntryOverhead + len(e.key) + len(e.response.body)
for name, values := range e.response.response.Header {
size += len(name)
for _, value := range values {
size += len(value)
}
}
return size
}
func (s *storedResponse) copyFor(req *http.Request) *http.Response {
resp := *s.response
resp.Header = s.response.Header.Clone()
resp.Trailer = s.response.Trailer.Clone()
resp.Body = io.NopCloser(bytes.NewReader(s.body))
// A HEAD response has an empty body and a Content-Length for the body it
// did not send. Matchers read that length, so a reused HEAD keeps it.
if req.Method != http.MethodHead {
resp.ContentLength = int64(len(s.body))
}
resp.Request = req
return &resp
}
// randomUserAgents are the values nuclei picks a User-Agent from for every
// request a template does not set one for. Two templates sending the same
// request almost never pick the same one, and either could have picked the
// other's, so a picked User-Agent is left out of the key.
var randomUserAgents = sync.OnceValue(func() map[string]struct{} {
values := make(map[string]struct{}, len(useragent.UserAgents))
for _, userAgent := range useragent.UserAgents {
values[userAgent.Raw] = struct{}{}
}
return values
})
// sharedRequestKey is the request as the server receives it, apart from the
// order of headers and a picked User-Agent.
func sharedRequestKey(req *http.Request) string {
var key strings.Builder
key.WriteString(req.Method)
key.WriteByte(' ')
key.WriteString(req.URL.String())
key.WriteByte('\n')
key.WriteString(req.Host)
for _, name := range slices.Sorted(maps.Keys(req.Header)) {
values := req.Header[name]
if _, picked := randomUserAgents()[req.Header.Get("User-Agent")]; name == "User-Agent" && len(values) == 1 && picked {
continue
}
for _, value := range values {
key.WriteByte('\n')
key.WriteString(name)
key.WriteString(": ")
key.WriteString(value)
}
}
return key.String()
}
@@ -0,0 +1,308 @@
package httpclientpool
import (
"container/list"
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/useragent"
"github.com/stretchr/testify/require"
)
func newSharingTransport(base http.RoundTripper) *sharingTransport {
return &sharingTransport{base: base, shared: &sharedResponses{entries: make(map[string]*sharedEntry), order: list.New(), maxBytes: sharedResponsesMaxBytes}}
}
func sharedGet(t *testing.T, transport http.RoundTripper, method, url, templateID string, header http.Header) string {
t.Helper()
ctx := context.Background()
if templateID != "" {
ctx = WithSharedResponse(ctx, templateID)
}
req, err := http.NewRequestWithContext(ctx, method, url, nil)
require.NoError(t, err)
for name, values := range header {
req.Header[name] = values
}
resp, err := transport.RoundTrip(req)
require.NoError(t, err)
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
return string(body)
}
func TestSharedResponsesSendEachRequestOncePerScan(t *testing.T) {
var hits atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := hits.Add(1)
_, _ = io.WriteString(w, r.URL.Path+strings.Repeat("!", int(n)))
}))
defer server.Close()
transport := newSharingTransport(http.DefaultTransport)
first := sharedGet(t, transport, http.MethodGet, server.URL+"/a", "one", nil)
require.Equal(t, first, sharedGet(t, transport, http.MethodGet, server.URL+"/a", "two", nil), "another template gets the stored response")
require.EqualValues(t, 1, hits.Load())
// nuclei builds a GET with an empty body
empty, err := retryablehttp.NewRequestWithContext(WithSharedResponse(context.Background(), "four"), http.MethodGet, server.URL+"/a", strings.NewReader(""))
require.NoError(t, err)
resp, err := transport.RoundTrip(empty.Request)
require.NoError(t, err)
_ = resp.Body.Close()
require.EqualValues(t, 1, hits.Load(), "an empty body is no body")
sharedGet(t, transport, http.MethodGet, server.URL+"/a", "one", nil)
require.EqualValues(t, 2, hits.Load(), "a template repeating its own request sends it")
sharedGet(t, transport, http.MethodGet, server.URL+"/a", "", nil)
require.EqualValues(t, 3, hits.Load(), "an unmarked request is always sent")
sharedGet(t, transport, http.MethodGet, server.URL+"/a", "three", http.Header{"Cookie": {"session=1"}})
require.EqualValues(t, 4, hits.Load(), "other cookies make another request")
picked := func(i int) http.Header { return http.Header{"User-Agent": {useragent.UserAgents[i].Raw}} }
sharedGet(t, transport, http.MethodGet, server.URL+"/b", "one", picked(0))
sharedGet(t, transport, http.MethodGet, server.URL+"/b", "two", picked(1))
require.EqualValues(t, 5, hits.Load(), "a User-Agent nuclei picked at random is not part of the request")
sharedGet(t, transport, http.MethodGet, server.URL+"/b", "three", http.Header{"User-Agent": {"custom-scanner"}})
require.EqualValues(t, 6, hits.Load(), "one a template sets is")
for range 2 {
// A new request each time. Clone shares the body, so a second clone
// of one request can send ContentLength with an already-read body.
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), "one"), http.MethodPost, server.URL+"/a", strings.NewReader("x"))
require.NoError(t, err)
resp, err := transport.RoundTrip(req)
require.NoError(t, err)
_ = resp.Body.Close()
}
require.EqualValues(t, 8, hits.Load(), "a request with a body is always sent")
}
func TestSharedResponsesWaitForTheRequestInFlight(t *testing.T) {
var hits atomic.Int32
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
<-release
_, _ = io.WriteString(w, "slow")
}))
defer server.Close()
transport := newSharingTransport(http.DefaultTransport)
var wg sync.WaitGroup
bodies := make([]string, 5)
for i := range bodies {
wg.Add(1)
go func() {
defer wg.Done()
bodies[i] = sharedGet(t, transport, http.MethodGet, server.URL, "template-"+string(rune('a'+i)), nil)
}()
}
require.Eventually(t, func() bool { return hits.Load() > 0 }, 5*time.Second, time.Millisecond)
close(release)
wg.Wait()
require.EqualValues(t, 1, hits.Load())
require.Equal(t, []string{"slow", "slow", "slow", "slow", "slow"}, bodies)
}
func TestSharedResponsesSendLargeBodiesEachTime(t *testing.T) {
var hits atomic.Int32
large := strings.Repeat("x", sharedResponseMaxBody+10)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
_, _ = io.WriteString(w, large)
}))
defer server.Close()
transport := newSharingTransport(http.DefaultTransport)
require.Equal(t, large, sharedGet(t, transport, http.MethodGet, server.URL, "one", nil), "the whole body still arrives")
require.Equal(t, large, sharedGet(t, transport, http.MethodGet, server.URL, "two", nil))
require.EqualValues(t, 2, hits.Load())
}
type failOnce struct {
failed atomic.Bool
base http.RoundTripper
}
func (f *failOnce) RoundTrip(req *http.Request) (*http.Response, error) {
if f.failed.CompareAndSwap(false, true) {
return nil, errors.New("connection reset")
}
return f.base.RoundTrip(req)
}
func TestSharedResponsesDoNotKeepFailures(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.WriteString(w, "ok")
}))
defer server.Close()
transport := newSharingTransport(&failOnce{base: http.DefaultTransport})
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), "one"), http.MethodGet, server.URL, nil)
require.NoError(t, err)
_, err = transport.RoundTrip(req)
require.Error(t, err)
require.Equal(t, "ok", sharedGet(t, transport, http.MethodGet, server.URL, "two", nil), "the next template sends it again")
}
func TestSharedResponsesStartEmptyEachScan(t *testing.T) {
var hits atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
_, _ = io.WriteString(w, "ok")
}))
defer server.Close()
opts := newTestOptions(t, "test-shared-responses-reset")
transport := &sharingTransport{base: http.DefaultTransport, shared: getSharedResponses(opts)}
sharedGet(t, transport, http.MethodGet, server.URL, "one", nil)
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
require.EqualValues(t, 1, hits.Load())
// an SDK engine runs its next scan on the same execution
ResetSharedResponses(opts)
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
require.EqualValues(t, 2, hits.Load(), "a new scan does not see the responses of the last one")
}
func TestSharedResponsesCapCountsBodylessResponses(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
defer server.Close()
transport := newSharingTransport(http.DefaultTransport)
transport.shared.maxBytes = 20 * sharedEntryOverhead
for i := range 40 {
sharedGet(t, transport, http.MethodHead, fmt.Sprintf("%s/%d", server.URL, i), "one", nil)
}
require.LessOrEqual(t, transport.shared.bytes, transport.shared.maxBytes)
require.Less(t, len(transport.shared.entries), 20, "HEAD responses are evicted too")
}
func TestSharedResponsesKeepHeadContentLength(t *testing.T) {
var hits atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.Header().Set("Content-Length", "1234")
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
transport := newSharingTransport(http.DefaultTransport)
length := func(templateID string) int64 {
t.Helper()
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), templateID), http.MethodHead, server.URL, nil)
require.NoError(t, err)
resp, err := transport.RoundTrip(req)
require.NoError(t, err)
t.Cleanup(func() { _ = resp.Body.Close() })
return resp.ContentLength
}
require.Equal(t, int64(1234), length("one"))
require.Equal(t, int64(1234), length("two"), "a reused HEAD keeps the length the server advertised")
require.EqualValues(t, 1, hits.Load())
}
func TestSharedResponsesDoNotKeepRefusals(t *testing.T) {
for _, status := range []int{http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout} {
t.Run(http.StatusText(status), func(t *testing.T) {
var hits atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
http.Error(w, "no", status)
}))
defer server.Close()
transport := newSharingTransport(http.DefaultTransport)
sharedGet(t, transport, http.MethodGet, server.URL, "one", nil)
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
require.EqualValues(t, 2, hits.Load())
})
}
}
func TestSharedResponsesKeepNotFound(t *testing.T) {
var hits atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
http.NotFound(w, r)
}))
defer server.Close()
transport := newSharingTransport(http.DefaultTransport)
sharedGet(t, transport, http.MethodGet, server.URL, "one", nil)
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
require.EqualValues(t, 1, hits.Load())
}
// shortBody yields a prefix and then ErrUnexpectedEOF, which is what a
// response with a Content-Length longer than its body looks like.
type shortBody struct {
prefix []byte
sent bool
}
func (s *shortBody) Read(p []byte) (int, error) {
if !s.sent {
s.sent = true
n := copy(p, s.prefix)
return n, io.ErrUnexpectedEOF
}
return 0, io.ErrUnexpectedEOF
}
func (s *shortBody) Close() error { return nil }
type countingTransport struct {
hits atomic.Int32
}
func (c *countingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
c.hits.Add(1)
return &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{"Content-Length": {"50000"}},
ContentLength: 50000,
Body: &shortBody{prefix: []byte("fourteen bytes")},
Request: req,
}, nil
}
func TestSharedResponsesDoNotKeepShortReads(t *testing.T) {
base := &countingTransport{}
transport := newSharingTransport(base)
send := func(templateID string) *http.Response {
t.Helper()
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), templateID), http.MethodGet, "http://example.com", nil)
require.NoError(t, err)
resp, err := transport.RoundTrip(req)
require.NoError(t, err)
return resp
}
first := send("one")
body, err := io.ReadAll(first.Body)
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
require.Equal(t, "fourteen bytes", string(body))
require.Equal(t, int64(50000), first.ContentLength)
_ = first.Body.Close()
second := send("two")
_ = second.Body.Close()
require.EqualValues(t, 2, base.hits.Load(), "a short read is not reused")
}
+3
View File
@@ -1340,6 +1340,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
}
}
if request.sharesResponse {
generatedRequest.request.WithContext(httpclientpool.WithSharedResponse(generatedRequest.request.Context(), request.options.TemplateID))
}
resp, err = executingClient.Do(generatedRequest.request)
sent = true
+1 -1
View File
@@ -1400,7 +1400,7 @@ func TestExecuteParallelHTTP_GoroutineLeaks(t *testing.T) {
defer goleak.VerifyNone(t,
goleak.IgnoreAnyContainingPkg("go.opencensus.io/stats/view"),
goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"),
goleak.IgnoreAnyContainingPkg("github.com/go-rod/rod"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/rod"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/server"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/client"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/ratelimit"),
+20
View File
@@ -0,0 +1,20 @@
package http
import (
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
"github.com/stretchr/testify/require"
)
func TestCanShareResponseKeepsTimingChecksOnTheirOwnRequest(t *testing.T) {
require.True(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Words: []string{"admin"}}}}}, ""))
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{DSL: []string{"duration>=5"}}}}}, ""))
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Part: "duration"}}}}, ""))
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Extractors: []*extractors.Extractor{{DSL: []string{"duration_2"}}}}}, ""))
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Type: matchers.MatcherTypeHolder{MatcherType: matchers.LLMMatcher}, Inputs: []string{"{{duration_1}}"}}}}}, ""), "llm inputs read the event too")
require.False(t, canShareResponse(&Request{}, "http(1) && template.duration > 5"))
require.False(t, canShareResponse(&Request{Race: true}, ""))
}
+3
View File
@@ -19,6 +19,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
@@ -377,6 +378,8 @@ func TestFlowProgressEndsComplete(t *testing.T) {
for i, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
hits.Store(0)
// each case is its own scan, which the engine starts this way
httpclientpool.ResetSharedResponses(executerOpts.Options)
p := &countingProgress{}
opts := *executerOpts
opts.Progress = p
+4
View File
@@ -254,6 +254,9 @@ type Options struct {
// PlanFile is where the work a scan would do is written as JSON lines;
// nuclei exits after writing it, without sending requests.
PlanFile string
// RunPlanFile is a plan whose entries the scan runs, skipping those its
// done log lists as finished.
RunPlanFile string
// StoreEvents is where the scan writes the event of every request, as
// JSON lines, for -replay.
StoreEvents string
@@ -721,6 +724,7 @@ func (options *Options) Copy() *Options {
Silent: options.Silent,
Validate: options.Validate,
PlanFile: options.PlanFile,
RunPlanFile: options.RunPlanFile,
StoreEvents: options.StoreEvents,
ReplayFile: options.ReplayFile,
NoStrictSyntax: options.NoStrictSyntax,
+4 -1
View File
@@ -18,10 +18,13 @@ func TestCopyPreservesPayloadRandomOrder(t *testing.T) {
}
func TestCopyPreservesPlanFile(t *testing.T) {
options := &Options{PlanFile: "plan.jsonl"}
options := &Options{PlanFile: "plan.jsonl", RunPlanFile: "run.jsonl"}
if options.Copy().PlanFile != "plan.jsonl" {
t.Fatal("Copy must preserve PlanFile, or an embedded caller loses plan mode")
}
if options.Copy().RunPlanFile != "run.jsonl" {
t.Fatal("Copy must preserve RunPlanFile, or an embedded caller runs the whole scan")
}
}
func TestToJSONMapRejectsKeysSharingAStringForm(t *testing.T) {