mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2026-11-13 05:41:25 -10:00
Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8903942700 | ||
|
|
aec1420bdb | ||
|
|
b2ea1489fd | ||
|
|
e6ce143015 | ||
|
|
d3d84b2be8 | ||
|
|
67cda5044b | ||
|
|
a335de0be0 | ||
|
|
df5b8b367c | ||
|
|
f9192b96ae | ||
|
|
448ee69710 | ||
|
|
cb1f0757d4 | ||
|
|
894f90dd11 | ||
|
|
a351263c86 |
No files matched your search
@@ -167,6 +167,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -165,6 +165,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -165,6 +165,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -182,6 +182,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -165,6 +165,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -165,6 +165,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -165,6 +165,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -165,6 +165,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -165,6 +165,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -314,6 +314,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
|
||||
flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
|
||||
flagSet.StringVar(&options.PlanFile, "plan", "", "write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target"),
|
||||
flagSet.StringVar(&options.RunPlanFile, "run-plan", "", "run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes"),
|
||||
flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
|
||||
flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
|
||||
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all templates matching current filters"),
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module github.com/projectdiscovery/nuclei/v3
|
||||
|
||||
go 1.27.3
|
||||
go 1.27.4
|
||||
|
||||
require (
|
||||
carvel.dev/ytt v0.52.0
|
||||
@@ -46,7 +46,6 @@ require (
|
||||
github.com/go-pdf/fpdf v0.9.0
|
||||
github.com/go-pg/pg/v10 v10.15.0
|
||||
github.com/go-playground/validator/v10 v10.26.0
|
||||
github.com/go-rod/rod v0.116.2
|
||||
github.com/go-sql-driver/mysql v1.10.0
|
||||
github.com/gobwas/ws v1.4.0
|
||||
github.com/goccy/go-json v0.10.5
|
||||
@@ -85,7 +84,7 @@ require (
|
||||
github.com/projectdiscovery/gcache v0.0.0-20241015120333-12546c6e3f4c
|
||||
github.com/projectdiscovery/goflags v0.2.1
|
||||
github.com/projectdiscovery/goimpacket v0.1.1-0.20261007205806-6042a8213eeb
|
||||
github.com/projectdiscovery/goja v0.0.0-20261105204359-4285f3c977ed
|
||||
github.com/projectdiscovery/goja v0.0.0-20261012075649-9b78f2e05b9c
|
||||
github.com/projectdiscovery/goja_nodejs v0.0.0-20260618132410-8519f75f703d
|
||||
github.com/projectdiscovery/gologger v1.1.73
|
||||
github.com/projectdiscovery/gostruct v0.0.2
|
||||
@@ -102,6 +101,7 @@ require (
|
||||
github.com/projectdiscovery/rdap v0.9.0
|
||||
github.com/projectdiscovery/retryabledns v1.0.116
|
||||
github.com/projectdiscovery/retryablehttp-go v1.3.31
|
||||
github.com/projectdiscovery/rod v0.116.3-0.20261113150229-d775599ff53d
|
||||
github.com/projectdiscovery/sarif v0.1.0
|
||||
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75
|
||||
github.com/projectdiscovery/uncover v1.2.1
|
||||
@@ -237,7 +237,7 @@ require (
|
||||
github.com/dimchansky/utfbom v1.1.1 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/djherbis/times v1.6.0 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.11.0 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.2.1 // indirect
|
||||
github.com/docker/cli v29.2.0+incompatible // indirect
|
||||
github.com/docker/go-connections v0.7.0 // indirect
|
||||
github.com/dop251/goja v0.0.0-20260311135729-065cd970411c // indirect
|
||||
@@ -427,11 +427,7 @@ require (
|
||||
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
github.com/ysmood/fetchup v0.2.3 // indirect
|
||||
github.com/ysmood/goob v0.4.0 // indirect
|
||||
github.com/ysmood/got v0.40.0 // indirect
|
||||
github.com/ysmood/gson v0.7.3 // indirect
|
||||
github.com/ysmood/leakless v0.9.0 // indirect
|
||||
github.com/yuin/goldmark v1.7.17 // indirect
|
||||
github.com/yuin/goldmark-emoji v1.0.6 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
|
||||
@@ -358,8 +358,8 @@ github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
|
||||
github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
|
||||
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
|
||||
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/dlclark/regexp2/v2 v2.11.0 h1:z5uRPHav+eTVpoffDSuUP2KPS4f1ONZaPDK73S8HA0w=
|
||||
github.com/dlclark/regexp2/v2 v2.11.0/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/dlclark/regexp2/v2 v2.2.1 h1:mf4KkFUj0gJuarK8P+LgiS+Lit7m9N1yAwEfPbee7R0=
|
||||
github.com/dlclark/regexp2/v2 v2.2.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/docker/cli v29.2.0+incompatible h1:9oBd9+YM7rxjZLfyMGxjraKBKE4/nVyvVfN4qNl9XRM=
|
||||
github.com/docker/cli v29.2.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c=
|
||||
@@ -493,8 +493,6 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k=
|
||||
github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
|
||||
github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA=
|
||||
github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg=
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
|
||||
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
|
||||
@@ -945,8 +943,8 @@ github.com/projectdiscovery/goflags v0.2.1 h1:XnSYvsCjUwHbZaaJUZ3YznCJPoXRk/IhTF
|
||||
github.com/projectdiscovery/goflags v0.2.1/go.mod h1:7nAP1r2Dqgn/rwmOE3EWbZWUCEJKNIhVSBGpuzJAIns=
|
||||
github.com/projectdiscovery/goimpacket v0.1.1-0.20261007205806-6042a8213eeb h1:+L1yW2PblRHSSHQRTmvxLkGDPZnM52N62oeBd2xHKF0=
|
||||
github.com/projectdiscovery/goimpacket v0.1.1-0.20261007205806-6042a8213eeb/go.mod h1:bfgULaYf+6pjjcLCekpXIYElnLJbmhvTttxMAbQJvhY=
|
||||
github.com/projectdiscovery/goja v0.0.0-20261105204359-4285f3c977ed h1:h6aBYgIrjnapch77HWnpCPPz+bTU2OMfWOrr1fr0knA=
|
||||
github.com/projectdiscovery/goja v0.0.0-20261105204359-4285f3c977ed/go.mod h1:sjYPrVMMCZNujHasGmCZGs3Hx8DymbJA2plsJjlC3LA=
|
||||
github.com/projectdiscovery/goja v0.0.0-20261012075649-9b78f2e05b9c h1:4H/CfVNlue9PIwsCM1IGxOynDI7cir+emXuDvGJTLJs=
|
||||
github.com/projectdiscovery/goja v0.0.0-20261012075649-9b78f2e05b9c/go.mod h1:SO0AP+uKfYeeoR6jyVH/PKRYJE/f5gJrPMAM00iGEMc=
|
||||
github.com/projectdiscovery/goja_nodejs v0.0.0-20260618132410-8519f75f703d h1:fqqH9LHpN2WDz9QuxFrhKNxXSRtzk+Sa6jAhbB7tXcQ=
|
||||
github.com/projectdiscovery/goja_nodejs v0.0.0-20260618132410-8519f75f703d/go.mod h1:Ezmbgdaw4EunGGBU4MQViLoGMJc37LA3ip55YV3KeRI=
|
||||
github.com/projectdiscovery/gologger v1.1.73 h1:WX2nzq29cL7HdvOHqIttacoid/5lMwapyM7ukGU1jFY=
|
||||
@@ -985,6 +983,8 @@ github.com/projectdiscovery/retryabledns v1.0.116 h1:tK28PJ4tLO7bgM89RpIFjkfTPMS
|
||||
github.com/projectdiscovery/retryabledns v1.0.116/go.mod h1:r3vH0AKYtJ9fculcqRFrdVYe2ABcHwZeR4C6j3rqCbY=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.3.31 h1:0hSEZ5+jj6Oo1IL8qrytD36TK/qCxfe543jEaRqSTjs=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.3.31/go.mod h1:/WEa7YejxTCXEMLCGfGYaC3JsgXgzuqw8rv8cb4buEc=
|
||||
github.com/projectdiscovery/rod v0.116.3-0.20261113150229-d775599ff53d h1:6mwNBdlYWe0hu1/QLLhIrZiz+G1JYsU7OIVmCOaF+q0=
|
||||
github.com/projectdiscovery/rod v0.116.3-0.20261113150229-d775599ff53d/go.mod h1:FLVLMgXw6OFcOU5nd4Il8cdSWgDfAlDeRZMol3drmGs=
|
||||
github.com/projectdiscovery/sarif v0.1.0 h1:O541T+a448nSJsmIMnXXSOeDQEzpnCAYvRfe0eG5h74=
|
||||
github.com/projectdiscovery/sarif v0.1.0/go.mod h1:LBC+reM3bkI3qIIhE0rZaINaYX6VG+En6u2hHa5mA7E=
|
||||
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
|
||||
@@ -1212,20 +1212,14 @@ github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+Seva
|
||||
github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
|
||||
github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ=
|
||||
github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
|
||||
github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
|
||||
github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
|
||||
github.com/ysmood/gop v0.2.0 h1:+tFrG0TWPxT6p9ZaZs+VY+opCvHU8/3Fk6BaNv6kqKg=
|
||||
github.com/ysmood/gop v0.2.0/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
|
||||
github.com/ysmood/got v0.40.0 h1:ZQk1B55zIvS7zflRrkGfPDrPG3d7+JOza1ZkNxcc74Q=
|
||||
github.com/ysmood/got v0.40.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg=
|
||||
github.com/ysmood/gop v0.4.2 h1:KtW1/RugapWs1iW9ynQdk52BK6TWdP9zC0qDOhW8EDE=
|
||||
github.com/ysmood/gop v0.4.2/go.mod h1:K4LAB/BdLArknXrVSy+DyXcfb8P85JwAtwYM8zlb0mU=
|
||||
github.com/ysmood/got v0.43.0 h1:bcHlKg+qnhFz5JxNZ+j4fIfxxB8WWjTh6qdqeS8FfZw=
|
||||
github.com/ysmood/got v0.43.0/go.mod h1:rsfH2KJ5rcS/5n9pmybIGodoHlT/NTYxjgh3ob7UdSU=
|
||||
github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
|
||||
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
|
||||
github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
|
||||
github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
|
||||
github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU=
|
||||
github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
|
||||
@@ -558,18 +558,31 @@ func getBoolEnvValue(key string) bool {
|
||||
// validatePlanOptions rejects options that change the templates or targets
|
||||
// only once scanning starts, which a plan written beforehand cannot show.
|
||||
func validatePlanOptions(options *types.Options) error {
|
||||
if options.PlanFile == "" {
|
||||
if options.RunPlanFile != "" && options.PlanFile != "" {
|
||||
return errors.New("-plan cannot be used with -run-plan")
|
||||
}
|
||||
if options.RunPlanFile != "" && options.Resume != "" {
|
||||
return errors.New("-run-plan cannot be used with -resume, the plan's done log resumes it")
|
||||
}
|
||||
if options.PlanFile == "" && options.RunPlanFile == "" {
|
||||
return nil
|
||||
}
|
||||
// -automatic-scan builds its own engine, so a plan set on the main engine
|
||||
// would never be consulted. The other flags change the targets only after
|
||||
// a plan has been written.
|
||||
flag := "-plan"
|
||||
if options.PlanFile == "" {
|
||||
flag = "-run-plan"
|
||||
}
|
||||
switch {
|
||||
case options.AutomaticScan:
|
||||
return errors.New("-plan cannot be used with -automatic-scan, which picks templates while scanning")
|
||||
return fmt.Errorf("%s cannot be used with -automatic-scan, which picks templates while scanning", flag)
|
||||
case options.Uncover || len(options.UncoverQuery) > 0:
|
||||
return errors.New("-plan cannot be used with -uncover, which finds targets over the network")
|
||||
return fmt.Errorf("%s cannot be used with -uncover, which finds targets over the network", flag)
|
||||
case options.PreflightPortScan:
|
||||
return errors.New("-plan cannot be used with -preflight-portscan, which drops targets after scanning their ports")
|
||||
return fmt.Errorf("%s cannot be used with -preflight-portscan, which drops targets after scanning their ports", flag)
|
||||
case options.DASTServer || options.DASTProxy:
|
||||
return errors.New("-plan cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan")
|
||||
return fmt.Errorf("%s cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan", flag)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -23,4 +23,19 @@ func TestPlanRejectsOptionsThatChangeTargetsLater(t *testing.T) {
|
||||
require.NoError(t, validatePlanOptions(&types.Options{Uncover: true}), "only -plan restricts them")
|
||||
require.NoError(t, validatePlanOptions(&types.Options{UncoverQuery: goflags.StringSlice{"example"}}))
|
||||
require.NoError(t, validatePlanOptions(&types.Options{DASTServer: true}))
|
||||
|
||||
require.Error(t, validatePlanOptions(&types.Options{RunPlanFile: "p", PlanFile: "q"}))
|
||||
require.Error(t, validatePlanOptions(&types.Options{RunPlanFile: "p", Resume: "r"}))
|
||||
require.NoError(t, validatePlanOptions(&types.Options{RunPlanFile: "p"}))
|
||||
|
||||
for name, options := range map[string]*types.Options{
|
||||
"run automatic scan": {RunPlanFile: "p", AutomaticScan: true},
|
||||
"run uncover": {RunPlanFile: "p", Uncover: true},
|
||||
"run uncover query": {RunPlanFile: "p", UncoverQuery: goflags.StringSlice{"example"}},
|
||||
"run preflight": {RunPlanFile: "p", PreflightPortScan: true},
|
||||
"run dast server": {RunPlanFile: "p", DASTServer: true},
|
||||
"run dast proxy": {RunPlanFile: "p", DASTProxy: true},
|
||||
} {
|
||||
require.Error(t, validatePlanOptions(options), name)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
)
|
||||
|
||||
// planRun limits a scan to the entries of a plan file and appends the id of
|
||||
// each finished entry to the plan's done log, so running the plan again skips
|
||||
// them. Ids are derived from what an entry runs, not from its position, so the
|
||||
// log still applies after the plan is written again for changed targets or
|
||||
// templates.
|
||||
type planRun struct {
|
||||
mu sync.Mutex
|
||||
entries map[uint64]plannedEntry
|
||||
// done holds finished origin groups by the id of their first entry; an
|
||||
// entry outside a group is its own group
|
||||
done map[uint64]struct{}
|
||||
log *os.File
|
||||
logErr error
|
||||
// resumed counts the entries finished before this run
|
||||
resumed int
|
||||
}
|
||||
|
||||
type plannedEntry struct {
|
||||
group uint64
|
||||
seen bool
|
||||
}
|
||||
|
||||
func planDoneLog(planFile string) string {
|
||||
return planFile + ".done"
|
||||
}
|
||||
|
||||
func openPlanRun(planFile string) (*planRun, error) {
|
||||
run := &planRun{entries: make(map[uint64]plannedEntry), done: make(map[uint64]struct{})}
|
||||
if err := run.readPlan(planFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := run.readDoneLog(planDoneLog(planFile)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log, err := os.OpenFile(planDoneLog(planFile), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// an interrupted write leaves a partial line, which the next id must not
|
||||
// extend
|
||||
if info, err := log.Stat(); err == nil && info.Size() > 0 {
|
||||
last := make([]byte, 1)
|
||||
if _, err := log.ReadAt(last, info.Size()-1); err == nil && last[0] != '\n' {
|
||||
_, _ = log.WriteString("\n")
|
||||
}
|
||||
}
|
||||
run.log = log
|
||||
return run, nil
|
||||
}
|
||||
|
||||
func (run *planRun) readPlan(planFile string) error {
|
||||
file, err := os.Open(planFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
reader := bufio.NewReader(file)
|
||||
for number := 1; ; number++ {
|
||||
line, readErr := reader.ReadBytes('\n')
|
||||
if line = bytes.TrimSpace(line); len(line) > 0 {
|
||||
var entry struct {
|
||||
ID string `json:"id"`
|
||||
AlternativeTo string `json:"alternative-to"`
|
||||
}
|
||||
if err := json.Unmarshal(line, &entry); err != nil {
|
||||
return fmt.Errorf("%s:%d: %w", planFile, number, err)
|
||||
}
|
||||
// the summary line has no id
|
||||
if entry.ID != "" {
|
||||
id, ok := parsePlanID(entry.ID)
|
||||
if !ok {
|
||||
return fmt.Errorf("%s:%d: invalid id %q", planFile, number, entry.ID)
|
||||
}
|
||||
group := id
|
||||
if entry.AlternativeTo != "" {
|
||||
if group, ok = parsePlanID(entry.AlternativeTo); !ok {
|
||||
return fmt.Errorf("%s:%d: invalid alternative-to %q", planFile, number, entry.AlternativeTo)
|
||||
}
|
||||
}
|
||||
run.entries[id] = plannedEntry{group: group}
|
||||
}
|
||||
}
|
||||
if readErr == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (run *planRun) readDoneLog(path string) error {
|
||||
file, err := os.Open(path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
// a partial line from an interrupted write does not parse and is
|
||||
// run again
|
||||
id, ok := parsePlanID(scanner.Text())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if entry, planned := run.entries[id]; planned {
|
||||
run.done[entry.group] = struct{}{}
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, entry := range run.entries {
|
||||
if _, done := run.done[entry.group]; done {
|
||||
run.resumed++
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parsePlanID keeps ids as integers, which a plan of millions of entries
|
||||
// holds in a fraction of the memory of strings.
|
||||
func parsePlanID(id string) (uint64, bool) {
|
||||
if len(id) != 16 {
|
||||
return 0, false
|
||||
}
|
||||
value, err := strconv.ParseUint(id, 16, 64)
|
||||
return value, err == nil
|
||||
}
|
||||
|
||||
func (run *planRun) Skip(id string) bool {
|
||||
key, ok := parsePlanID(id)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
run.mu.Lock()
|
||||
defer run.mu.Unlock()
|
||||
entry, planned := run.entries[key]
|
||||
if !planned {
|
||||
return true
|
||||
}
|
||||
if !entry.seen {
|
||||
entry.seen = true
|
||||
run.entries[key] = entry
|
||||
}
|
||||
_, done := run.done[entry.group]
|
||||
return done
|
||||
}
|
||||
|
||||
func (run *planRun) Finished(id string) {
|
||||
key, ok := parsePlanID(id)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
run.mu.Lock()
|
||||
defer run.mu.Unlock()
|
||||
entry, planned := run.entries[key]
|
||||
if !planned {
|
||||
return
|
||||
}
|
||||
run.done[entry.group] = struct{}{}
|
||||
// written straight to the file: ctrl+c exits without running defers
|
||||
if _, err := run.log.WriteString(id + "\n"); err != nil && run.logErr == nil {
|
||||
run.logErr = err
|
||||
}
|
||||
}
|
||||
|
||||
// unreached counts unfinished entries the scan did not get to: those of a
|
||||
// host it found unresponsive, and those it does not have because it runs
|
||||
// with other templates, targets or options than the plan.
|
||||
func (run *planRun) unreached() int {
|
||||
run.mu.Lock()
|
||||
defer run.mu.Unlock()
|
||||
count := 0
|
||||
for _, entry := range run.entries {
|
||||
if _, done := run.done[entry.group]; !done && !entry.seen {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (run *planRun) close() error {
|
||||
if err := run.log.Close(); err != nil && run.logErr == nil {
|
||||
run.logErr = err
|
||||
}
|
||||
return run.logErr
|
||||
}
|
||||
|
||||
// finishPlanRun closes the done log and reports the entries left to run.
|
||||
func (r *Runner) finishPlanRun(run *planRun) {
|
||||
if err := run.close(); err != nil {
|
||||
r.Logger.Error().Msgf("Could not write %s: %s", planDoneLog(r.options.RunPlanFile), err)
|
||||
}
|
||||
if unreached := run.unreached(); unreached > 0 {
|
||||
r.Logger.Info().Msgf("%d plan entries did not run; running the plan again retries them. Entries run only when the scan has the templates, targets and options the plan was written with", unreached)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPlanRunResumesFromItsDoneLog(t *testing.T) {
|
||||
planFile := filepath.Join(t.TempDir(), "plan.jsonl")
|
||||
require.NoError(t, os.WriteFile(planFile, []byte(`{"id":"00000000000000a1","target":"https://a.example/x","templates":["root"]}
|
||||
{"id":"00000000000000a2","target":"https://a.example/y","templates":["root"],"alternative-to":"00000000000000a1"}
|
||||
{"id":"00000000000000b1","target":"https://b.example","templates":["generic"]}
|
||||
{"id":"00000000000000c1","target":"https://c.example","templates":["generic"]}
|
||||
{"summary":{"entries":3,"alternatives":1}}
|
||||
`), 0o644))
|
||||
// the last line was cut off by an interrupted write
|
||||
require.NoError(t, os.WriteFile(planDoneLog(planFile), []byte("00000000000000b1\n00000000"), 0o644))
|
||||
|
||||
run, err := openPlanRun(planFile)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, run.entries, 4)
|
||||
require.Equal(t, 1, run.resumed)
|
||||
|
||||
require.True(t, run.Skip("00000000000000b1"), "finished in the previous run")
|
||||
require.True(t, run.Skip("00000000000000ff"), "not in the plan")
|
||||
require.False(t, run.Skip("00000000000000a1"))
|
||||
run.Finished("00000000000000a1")
|
||||
require.True(t, run.Skip("00000000000000a2"), "its origin is covered")
|
||||
require.Equal(t, 1, run.unreached(), "c1 never ran")
|
||||
require.NoError(t, run.close())
|
||||
|
||||
log, err := os.ReadFile(planDoneLog(planFile))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "00000000000000b1\n00000000\n00000000000000a1\n", string(log), "the partial line is not extended")
|
||||
|
||||
again, err := openPlanRun(planFile)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 3, again.resumed, "b1, and a1 with its alternative")
|
||||
require.True(t, again.Skip("00000000000000a2"), "a finished origin stays covered")
|
||||
require.False(t, again.Skip("00000000000000c1"))
|
||||
require.NoError(t, again.close())
|
||||
}
|
||||
|
||||
func TestPlanRunRejectsAMalformedPlan(t *testing.T) {
|
||||
planFile := filepath.Join(t.TempDir(), "plan.jsonl")
|
||||
require.NoError(t, os.WriteFile(planFile, []byte("{\"id\":\"00000000000000a1\"}\nnot json\n"), 0o644))
|
||||
_, err := openPlanRun(planFile)
|
||||
require.ErrorContains(t, err, "plan.jsonl:2")
|
||||
}
|
||||
@@ -855,6 +855,15 @@ func (r *Runner) RunEnumeration() error {
|
||||
}
|
||||
}()
|
||||
}
|
||||
if r.options.RunPlanFile != "" {
|
||||
run, err := openPlanRun(r.options.RunPlanFile)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not read plan")
|
||||
}
|
||||
r.Logger.Info().Msgf("Running %d plan entries from %s, %d already finished", len(run.entries), r.options.RunPlanFile, run.resumed)
|
||||
executorEngine.SetPlanRun(run)
|
||||
defer r.finishPlanRun(run)
|
||||
}
|
||||
// TODO: remove below functions after v3 or update warning messages
|
||||
templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@ type Engine struct {
|
||||
executerOpts *protocols.ExecutorOptions
|
||||
Callback func(*output.ResultEvent) // Executed on results
|
||||
templateExecutionCallback TemplateExecutionCallback
|
||||
planRun PlanRun
|
||||
Logger *gologger.Logger
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
|
||||
@@ -35,6 +36,7 @@ func (e *Engine) ExecuteWithResults(ctx context.Context, templatesList []*templa
|
||||
func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider, noCluster bool) *atomic.Bool {
|
||||
results := &atomic.Bool{}
|
||||
selfcontainedWg := &sync.WaitGroup{}
|
||||
httpclientpool.ResetSharedResponses(e.options)
|
||||
|
||||
totalReqBeforeCluster := getRequestCount(templatesList) * int(target.Count())
|
||||
|
||||
|
||||
+17
-1
@@ -36,9 +36,13 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
|
||||
if sharedBudget {
|
||||
defer e.options.ReleaseTemplateThread()
|
||||
}
|
||||
defer e.tasksDone(1)
|
||||
planned, skip := e.claimPlanned(template, &contextargs.MetaInput{})
|
||||
if skip {
|
||||
return
|
||||
}
|
||||
finished := e.templateExecutionStarted(template, "")
|
||||
defer func() { finished(ctx.Err()) }()
|
||||
defer e.tasksDone(1)
|
||||
var err error
|
||||
var match bool
|
||||
ctx := scan.NewScanContext(ctx, contextargs.New(ctx))
|
||||
@@ -55,6 +59,8 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
|
||||
}
|
||||
if err != nil {
|
||||
e.options.Logger.Warning().Msgf("[%s] Could not execute step (self-contained): %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
|
||||
} else if ctx.Context().Err() == nil {
|
||||
planned()
|
||||
}
|
||||
results.CompareAndSwap(false, match)
|
||||
}(v, usesSharedTemplateBudget)
|
||||
@@ -330,6 +336,16 @@ func (e *Engine) tasksDone(count int64) {
|
||||
func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates.Template, value *contextargs.MetaInput) (matched bool, err error) {
|
||||
// a pair skipped as a repeat of its origin is done as well
|
||||
defer e.tasksDone(1)
|
||||
planned, skip := e.claimPlanned(template, value)
|
||||
if skip {
|
||||
return false, nil
|
||||
}
|
||||
// an origin a sibling already covered is settled too, so it is recorded
|
||||
defer func() {
|
||||
if err == nil && ctx.Err() == nil {
|
||||
planned()
|
||||
}
|
||||
}()
|
||||
// A template fixed to the target's origin makes the same requests for every
|
||||
// target that shares one, which a crawled list produces by the hundred.
|
||||
// Running it once per origin sends exactly the same traffic, minus the
|
||||
|
||||
+43
-2
@@ -104,7 +104,7 @@ func planEntry(template *templates.Template, value *contextargs.MetaInput, selec
|
||||
entry := PlanEntry{
|
||||
Target: value.Input,
|
||||
IP: value.CustomIP,
|
||||
Templates: []string{template.ID},
|
||||
Templates: planTemplates(template, selection),
|
||||
Protocol: template.Type().String(),
|
||||
Requests: template.TotalRequests,
|
||||
Dynamic: template.Flow != "" || template.IsFuzzableRequest(),
|
||||
@@ -126,7 +126,6 @@ func planEntry(template *templates.Template, value *contextargs.MetaInput, selec
|
||||
entry.Dynamic = true
|
||||
case isCluster(template):
|
||||
cluster := template.Executer.(*templates.ClusterExecuter)
|
||||
entry.Templates = slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
|
||||
// TotalRequests of a cluster counts only its http and dns requests,
|
||||
// so an ssl cluster would show none
|
||||
entry.Requests = cluster.Requests()
|
||||
@@ -143,6 +142,15 @@ func planEntry(template *templates.Template, value *contextargs.MetaInput, selec
|
||||
return entry
|
||||
}
|
||||
|
||||
// planTemplates lists the templates an entry runs, sorted so the entry id does
|
||||
// not depend on clustering order.
|
||||
func planTemplates(template *templates.Template, selection protocols.TemplateSelection) []string {
|
||||
if cluster, ok := template.Executer.(*templates.ClusterExecuter); ok {
|
||||
return slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
|
||||
}
|
||||
return []string{template.ID}
|
||||
}
|
||||
|
||||
func isCluster(template *templates.Template) bool {
|
||||
_, ok := template.Executer.(*templates.ClusterExecuter)
|
||||
return ok
|
||||
@@ -154,3 +162,36 @@ func planEntryID(entry PlanEntry) string {
|
||||
sum := sha256.Sum256([]byte(entry.Target + "\x00" + entry.IP + "\x00" + strings.Join(entry.Templates, "\x00")))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// PlanRun limits a scan to the entries of a plan and records the ones that
|
||||
// finish, so running the plan again resumes it.
|
||||
type PlanRun interface {
|
||||
// Skip reports whether the entry is not in the plan or already finished.
|
||||
Skip(id string) bool
|
||||
// Finished records an entry that ran without error.
|
||||
Finished(id string)
|
||||
}
|
||||
|
||||
// SetPlanRun makes the scan run only the entries run allows.
|
||||
func (e *Engine) SetPlanRun(run PlanRun) {
|
||||
e.planRun = run
|
||||
}
|
||||
|
||||
// claimPlanned reports whether the pair is skipped under a plan run, and
|
||||
// otherwise returns the func that records it finished. Without a plan run
|
||||
// every pair runs.
|
||||
func (e *Engine) claimPlanned(template *templates.Template, value *contextargs.MetaInput) (finished func(), skip bool) {
|
||||
if e.planRun == nil {
|
||||
return func() {}, false
|
||||
}
|
||||
// as in Plan, a self-contained template is not narrowed by any target
|
||||
var selection protocols.TemplateSelection
|
||||
if !template.SelfContained {
|
||||
selection = e.selectionFor(value)
|
||||
}
|
||||
id := planEntryID(PlanEntry{Target: value.Input, IP: value.CustomIP, Templates: planTemplates(template, selection)})
|
||||
if e.planRun.Skip(id) {
|
||||
return nil, true
|
||||
}
|
||||
return func() { e.planRun.Finished(id) }, false
|
||||
}
|
||||
@@ -154,3 +154,62 @@ func TestPlanListsOriginAlternatives(t *testing.T) {
|
||||
})
|
||||
require.Equal(t, plan, again)
|
||||
}
|
||||
|
||||
// fixedPlanRun is a plan run over fixed ids that records the finished ones.
|
||||
type fixedPlanRun struct {
|
||||
mu sync.Mutex
|
||||
allowed map[string]bool
|
||||
finished []string
|
||||
}
|
||||
|
||||
func (a *fixedPlanRun) Skip(id string) bool { return !a.allowed[id] }
|
||||
|
||||
func (a *fixedPlanRun) Finished(id string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.finished = append(a.finished, id)
|
||||
}
|
||||
|
||||
func TestPlanRunRunsOnlyThePlannedPairs(t *testing.T) {
|
||||
options := &types.Options{BulkSize: 2, TemplateThreads: 2, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
|
||||
engine := New(options)
|
||||
engine.SetExecuterOptions(&protocols.ExecutorOptions{
|
||||
Logger: engine.Logger,
|
||||
Options: options,
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
ProtocolType: tmpltypes.HTTPProtocol,
|
||||
Progress: &progressRecorder{},
|
||||
TargetScope: scopeByInput{"php": {paths: []string{"thinkphp.yaml"}}},
|
||||
})
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
runs []string
|
||||
)
|
||||
var templatesList []*templates.Template
|
||||
for _, path := range []string{"thinkphp.yaml", "generic.yaml"} {
|
||||
templatesList = append(templatesList, &templates.Template{
|
||||
ID: path,
|
||||
Path: path,
|
||||
TotalRequests: 1,
|
||||
Executer: &recordingExecuter{path: path, mu: &mu, runs: &runs},
|
||||
})
|
||||
}
|
||||
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "php"}, {Input: "plain"}}}
|
||||
|
||||
var plan []PlanEntry
|
||||
engine.Plan(templatesList, targets, true, func(entry PlanEntry) { plan = append(plan, entry) })
|
||||
require.Len(t, plan, 3)
|
||||
|
||||
// the first entry is left out of the plan, as if an agent removed it
|
||||
run := &fixedPlanRun{allowed: map[string]bool{plan[1].ID: true, plan[2].ID: true}}
|
||||
engine.SetPlanRun(run)
|
||||
engine.ExecuteScanWithOpts(context.Background(), templatesList, targets, true)
|
||||
|
||||
var want []string
|
||||
for _, entry := range plan[1:] {
|
||||
want = append(want, entry.Templates[0]+"@"+entry.Target)
|
||||
}
|
||||
require.ElementsMatch(t, want, runs, "only the planned pairs run")
|
||||
require.ElementsMatch(t, []string{plan[1].ID, plan[2].ID}, run.finished, "the scan derives the ids the plan wrote")
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package compiler
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// A Set that contains itself used to overflow the Go stack on export and kill the process.
|
||||
func TestExecuteSelfReferencingSetResult(t *testing.T) {
|
||||
result, err := executeScript(t, t.Name(), false, `var s = new Set(); s.add(s); s`)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result["response"])
|
||||
}
|
||||
|
||||
// Deeply nested source used to overflow the Go stack in the parser.
|
||||
func TestCompileDeepNesting(t *testing.T) {
|
||||
const n = 1000000
|
||||
_, err := SourceAutoMode(strings.Repeat("(", n)+"a"+strings.Repeat(")", n), false)
|
||||
require.ErrorContains(t, err, "Maximum nesting depth exceeded")
|
||||
}
|
||||
|
||||
// A context-first Go function called with fewer arguments used to panic in reflect.
|
||||
func TestContextFunctionMissingArguments(t *testing.T) {
|
||||
result, _ := executeScript(t, t.Name(), false, `require("nuclei/redis").GetServerInfo("127.0.0.1")`)
|
||||
errText, _ := result["error"].(string)
|
||||
require.NotContains(t, errText, "reflect")
|
||||
require.NotContains(t, errText, "panic")
|
||||
}
|
||||
@@ -16,6 +16,7 @@ type Dialers struct {
|
||||
HTTPClientPool *HTTPPool
|
||||
PerHostRateLimitPool any // *hostratelimit.PerHostRateLimitPool
|
||||
HTTPToHTTPSPortTracker any // *httpclientpool.HTTPToHTTPSPortTracker
|
||||
SharedResponses any // *httpclientpool.sharedResponses
|
||||
HTTPDesyncHosts *ExpiringSet
|
||||
NetworkPolicy *networkpolicy.NetworkPolicy
|
||||
LocalFileAccessAllowed bool
|
||||
|
||||
@@ -7,10 +7,10 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/projectdiscovery/networkpolicy"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
|
||||
@@ -15,7 +15,7 @@ func TestMemGuardianGoroutineLeak(t *testing.T) {
|
||||
defer goleak.VerifyNone(t,
|
||||
goleak.IgnoreAnyContainingPkg("go.opencensus.io/stats/view"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/go-rod/rod"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/rod"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/server"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/ratelimit"),
|
||||
// expirable LRU cache creates a background goroutine for TTL expiration that persists
|
||||
@@ -104,7 +104,7 @@ func TestMemGuardianReset(t *testing.T) {
|
||||
defer goleak.VerifyNone(t,
|
||||
goleak.IgnoreAnyContainingPkg("go.opencensus.io/stats/view"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/go-rod/rod"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/rod"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/server"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/ratelimit"),
|
||||
// expirable LRU cache creates a background goroutine for TTL expiration that persists
|
||||
|
||||
@@ -7,10 +7,10 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/go-rod/rod/lib/launcher/flags"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/launcher"
|
||||
"github.com/projectdiscovery/rod/lib/launcher/flags"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
|
||||
@@ -5,8 +5,8 @@ package engine
|
||||
import (
|
||||
"encoding/base64"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
)
|
||||
|
||||
// NewHijack create hijack from page.
|
||||
|
||||
@@ -5,11 +5,11 @@ import (
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/utils"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/utils"
|
||||
)
|
||||
|
||||
// Instance is an isolated browser instance opened for doing operations with it.
|
||||
|
||||
@@ -5,9 +5,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/cdp"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/cdp"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
||||
@@ -11,8 +11,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
@@ -21,6 +19,8 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
)
|
||||
|
||||
@@ -11,10 +11,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/input"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/go-rod/rod/lib/utils"
|
||||
"github.com/kitabisa/go-ci"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
@@ -24,6 +20,10 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
filepathutil "github.com/projectdiscovery/nuclei/v3/pkg/utils/filepath"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/input"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
"github.com/projectdiscovery/rod/lib/utils"
|
||||
contextutil "github.com/projectdiscovery/utils/context"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
|
||||
@@ -11,12 +11,12 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/cdp"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/cdp"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
||||
@@ -10,12 +10,12 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/projectdiscovery/networkpolicy"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/launcher"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
||||
@@ -6,10 +6,10 @@ import (
|
||||
"net/http/httputil"
|
||||
"strings"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
)
|
||||
|
||||
// routingRuleHandler handles proxy rule for actions related to request/response modification
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/go-rod/rod/lib/launcher/flags"
|
||||
"github.com/projectdiscovery/rod/lib/launcher/flags"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
|
||||
@@ -148,6 +148,9 @@ type Request struct {
|
||||
replayClient *http.Client // set when -replay-proxy is given
|
||||
dialer *fastdialer.Dialer
|
||||
bodySize *bodySizeEstimate
|
||||
// sharesResponse lets other templates reuse a response this request gets,
|
||||
// and this request reuse theirs (see httpclientpool.WithSharedResponse)
|
||||
sharesResponse bool
|
||||
|
||||
// description: |
|
||||
// SelfContained specifies if the request is self-contained.
|
||||
@@ -587,9 +590,24 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
|
||||
if shouldDisableKeepAliveForHTTPProxy(request, options) {
|
||||
request.connConfiguration.Connection.DisableKeepAlive = true
|
||||
}
|
||||
request.sharesResponse = canShareResponse(request, options.Flow)
|
||||
return nil
|
||||
}
|
||||
|
||||
// canShareResponse reports whether a response another template got for the
|
||||
// same request is as good as sending it again. A shared response arrives at
|
||||
// once, so nothing that measures time may take one: race requests, timing
|
||||
// analyzers, and a request whose operators or flow read the duration. The
|
||||
// operators are searched whole, so a new field that can read it, as LLM
|
||||
// matcher inputs do, is covered without being listed.
|
||||
func canShareResponse(request *Request, flow string) bool {
|
||||
if request.Race || request.Analyzer != nil || strings.Contains(flow, "duration") {
|
||||
return false
|
||||
}
|
||||
operators, err := json.Marshal(request.Operators)
|
||||
return err == nil && !bytes.Contains(operators, []byte("duration"))
|
||||
}
|
||||
|
||||
// RebuildGenerator rebuilds the generator for the request
|
||||
func (request *Request) RebuildGenerator() error {
|
||||
generator, err := generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options)
|
||||
|
||||
@@ -514,7 +514,11 @@ func wrappedGet(options *types.Options, configuration *Configuration, host strin
|
||||
}
|
||||
}
|
||||
|
||||
return &connTrackingTransport{base: transport}, nil
|
||||
tracking := &connTrackingTransport{base: transport}
|
||||
if shared := getSharedResponses(options); shared != nil {
|
||||
return &sharingTransport{base: tracking, shared: shared}, nil
|
||||
}
|
||||
return tracking, nil
|
||||
}
|
||||
redirectFlow := configuration.RedirectFlow
|
||||
maxRedirects := configuration.MaxRedirects
|
||||
|
||||
@@ -175,7 +175,9 @@ func TestIdleConnPoolSizeIsWiredToTransport(t *testing.T) {
|
||||
client, err := Get(opts, &Configuration{}, "example.com")
|
||||
require.NoError(t, err)
|
||||
|
||||
tracking, ok := client.HTTPClient.Transport.(*connTrackingTransport)
|
||||
sharing, ok := client.HTTPClient.Transport.(*sharingTransport)
|
||||
require.True(t, ok, "expected the shared-response transport wrapper")
|
||||
tracking, ok := sharing.base.(*connTrackingTransport)
|
||||
require.True(t, ok, "expected the connection-tracking transport wrapper")
|
||||
transport, ok := tracking.base.(*http.Transport)
|
||||
require.True(t, ok, "expected an *http.Transport underneath")
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
package httpclientpool
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"container/list"
|
||||
"context"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/useragent"
|
||||
)
|
||||
|
||||
// Templates send many requests another template already sent to the same
|
||||
// target, byte for byte: clustering merges only single-request templates. A
|
||||
// request marked with WithSharedResponse is sent once per scan and every
|
||||
// other template sending the same bytes gets a copy of the response. The key
|
||||
// is the request as it goes on the wire, cookies included, and the client
|
||||
// still processes each copy, so redirects and cookies behave as if it was
|
||||
// sent again.
|
||||
const (
|
||||
// sharedResponseMaxBody keeps large downloads out of the store; they are
|
||||
// sent each time
|
||||
sharedResponseMaxBody = 1 << 20
|
||||
sharedResponsesMaxBytes = 64 << 20
|
||||
)
|
||||
|
||||
type sharedResponseKey struct{}
|
||||
|
||||
// WithSharedResponse marks the request of templateID as one whose response
|
||||
// other templates may reuse. A template sending the same request again gets
|
||||
// it sent again: a repeat within one template is intended.
|
||||
func WithSharedResponse(ctx context.Context, templateID string) context.Context {
|
||||
return context.WithValue(ctx, sharedResponseKey{}, templateID)
|
||||
}
|
||||
|
||||
type sharedResponses struct {
|
||||
mu sync.Mutex
|
||||
entries map[string]*sharedEntry
|
||||
// order holds stored entries oldest first, for eviction
|
||||
order *list.List
|
||||
bytes int
|
||||
maxBytes int
|
||||
}
|
||||
|
||||
type sharedEntry struct {
|
||||
key string
|
||||
templateID string
|
||||
ready chan struct{}
|
||||
response *storedResponse
|
||||
}
|
||||
|
||||
type storedResponse struct {
|
||||
response *http.Response
|
||||
body []byte
|
||||
}
|
||||
|
||||
func getSharedResponses(options *types.Options) *sharedResponses {
|
||||
dialers := protocolstate.GetDialersWithId(options.ExecutionId)
|
||||
if dialers == nil {
|
||||
return nil
|
||||
}
|
||||
dialers.Lock()
|
||||
defer dialers.Unlock()
|
||||
if dialers.SharedResponses == nil {
|
||||
dialers.SharedResponses = &sharedResponses{entries: make(map[string]*sharedEntry), order: list.New(), maxBytes: sharedResponsesMaxBytes}
|
||||
}
|
||||
shared, _ := dialers.SharedResponses.(*sharedResponses)
|
||||
return shared
|
||||
}
|
||||
|
||||
// ResetSharedResponses forgets the responses of earlier scans, so a scan
|
||||
// that reuses the execution, as an SDK engine does, sends its requests again.
|
||||
func ResetSharedResponses(options *types.Options) {
|
||||
shared := getSharedResponses(options)
|
||||
if shared == nil {
|
||||
return
|
||||
}
|
||||
shared.mu.Lock()
|
||||
defer shared.mu.Unlock()
|
||||
clear(shared.entries)
|
||||
shared.order.Init()
|
||||
shared.bytes = 0
|
||||
}
|
||||
|
||||
type sharingTransport struct {
|
||||
base http.RoundTripper
|
||||
shared *sharedResponses
|
||||
}
|
||||
|
||||
func (t *sharingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
templateID, marked := req.Context().Value(sharedResponseKey{}).(string)
|
||||
if !marked || (req.Method != http.MethodGet && req.Method != http.MethodHead) || hasBody(req) {
|
||||
return t.base.RoundTrip(req)
|
||||
}
|
||||
key := sharedRequestKey(req)
|
||||
|
||||
t.shared.mu.Lock()
|
||||
entry, found := t.shared.entries[key]
|
||||
if !found {
|
||||
entry = &sharedEntry{key: key, templateID: templateID, ready: make(chan struct{})}
|
||||
t.shared.entries[key] = entry
|
||||
t.shared.mu.Unlock()
|
||||
return t.send(req, entry)
|
||||
}
|
||||
t.shared.mu.Unlock()
|
||||
if entry.templateID == templateID {
|
||||
return t.base.RoundTrip(req)
|
||||
}
|
||||
select {
|
||||
case <-entry.ready:
|
||||
case <-req.Context().Done():
|
||||
return nil, req.Context().Err()
|
||||
}
|
||||
if entry.response == nil {
|
||||
// the first send failed or was too large to keep
|
||||
return t.base.RoundTrip(req)
|
||||
}
|
||||
return entry.response.copyFor(req), nil
|
||||
}
|
||||
|
||||
func (t *sharingTransport) CloseIdleConnections() {
|
||||
type closeIdler interface{ CloseIdleConnections() }
|
||||
if ci, ok := t.base.(closeIdler); ok {
|
||||
ci.CloseIdleConnections()
|
||||
}
|
||||
}
|
||||
|
||||
// send sends the first request of key and stores its response for the rest.
|
||||
func (t *sharingTransport) send(req *http.Request, entry *sharedEntry) (*http.Response, error) {
|
||||
defer close(entry.ready)
|
||||
resp, err := t.base.RoundTrip(req)
|
||||
if err != nil {
|
||||
t.forget(entry)
|
||||
return resp, err
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, sharedResponseMaxBody+1))
|
||||
if err != nil || len(body) > sharedResponseMaxBody || !shareableStatus(resp.StatusCode) {
|
||||
t.forget(entry)
|
||||
// The bytes already taken off the body have to go back in front of it.
|
||||
// A short read is still this caller's response: a Content-Length that
|
||||
// is longer than the body is what some matchers look for. It is not
|
||||
// stored, and neither is a body too large to keep or a refusal.
|
||||
resp.Body = struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{io.MultiReader(bytes.NewReader(body), resp.Body), resp.Body}
|
||||
return resp, nil
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
entry.response = &storedResponse{response: resp, body: body}
|
||||
t.store(entry)
|
||||
return entry.response.copyFor(req), nil
|
||||
}
|
||||
|
||||
// shareableStatus reports whether code is an answer a later template can
|
||||
// reuse. 404 and 500 are answers templates match on. 429 and the gateway
|
||||
// errors mean the server did not answer, so the next template must send.
|
||||
func shareableStatus(code int) bool {
|
||||
switch code {
|
||||
case http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// hasBody reports whether req sends a body. retryablehttp gives every
|
||||
// request one, empty for a GET, so its content is what counts.
|
||||
func hasBody(req *http.Request) bool {
|
||||
if req.Body == nil || req.Body == http.NoBody {
|
||||
return false
|
||||
}
|
||||
if req.ContentLength > 0 || req.GetBody == nil {
|
||||
return true
|
||||
}
|
||||
body, err := req.GetBody()
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
defer func() { _ = body.Close() }()
|
||||
var first [1]byte
|
||||
n, _ := body.Read(first[:])
|
||||
return n > 0
|
||||
}
|
||||
|
||||
func (t *sharingTransport) forget(entry *sharedEntry) {
|
||||
t.shared.mu.Lock()
|
||||
defer t.shared.mu.Unlock()
|
||||
if t.shared.entries[entry.key] == entry {
|
||||
delete(t.shared.entries, entry.key)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *sharingTransport) store(entry *sharedEntry) {
|
||||
t.shared.mu.Lock()
|
||||
defer t.shared.mu.Unlock()
|
||||
// a reset while it was in flight dropped it
|
||||
if t.shared.entries[entry.key] != entry {
|
||||
return
|
||||
}
|
||||
t.shared.order.PushBack(entry)
|
||||
t.shared.bytes += entry.size()
|
||||
for t.shared.bytes > t.shared.maxBytes && t.shared.order.Len() > 1 {
|
||||
oldest := t.shared.order.Remove(t.shared.order.Front()).(*sharedEntry)
|
||||
t.shared.bytes -= oldest.size()
|
||||
if t.shared.entries[oldest.key] == oldest {
|
||||
delete(t.shared.entries, oldest.key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sharedEntryOverhead stands for the response struct and bookkeeping, so
|
||||
// bodyless responses, as every HEAD gets, still count toward the cap.
|
||||
const sharedEntryOverhead = 512
|
||||
|
||||
func (e *sharedEntry) size() int {
|
||||
size := sharedEntryOverhead + len(e.key) + len(e.response.body)
|
||||
for name, values := range e.response.response.Header {
|
||||
size += len(name)
|
||||
for _, value := range values {
|
||||
size += len(value)
|
||||
}
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
func (s *storedResponse) copyFor(req *http.Request) *http.Response {
|
||||
resp := *s.response
|
||||
resp.Header = s.response.Header.Clone()
|
||||
resp.Trailer = s.response.Trailer.Clone()
|
||||
resp.Body = io.NopCloser(bytes.NewReader(s.body))
|
||||
// A HEAD response has an empty body and a Content-Length for the body it
|
||||
// did not send. Matchers read that length, so a reused HEAD keeps it.
|
||||
if req.Method != http.MethodHead {
|
||||
resp.ContentLength = int64(len(s.body))
|
||||
}
|
||||
resp.Request = req
|
||||
return &resp
|
||||
}
|
||||
|
||||
// randomUserAgents are the values nuclei picks a User-Agent from for every
|
||||
// request a template does not set one for. Two templates sending the same
|
||||
// request almost never pick the same one, and either could have picked the
|
||||
// other's, so a picked User-Agent is left out of the key.
|
||||
var randomUserAgents = sync.OnceValue(func() map[string]struct{} {
|
||||
values := make(map[string]struct{}, len(useragent.UserAgents))
|
||||
for _, userAgent := range useragent.UserAgents {
|
||||
values[userAgent.Raw] = struct{}{}
|
||||
}
|
||||
return values
|
||||
})
|
||||
|
||||
// sharedRequestKey is the request as the server receives it, apart from the
|
||||
// order of headers and a picked User-Agent.
|
||||
func sharedRequestKey(req *http.Request) string {
|
||||
var key strings.Builder
|
||||
key.WriteString(req.Method)
|
||||
key.WriteByte(' ')
|
||||
key.WriteString(req.URL.String())
|
||||
key.WriteByte('\n')
|
||||
key.WriteString(req.Host)
|
||||
for _, name := range slices.Sorted(maps.Keys(req.Header)) {
|
||||
values := req.Header[name]
|
||||
if _, picked := randomUserAgents()[req.Header.Get("User-Agent")]; name == "User-Agent" && len(values) == 1 && picked {
|
||||
continue
|
||||
}
|
||||
for _, value := range values {
|
||||
key.WriteByte('\n')
|
||||
key.WriteString(name)
|
||||
key.WriteString(": ")
|
||||
key.WriteString(value)
|
||||
}
|
||||
}
|
||||
return key.String()
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
package httpclientpool
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/projectdiscovery/useragent"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newSharingTransport(base http.RoundTripper) *sharingTransport {
|
||||
return &sharingTransport{base: base, shared: &sharedResponses{entries: make(map[string]*sharedEntry), order: list.New(), maxBytes: sharedResponsesMaxBytes}}
|
||||
}
|
||||
|
||||
func sharedGet(t *testing.T, transport http.RoundTripper, method, url, templateID string, header http.Header) string {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
if templateID != "" {
|
||||
ctx = WithSharedResponse(ctx, templateID)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, nil)
|
||||
require.NoError(t, err)
|
||||
for name, values := range header {
|
||||
req.Header[name] = values
|
||||
}
|
||||
resp, err := transport.RoundTrip(req)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
return string(body)
|
||||
}
|
||||
|
||||
func TestSharedResponsesSendEachRequestOncePerScan(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
n := hits.Add(1)
|
||||
_, _ = io.WriteString(w, r.URL.Path+strings.Repeat("!", int(n)))
|
||||
}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(http.DefaultTransport)
|
||||
|
||||
first := sharedGet(t, transport, http.MethodGet, server.URL+"/a", "one", nil)
|
||||
require.Equal(t, first, sharedGet(t, transport, http.MethodGet, server.URL+"/a", "two", nil), "another template gets the stored response")
|
||||
require.EqualValues(t, 1, hits.Load())
|
||||
|
||||
// nuclei builds a GET with an empty body
|
||||
empty, err := retryablehttp.NewRequestWithContext(WithSharedResponse(context.Background(), "four"), http.MethodGet, server.URL+"/a", strings.NewReader(""))
|
||||
require.NoError(t, err)
|
||||
resp, err := transport.RoundTrip(empty.Request)
|
||||
require.NoError(t, err)
|
||||
_ = resp.Body.Close()
|
||||
require.EqualValues(t, 1, hits.Load(), "an empty body is no body")
|
||||
|
||||
sharedGet(t, transport, http.MethodGet, server.URL+"/a", "one", nil)
|
||||
require.EqualValues(t, 2, hits.Load(), "a template repeating its own request sends it")
|
||||
|
||||
sharedGet(t, transport, http.MethodGet, server.URL+"/a", "", nil)
|
||||
require.EqualValues(t, 3, hits.Load(), "an unmarked request is always sent")
|
||||
|
||||
sharedGet(t, transport, http.MethodGet, server.URL+"/a", "three", http.Header{"Cookie": {"session=1"}})
|
||||
require.EqualValues(t, 4, hits.Load(), "other cookies make another request")
|
||||
|
||||
picked := func(i int) http.Header { return http.Header{"User-Agent": {useragent.UserAgents[i].Raw}} }
|
||||
sharedGet(t, transport, http.MethodGet, server.URL+"/b", "one", picked(0))
|
||||
sharedGet(t, transport, http.MethodGet, server.URL+"/b", "two", picked(1))
|
||||
require.EqualValues(t, 5, hits.Load(), "a User-Agent nuclei picked at random is not part of the request")
|
||||
sharedGet(t, transport, http.MethodGet, server.URL+"/b", "three", http.Header{"User-Agent": {"custom-scanner"}})
|
||||
require.EqualValues(t, 6, hits.Load(), "one a template sets is")
|
||||
|
||||
for range 2 {
|
||||
// A new request each time. Clone shares the body, so a second clone
|
||||
// of one request can send ContentLength with an already-read body.
|
||||
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), "one"), http.MethodPost, server.URL+"/a", strings.NewReader("x"))
|
||||
require.NoError(t, err)
|
||||
resp, err := transport.RoundTrip(req)
|
||||
require.NoError(t, err)
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
require.EqualValues(t, 8, hits.Load(), "a request with a body is always sent")
|
||||
}
|
||||
|
||||
func TestSharedResponsesWaitForTheRequestInFlight(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
release := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
<-release
|
||||
_, _ = io.WriteString(w, "slow")
|
||||
}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(http.DefaultTransport)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
bodies := make([]string, 5)
|
||||
for i := range bodies {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
bodies[i] = sharedGet(t, transport, http.MethodGet, server.URL, "template-"+string(rune('a'+i)), nil)
|
||||
}()
|
||||
}
|
||||
require.Eventually(t, func() bool { return hits.Load() > 0 }, 5*time.Second, time.Millisecond)
|
||||
close(release)
|
||||
wg.Wait()
|
||||
require.EqualValues(t, 1, hits.Load())
|
||||
require.Equal(t, []string{"slow", "slow", "slow", "slow", "slow"}, bodies)
|
||||
}
|
||||
|
||||
func TestSharedResponsesSendLargeBodiesEachTime(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
large := strings.Repeat("x", sharedResponseMaxBody+10)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
_, _ = io.WriteString(w, large)
|
||||
}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(http.DefaultTransport)
|
||||
|
||||
require.Equal(t, large, sharedGet(t, transport, http.MethodGet, server.URL, "one", nil), "the whole body still arrives")
|
||||
require.Equal(t, large, sharedGet(t, transport, http.MethodGet, server.URL, "two", nil))
|
||||
require.EqualValues(t, 2, hits.Load())
|
||||
}
|
||||
|
||||
type failOnce struct {
|
||||
failed atomic.Bool
|
||||
base http.RoundTripper
|
||||
}
|
||||
|
||||
func (f *failOnce) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
if f.failed.CompareAndSwap(false, true) {
|
||||
return nil, errors.New("connection reset")
|
||||
}
|
||||
return f.base.RoundTrip(req)
|
||||
}
|
||||
|
||||
func TestSharedResponsesDoNotKeepFailures(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.WriteString(w, "ok")
|
||||
}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(&failOnce{base: http.DefaultTransport})
|
||||
|
||||
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), "one"), http.MethodGet, server.URL, nil)
|
||||
require.NoError(t, err)
|
||||
_, err = transport.RoundTrip(req)
|
||||
require.Error(t, err)
|
||||
require.Equal(t, "ok", sharedGet(t, transport, http.MethodGet, server.URL, "two", nil), "the next template sends it again")
|
||||
}
|
||||
|
||||
func TestSharedResponsesStartEmptyEachScan(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
_, _ = io.WriteString(w, "ok")
|
||||
}))
|
||||
defer server.Close()
|
||||
opts := newTestOptions(t, "test-shared-responses-reset")
|
||||
transport := &sharingTransport{base: http.DefaultTransport, shared: getSharedResponses(opts)}
|
||||
|
||||
sharedGet(t, transport, http.MethodGet, server.URL, "one", nil)
|
||||
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
|
||||
require.EqualValues(t, 1, hits.Load())
|
||||
|
||||
// an SDK engine runs its next scan on the same execution
|
||||
ResetSharedResponses(opts)
|
||||
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
|
||||
require.EqualValues(t, 2, hits.Load(), "a new scan does not see the responses of the last one")
|
||||
}
|
||||
|
||||
func TestSharedResponsesCapCountsBodylessResponses(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(http.DefaultTransport)
|
||||
transport.shared.maxBytes = 20 * sharedEntryOverhead
|
||||
|
||||
for i := range 40 {
|
||||
sharedGet(t, transport, http.MethodHead, fmt.Sprintf("%s/%d", server.URL, i), "one", nil)
|
||||
}
|
||||
require.LessOrEqual(t, transport.shared.bytes, transport.shared.maxBytes)
|
||||
require.Less(t, len(transport.shared.entries), 20, "HEAD responses are evicted too")
|
||||
}
|
||||
|
||||
func TestSharedResponsesKeepHeadContentLength(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.Header().Set("Content-Length", "1234")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(http.DefaultTransport)
|
||||
|
||||
length := func(templateID string) int64 {
|
||||
t.Helper()
|
||||
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), templateID), http.MethodHead, server.URL, nil)
|
||||
require.NoError(t, err)
|
||||
resp, err := transport.RoundTrip(req)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = resp.Body.Close() })
|
||||
return resp.ContentLength
|
||||
}
|
||||
|
||||
require.Equal(t, int64(1234), length("one"))
|
||||
require.Equal(t, int64(1234), length("two"), "a reused HEAD keeps the length the server advertised")
|
||||
require.EqualValues(t, 1, hits.Load())
|
||||
}
|
||||
|
||||
func TestSharedResponsesDoNotKeepRefusals(t *testing.T) {
|
||||
for _, status := range []int{http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout} {
|
||||
t.Run(http.StatusText(status), func(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
http.Error(w, "no", status)
|
||||
}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(http.DefaultTransport)
|
||||
|
||||
sharedGet(t, transport, http.MethodGet, server.URL, "one", nil)
|
||||
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
|
||||
require.EqualValues(t, 2, hits.Load())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedResponsesKeepNotFound(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(http.DefaultTransport)
|
||||
|
||||
sharedGet(t, transport, http.MethodGet, server.URL, "one", nil)
|
||||
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
|
||||
require.EqualValues(t, 1, hits.Load())
|
||||
}
|
||||
|
||||
// shortBody yields a prefix and then ErrUnexpectedEOF, which is what a
|
||||
// response with a Content-Length longer than its body looks like.
|
||||
type shortBody struct {
|
||||
prefix []byte
|
||||
sent bool
|
||||
}
|
||||
|
||||
func (s *shortBody) Read(p []byte) (int, error) {
|
||||
if !s.sent {
|
||||
s.sent = true
|
||||
n := copy(p, s.prefix)
|
||||
return n, io.ErrUnexpectedEOF
|
||||
}
|
||||
return 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
func (s *shortBody) Close() error { return nil }
|
||||
|
||||
type countingTransport struct {
|
||||
hits atomic.Int32
|
||||
}
|
||||
|
||||
func (c *countingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
c.hits.Add(1)
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: http.Header{"Content-Length": {"50000"}},
|
||||
ContentLength: 50000,
|
||||
Body: &shortBody{prefix: []byte("fourteen bytes")},
|
||||
Request: req,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func TestSharedResponsesDoNotKeepShortReads(t *testing.T) {
|
||||
base := &countingTransport{}
|
||||
transport := newSharingTransport(base)
|
||||
|
||||
send := func(templateID string) *http.Response {
|
||||
t.Helper()
|
||||
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), templateID), http.MethodGet, "http://example.com", nil)
|
||||
require.NoError(t, err)
|
||||
resp, err := transport.RoundTrip(req)
|
||||
require.NoError(t, err)
|
||||
return resp
|
||||
}
|
||||
|
||||
first := send("one")
|
||||
body, err := io.ReadAll(first.Body)
|
||||
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
require.Equal(t, "fourteen bytes", string(body))
|
||||
require.Equal(t, int64(50000), first.ContentLength)
|
||||
_ = first.Body.Close()
|
||||
|
||||
second := send("two")
|
||||
_ = second.Body.Close()
|
||||
require.EqualValues(t, 2, base.hits.Load(), "a short read is not reused")
|
||||
}
|
||||
@@ -1340,6 +1340,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
||||
}
|
||||
}
|
||||
|
||||
if request.sharesResponse {
|
||||
generatedRequest.request.WithContext(httpclientpool.WithSharedResponse(generatedRequest.request.Context(), request.options.TemplateID))
|
||||
}
|
||||
resp, err = executingClient.Do(generatedRequest.request)
|
||||
sent = true
|
||||
|
||||
|
||||
@@ -1400,7 +1400,7 @@ func TestExecuteParallelHTTP_GoroutineLeaks(t *testing.T) {
|
||||
defer goleak.VerifyNone(t,
|
||||
goleak.IgnoreAnyContainingPkg("go.opencensus.io/stats/view"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/go-rod/rod"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/rod"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/server"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/client"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/ratelimit"),
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCanShareResponseKeepsTimingChecksOnTheirOwnRequest(t *testing.T) {
|
||||
require.True(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Words: []string{"admin"}}}}}, ""))
|
||||
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{DSL: []string{"duration>=5"}}}}}, ""))
|
||||
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Part: "duration"}}}}, ""))
|
||||
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Extractors: []*extractors.Extractor{{DSL: []string{"duration_2"}}}}}, ""))
|
||||
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Type: matchers.MatcherTypeHolder{MatcherType: matchers.LLMMatcher}, Inputs: []string{"{{duration_1}}"}}}}}, ""), "llm inputs read the event too")
|
||||
require.False(t, canShareResponse(&Request{}, "http(1) && template.duration > 5"))
|
||||
require.False(t, canShareResponse(&Request{Race: true}, ""))
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
@@ -377,6 +378,8 @@ func TestFlowProgressEndsComplete(t *testing.T) {
|
||||
for i, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
hits.Store(0)
|
||||
// each case is its own scan, which the engine starts this way
|
||||
httpclientpool.ResetSharedResponses(executerOpts.Options)
|
||||
p := &countingProgress{}
|
||||
opts := *executerOpts
|
||||
opts.Progress = p
|
||||
|
||||
@@ -254,6 +254,9 @@ type Options struct {
|
||||
// PlanFile is where the work a scan would do is written as JSON lines;
|
||||
// nuclei exits after writing it, without sending requests.
|
||||
PlanFile string
|
||||
// RunPlanFile is a plan whose entries the scan runs, skipping those its
|
||||
// done log lists as finished.
|
||||
RunPlanFile string
|
||||
// StoreEvents is where the scan writes the event of every request, as
|
||||
// JSON lines, for -replay.
|
||||
StoreEvents string
|
||||
@@ -721,6 +724,7 @@ func (options *Options) Copy() *Options {
|
||||
Silent: options.Silent,
|
||||
Validate: options.Validate,
|
||||
PlanFile: options.PlanFile,
|
||||
RunPlanFile: options.RunPlanFile,
|
||||
StoreEvents: options.StoreEvents,
|
||||
ReplayFile: options.ReplayFile,
|
||||
NoStrictSyntax: options.NoStrictSyntax,
|
||||
|
||||
@@ -18,10 +18,13 @@ func TestCopyPreservesPayloadRandomOrder(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCopyPreservesPlanFile(t *testing.T) {
|
||||
options := &Options{PlanFile: "plan.jsonl"}
|
||||
options := &Options{PlanFile: "plan.jsonl", RunPlanFile: "run.jsonl"}
|
||||
if options.Copy().PlanFile != "plan.jsonl" {
|
||||
t.Fatal("Copy must preserve PlanFile, or an embedded caller loses plan mode")
|
||||
}
|
||||
if options.Copy().RunPlanFile != "run.jsonl" {
|
||||
t.Fatal("Copy must preserve RunPlanFile, or an embedded caller runs the whole scan")
|
||||
}
|
||||
}
|
||||
|
||||
func TestToJSONMapRejectsKeysSharingAStringForm(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user