mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2026-11-13 05:41:25 -10:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
55bd0bbfb6 | ||
|
|
8f1c88e1f3 | ||
|
|
0305d5468c |
No files matched your search
@@ -17,4 +17,4 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: crate-ci/[email protected]2.0
|
||||
- uses: crate-ci/[email protected]1.1
|
||||
@@ -166,8 +166,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -207,7 +205,6 @@ OUTPUT:
|
||||
-o, -output string output file to write found issues/vulnerabilities
|
||||
-sresp, -store-resp store all request/response passed through nuclei to output directory
|
||||
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
|
||||
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
|
||||
-silent display findings only
|
||||
-nc, -no-color disable output content coloring (ANSI escape codes)
|
||||
-j, -jsonl write output in JSONL(ines) format
|
||||
@@ -222,7 +219,6 @@ OUTPUT:
|
||||
-se, -sarif-export string file to export results in SARIF format
|
||||
-je, -json-export string file to export results in JSON format
|
||||
-jle, -jsonl-export string file to export results in JSONL(ine) format
|
||||
-ce, -csv-export string file to export results in CSV format
|
||||
-pe, -pdf-export string file to export results in PDF format
|
||||
-rd, -redact string[] redact given list of keys from query parameter, request header and body
|
||||
-nrd, -no-redact disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted
|
||||
@@ -242,7 +238,6 @@ CONFIGURATIONS:
|
||||
-sr, -system-resolvers use system DNS resolving as error fallback
|
||||
-dc, -disable-clustering disable clustering of requests
|
||||
-passive enable passive HTTP response processing mode
|
||||
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
|
||||
-fh2, -force-http2 force http2 connection on requests
|
||||
-ev, -env-vars enable environment variables to be used in template
|
||||
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
|
||||
@@ -261,7 +256,6 @@ CONFIGURATIONS:
|
||||
-rsr, -response-size-read int max response size to read in bytes
|
||||
-rss, -response-size-save int max response size to read in bytes (default 1048576)
|
||||
-rqss, -request-size-save int max request body size to save in bytes (0 to save the whole body) (default 32768)
|
||||
-mrbi, -max-request-body-inflight int max request body bytes held by in-flight http requests; requests with bodies of 1MB or more queue past it (0 disables)
|
||||
-reset reset removes all nuclei configuration and data files (including nuclei-templates)
|
||||
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
|
||||
-hae, -http-api-endpoint string experimental http api endpoint
|
||||
|
||||
@@ -164,8 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -205,7 +203,6 @@ OUTPUT:
|
||||
-o, -output string output file to write found issues/vulnerabilities
|
||||
-sresp, -store-resp store all request/response passed through nuclei to output directory
|
||||
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
|
||||
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
|
||||
-silent display findings only
|
||||
-nc, -no-color disable output content coloring (ANSI escape codes)
|
||||
-j, -jsonl write output in JSONL(ines) format
|
||||
@@ -239,7 +236,6 @@ CONFIGURATIONS:
|
||||
-sr, -system-resolvers use system DNS resolving as error fallback
|
||||
-dc, -disable-clustering disable clustering of requests
|
||||
-passive enable passive HTTP response processing mode
|
||||
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
|
||||
-fh2, -force-http2 force http2 connection on requests
|
||||
-ev, -env-vars enable environment variables to be used in template
|
||||
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
|
||||
|
||||
@@ -164,8 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -205,7 +203,6 @@ OUTPUT:
|
||||
-o, -output string output file to write found issues/vulnerabilities
|
||||
-sresp, -store-resp store all request/response passed through nuclei to output directory
|
||||
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
|
||||
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
|
||||
-silent display findings only
|
||||
-nc, -no-color disable output content coloring (ANSI escape codes)
|
||||
-j, -jsonl write output in JSONL(ines) format
|
||||
@@ -239,7 +236,6 @@ CONFIGURATIONS:
|
||||
-sr, -system-resolvers use system DNS resolving as error fallback
|
||||
-dc, -disable-clustering disable clustering of requests
|
||||
-passive enable passive HTTP response processing mode
|
||||
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
|
||||
-fh2, -force-http2 force http2 connection on requests
|
||||
-ev, -env-vars enable environment variables to be used in template
|
||||
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
|
||||
|
||||
@@ -181,8 +181,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -222,7 +220,6 @@ OUTPUT:
|
||||
-o, -output string output file to write found issues/vulnerabilities
|
||||
-sresp, -store-resp store all request/response passed through nuclei to output directory
|
||||
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
|
||||
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
|
||||
-silent display findings only
|
||||
-nc, -no-color disable output content coloring (ANSI escape codes)
|
||||
-j, -jsonl write output in JSONL(ines) format
|
||||
@@ -256,7 +253,6 @@ CONFIGURATIONS:
|
||||
-sr, -system-resolvers use system DNS resolving as error fallback
|
||||
-dc, -disable-clustering disable clustering of requests
|
||||
-passive enable passive HTTP response processing mode
|
||||
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
|
||||
-fh2, -force-http2 force http2 connection on requests
|
||||
-ev, -env-vars enable environment variables to be used in template
|
||||
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
|
||||
|
||||
@@ -164,8 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -205,7 +203,6 @@ OUTPUT:
|
||||
-o, -output string output file to write found issues/vulnerabilities
|
||||
-sresp, -store-resp store all request/response passed through nuclei to output directory
|
||||
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
|
||||
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
|
||||
-silent display findings only
|
||||
-nc, -no-color disable output content coloring (ANSI escape codes)
|
||||
-j, -jsonl write output in JSONL(ines) format
|
||||
@@ -239,7 +236,6 @@ CONFIGURATIONS:
|
||||
-sr, -system-resolvers use system DNS resolving as error fallback
|
||||
-dc, -disable-clustering disable clustering of requests
|
||||
-passive enable passive HTTP response processing mode
|
||||
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
|
||||
-fh2, -force-http2 force http2 connection on requests
|
||||
-ev, -env-vars enable environment variables to be used in template
|
||||
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
|
||||
|
||||
@@ -164,8 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -205,7 +203,6 @@ OUTPUT:
|
||||
-o, -output string output file to write found issues/vulnerabilities
|
||||
-sresp, -store-resp store all request/response passed through nuclei to output directory
|
||||
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
|
||||
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
|
||||
-silent display findings only
|
||||
-nc, -no-color disable output content coloring (ANSI escape codes)
|
||||
-j, -jsonl write output in JSONL(ines) format
|
||||
@@ -239,7 +236,6 @@ CONFIGURATIONS:
|
||||
-sr, -system-resolvers use system DNS resolving as error fallback
|
||||
-dc, -disable-clustering disable clustering of requests
|
||||
-passive enable passive HTTP response processing mode
|
||||
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
|
||||
-fh2, -force-http2 force http2 connection on requests
|
||||
-ev, -env-vars enable environment variables to be used in template
|
||||
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
|
||||
|
||||
@@ -164,8 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -205,7 +203,6 @@ OUTPUT:
|
||||
-o, -output string output file to write found issues/vulnerabilities
|
||||
-sresp, -store-resp store all request/response passed through nuclei to output directory
|
||||
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
|
||||
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
|
||||
-silent display findings only
|
||||
-nc, -no-color disable output content coloring (ANSI escape codes)
|
||||
-j, -jsonl write output in JSONL(ines) format
|
||||
@@ -239,7 +236,6 @@ CONFIGURATIONS:
|
||||
-sr, -system-resolvers use system DNS resolving as error fallback
|
||||
-dc, -disable-clustering disable clustering of requests
|
||||
-passive enable passive HTTP response processing mode
|
||||
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
|
||||
-fh2, -force-http2 force http2 connection on requests
|
||||
-ev, -env-vars enable environment variables to be used in template
|
||||
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
|
||||
|
||||
@@ -164,8 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -205,7 +203,6 @@ OUTPUT:
|
||||
-o, -output string output file to write found issues/vulnerabilities
|
||||
-sresp, -store-resp store all request/response passed through nuclei to output directory
|
||||
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
|
||||
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
|
||||
-silent display findings only
|
||||
-nc, -no-color disable output content coloring (ANSI escape codes)
|
||||
-j, -jsonl write output in JSONL(ines) format
|
||||
@@ -239,7 +236,6 @@ CONFIGURATIONS:
|
||||
-sr, -system-resolvers use system DNS resolving as error fallback
|
||||
-dc, -disable-clustering disable clustering of requests
|
||||
-passive enable passive HTTP response processing mode
|
||||
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
|
||||
-fh2, -force-http2 force http2 connection on requests
|
||||
-ev, -env-vars enable environment variables to be used in template
|
||||
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
|
||||
|
||||
@@ -164,8 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -205,7 +203,6 @@ OUTPUT:
|
||||
-o, -output string output file to write found issues/vulnerabilities
|
||||
-sresp, -store-resp store all request/response passed through nuclei to output directory
|
||||
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
|
||||
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
|
||||
-silent display findings only
|
||||
-nc, -no-color disable output content coloring (ANSI escape codes)
|
||||
-j, -jsonl write output in JSONL(ines) format
|
||||
@@ -239,7 +236,6 @@ CONFIGURATIONS:
|
||||
-sr, -system-resolvers use system DNS resolving as error fallback
|
||||
-dc, -disable-clustering disable clustering of requests
|
||||
-passive enable passive HTTP response processing mode
|
||||
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
|
||||
-fh2, -force-http2 force http2 connection on requests
|
||||
-ev, -env-vars enable environment variables to be used in template
|
||||
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
|
||||
|
||||
+13
-21
@@ -1,13 +1,13 @@
|
||||
<!-- Generated by securitycontext.dev. Source HEAD: 40d08ee493f6f84c9a6b3c7f15f22da83853130c. -->
|
||||
<!-- Generated by securitycontext.dev. Source HEAD: df57ae6d2955864fd92a1bd2565fe9e4e9a7592b. -->
|
||||
# Security Context: projectdiscovery/nuclei @ dev
|
||||
|
||||
> Reference context for editing projectdiscovery/nuclei. Before changing any component below, **preserve the listed guards and do not reintroduce these fixed issues**. Each fingerprint is a regression check, and each shared attack surface is a guard that must keep holding on every path.
|
||||
|
||||
_Analyzed [40d08ee493][c-40d08ee493] on `dev` · 5242 commits, 586 security-relevant examined · generated 2026-11-09_
|
||||
_Analyzed [df57ae6d29][c-df57ae6d29] on `dev` · 5174 commits, 548 security-relevant examined · generated 2026-11-02_
|
||||
|
||||
## Recurring Security Patterns
|
||||
|
||||
**12.5% of classified fixes repeat a previously fixed pattern** (2 recurring, 14 first-seen). A recurring pattern means this repository fixed a similar issue before — not that the code is currently vulnerable.
|
||||
**7.1% of classified fixes repeat a previously fixed pattern** (1 recurring, 13 first-seen). A recurring pattern means this repository fixed a similar issue before — not that the code is currently vulnerable.
|
||||
|
||||
## Top Risks
|
||||
|
||||
@@ -15,12 +15,12 @@ Components with the strongest concentration of past fixes for one class.
|
||||
|
||||
| # | Class | Component | Severity | Fixes | Evidence |
|
||||
|---|---|---|---|---|---|
|
||||
| 1 | Code Execution | `pkg/fuzz/execute.go` | high | 2 | [4fed40f75a][c-4fed40f75a], [ccbfb12bd0][c-ccbfb12bd0] |
|
||||
| 2 | SSRF | `pkg/protocols/common/protocolstate/headless.go` | high | 2 | [d43c1f547f][c-d43c1f547f], [d9dabbb4a1][c-d9dabbb4a1] |
|
||||
| 3 | Arbitrary File Access | `v2/pkg/protocols/common/generators/load.go` | high | 0 (1 component gone, not counted) | [e5154d362a][c-e5154d362a] |
|
||||
| 4 | Arbitrary File Access | `pkg/js/libs/mysql/mysql.go` | high | 1 | [634cecbb28][c-634cecbb28] |
|
||||
| 5 | Certificate Validation Bypass | `pkg/reporting/exporters/es/elasticsearch.go` | high | 1 | [82d5a8acc2][c-82d5a8acc2] |
|
||||
| 6 | Code Execution | `pkg/protocols/common/expressions/expressions.go` | high | 1 | [d221732016][c-d221732016] |
|
||||
| 1 | SSRF | `pkg/protocols/common/protocolstate/headless.go` | high | 2 | [d43c1f547f][c-d43c1f547f], [d9dabbb4a1][c-d9dabbb4a1] |
|
||||
| 2 | Arbitrary File Access | `v2/pkg/protocols/common/generators/load.go` | high | 0 (1 component gone, not counted) | [e5154d362a][c-e5154d362a] |
|
||||
| 3 | Arbitrary File Access | `pkg/js/libs/mysql/mysql.go` | high | 1 | [634cecbb28][c-634cecbb28] |
|
||||
| 4 | Code Execution | `pkg/fuzz/execute.go` | high | 1 | [ccbfb12bd0][c-ccbfb12bd0] |
|
||||
| 5 | Code Execution | `pkg/protocols/common/expressions/expressions.go` | high | 1 | [d221732016][c-d221732016] |
|
||||
| 6 | Information Disclosure | `internal/httpapi/apiendpoint.go` | high | 1 | [a20aa26024][c-a20aa26024] |
|
||||
|
||||
## Fixed-Issue Fingerprints
|
||||
|
||||
@@ -28,13 +28,8 @@ Real security fixes mined from git history; use them as regression checks. `veri
|
||||
|
||||
**By component** (directories with the most fixes):
|
||||
|
||||
- `pkg/fuzz` — 2 fixes: Code Execution (2)
|
||||
- `pkg/protocols/common/protocolstate` — 2 fixes: SSRF (2)
|
||||
|
||||
- [4fed40f75a][c-4fed40f75a] **fix(fuzz): stop re-rendering evaluated template variables (#8057)** — Code Execution, high [CWE-94] · inferred · 2026-11-05
|
||||
`pkg/fuzz/execute.go` · sink `rule.evaluateVarsWithInteractsh` · fix: safe parser configuration
|
||||
- [82d5a8acc2][c-82d5a8acc2] **exporters: verify TLS certificates (#8042)** — Certificate Validation Bypass, high [CWE-295] · inferred · 2026-11-04
|
||||
`pkg/reporting/exporters/es/elasticsearch.go` · sink `DialTLS` · fix: cryptographic verification
|
||||
- [1c0f77d28e][c-1c0f77d28e] **fix(js/net): enforce network policy on proxied target in JS net dial** — SSRF, high [CWE-918] · inferred · 2026-10-18
|
||||
`pkg/js/libs/net/net.go` · sink `dialHTTPProxy` · fix: input validation
|
||||
- [d43c1f547f][c-d43c1f547f] **fix(headless): canonicalize IPv4-mapped IPv6 literals and re-validate dial targets** — SSRF, high [CWE-918] · inferred · 2026-10-13
|
||||
@@ -68,12 +63,11 @@ Real security fixes mined from git history; use them as regression checks. `veri
|
||||
|
||||
| Class | Fixes | Verified | Most recent fix |
|
||||
|---|---|---|---|
|
||||
| Code Execution | 3 | 2 | [2026-11-05][c-4fed40f75a] 4fed40f75a |
|
||||
| SSRF | 3 | 0 | [2026-10-18][c-1c0f77d28e] 1c0f77d28e |
|
||||
| Arbitrary File Access | 1 (1 component gone, not counted) | 2 | [2026-06-19][c-634cecbb28] 634cecbb28 |
|
||||
| Code Execution | 2 | 2 | [2026-06-30][c-ccbfb12bd0] ccbfb12bd0 |
|
||||
| Sandbox Escape | 2 | 0 | [2026-06-24][c-3a8b65cb45] 3a8b65cb45 |
|
||||
| Signature Verification Bypass | 2 | 2 | [2026-07-03][c-fac0b7196d] fac0b7196d |
|
||||
| Certificate Validation Bypass | 1 | 0 | [2026-11-04][c-82d5a8acc2] 82d5a8acc2 |
|
||||
| Information Disclosure | 1 | 0 | [2026-08-03][c-a20aa26024] a20aa26024 |
|
||||
| Path Traversal | 1 | 0 | [2026-06-04][c-20cbf0bd73] 20cbf0bd73 |
|
||||
| SQL Injection | 1 | 0 | [2026-06-23][c-68daef44a7] 68daef44a7 |
|
||||
@@ -106,17 +100,15 @@ Published elsewhere, fixed before the history window, or not yet linked; full de
|
||||
| CVE-2024-40641 / GHSA-c3q9-c27p-cw9h | high 7.4 | 2024-07-17 | CWE-78 | Unsigned code template execution through workflows in projectdiscovery/nuclei. ### Summary... |
|
||||
| CVE-2024-27920 / GHSA-w5wx-6g2r-r78q | high 7.4 | 2024-03-15 | CWE-78 | Unsigned code template execution through workflows in projectdiscovery/nuclei. ### Overvie... |
|
||||
|
||||
[c-40d08ee493]: https://github.com/projectdiscovery/nuclei/commit/40d08ee493f6f84c9a6b3c7f15f22da83853130c
|
||||
[c-4fed40f75a]: https://github.com/projectdiscovery/nuclei/commit/4fed40f75a57c7c414aac62d213d772cc3f430d3
|
||||
[c-ccbfb12bd0]: https://github.com/projectdiscovery/nuclei/commit/ccbfb12bd01447ba25f6e755dfb2bee677736da6
|
||||
[c-df57ae6d29]: https://github.com/projectdiscovery/nuclei/commit/df57ae6d2955864fd92a1bd2565fe9e4e9a7592b
|
||||
[c-d43c1f547f]: https://github.com/projectdiscovery/nuclei/commit/d43c1f547f98ac56654f4335ad20af0cca4c229c
|
||||
[c-d9dabbb4a1]: https://github.com/projectdiscovery/nuclei/commit/d9dabbb4a19e897a6bc12334a030a43035b27b73
|
||||
[c-e5154d362a]: https://github.com/projectdiscovery/nuclei/commit/e5154d362af900e83b30fb79028bfab1a7f0c648
|
||||
[c-634cecbb28]: https://github.com/projectdiscovery/nuclei/commit/634cecbb289db494247c04a9c4ec6a113aef3cc0
|
||||
[c-82d5a8acc2]: https://github.com/projectdiscovery/nuclei/commit/82d5a8acc2c756394c7fa92eae08b41330e89d69
|
||||
[c-ccbfb12bd0]: https://github.com/projectdiscovery/nuclei/commit/ccbfb12bd01447ba25f6e755dfb2bee677736da6
|
||||
[c-d221732016]: https://github.com/projectdiscovery/nuclei/commit/d2217320162d5782ca7cb95bef9dda17063818f3
|
||||
[c-1c0f77d28e]: https://github.com/projectdiscovery/nuclei/commit/1c0f77d28ed639a34c2036fa23c7503ffac0bf27
|
||||
[c-a20aa26024]: https://github.com/projectdiscovery/nuclei/commit/a20aa260245e5db889c5a421d4a65aeb2a64bf46
|
||||
[c-1c0f77d28e]: https://github.com/projectdiscovery/nuclei/commit/1c0f77d28ed639a34c2036fa23c7503ffac0bf27
|
||||
[c-fac0b7196d]: https://github.com/projectdiscovery/nuclei/commit/fac0b7196d2fbca56c8af8ff8e1899c111041a37
|
||||
[c-3a8b65cb45]: https://github.com/projectdiscovery/nuclei/commit/3a8b65cb45f121fcdf418f60e5177ed9f250c122
|
||||
[c-68daef44a7]: https://github.com/projectdiscovery/nuclei/commit/68daef44a7eafd41c39690924620e418762909ef
|
||||
|
||||
@@ -22,6 +22,8 @@ extend-exclude = [
|
||||
"go.sum",
|
||||
# Generated history. Commit SHAs are not words.
|
||||
"SECURITY_CONTEXT.md",
|
||||
# Generated technology graph. Entries are third-party product identifiers.
|
||||
"pkg/protocols/common/automaticscan/techgraph/tech-graph.json",
|
||||
]
|
||||
|
||||
[default.extend-identifiers]
|
||||
|
||||
+1
-6
@@ -307,14 +307,13 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"),
|
||||
flagSet.StringSliceVarP(&options.NewTemplatesWithVersion, "new-templates-version", "ntv", nil, "run new templates added in specific version", goflags.CommaSeparatedStringSliceOptions),
|
||||
flagSet.BoolVarP(&options.AutomaticScan, "automatic-scan", "as", false, "automatic web scan using wappalyzer technology detection to tags mapping"),
|
||||
flagSet.StringVarP(&options.AutomaticScanCoverage, "automatic-scan-coverage", "asc", "balanced", "automatic scan coverage tier: lean, balanced, or thorough"),
|
||||
flagSet.StringSliceVarP(&options.Templates, "templates", "t", nil, "list of template or template directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
|
||||
flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "turl", nil, "template url or list containing template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
|
||||
flagSet.StringVarP(&options.AITemplatePrompt, "prompt", "ai", "", "generate and run template using ai prompt"),
|
||||
flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", nil, "list of workflow or workflow directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
|
||||
flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
|
||||
flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
|
||||
flagSet.StringVar(&options.PlanFile, "plan", "", "write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target"),
|
||||
flagSet.StringVar(&options.RunPlanFile, "run-plan", "", "run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes"),
|
||||
flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
|
||||
flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
|
||||
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all templates matching current filters"),
|
||||
@@ -357,7 +356,6 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write found issues/vulnerabilities"),
|
||||
flagSet.BoolVarP(&options.StoreResponse, "store-resp", "sresp", false, "store all request/response passed through nuclei to output directory"),
|
||||
flagSet.StringVarP(&options.StoreResponseDir, "store-resp-dir", "srd", runner.DefaultDumpTrafficOutputFolder, "store all request/response passed through nuclei to custom directory"),
|
||||
flagSet.StringVar(&options.StoreEvents, "store-events", "", "store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)"),
|
||||
flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"),
|
||||
flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"),
|
||||
flagSet.BoolVarP(&options.JSONL, "jsonl", "j", false, "write output in JSONL(ines) format"),
|
||||
@@ -372,7 +370,6 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"),
|
||||
flagSet.StringVarP(&options.JSONExport, "json-export", "je", "", "file to export results in JSON format"),
|
||||
flagSet.StringVarP(&options.JSONLExport, "jsonl-export", "jle", "", "file to export results in JSONL(ine) format"),
|
||||
flagSet.StringVarP(&options.CSVExport, "csv-export", "ce", "", "file to export results in CSV format"),
|
||||
flagSet.StringVarP(&options.PDFExport, "pdf-export", "pe", "", "file to export results in PDF format"),
|
||||
flagSet.StringSliceVarP(&options.Redact, "redact", "rd", nil, "redact given list of keys from query parameter, request header and body", goflags.CommaSeparatedStringSliceOptions),
|
||||
flagSet.BoolVarP(&options.NoRedact, "no-redact", "nrd", false, "disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted"),
|
||||
@@ -393,7 +390,6 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.BoolVarP(&options.SystemResolvers, "system-resolvers", "sr", false, "use system DNS resolving as error fallback"),
|
||||
flagSet.BoolVarP(&options.DisableClustering, "disable-clustering", "dc", false, "disable clustering of requests"),
|
||||
flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"),
|
||||
flagSet.StringVar(&options.ReplayFile, "replay", "", "run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests"),
|
||||
flagSet.BoolVarP(&options.ForceAttemptHTTP2, "force-http2", "fh2", false, "force http2 connection on requests"),
|
||||
flagSet.BoolVarP(&options.EnvironmentVariables, "env-vars", "ev", false, "enable environment variables to be used in template"),
|
||||
flagSet.StringVarP(&options.ClientCertFile, "client-cert", "cc", "", "client certificate file (PEM-encoded) used for authenticating against scanned hosts"),
|
||||
@@ -412,7 +408,6 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 0, "max response size to read in bytes"),
|
||||
flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", unitutils.Mega, "max response size to read in bytes"),
|
||||
flagSet.IntVarP(&options.RequestSaveSize, "request-size-save", "rqss", 32*unitutils.Kilo, "max request body size to save in bytes (0 to save the whole body)"),
|
||||
flagSet.IntVarP(&options.MaxRequestBodyInFlight, "max-request-body-inflight", "mrbi", 0, "max request body bytes held by in-flight http requests; requests with bodies of 1MB or more queue past it (0 disables)"),
|
||||
flagSet.CallbackVar(func() { resetRequested = true }, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
|
||||
flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
|
||||
flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"),
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module github.com/projectdiscovery/nuclei/v3
|
||||
|
||||
go 1.27.4
|
||||
go 1.27.3
|
||||
|
||||
require (
|
||||
carvel.dev/ytt v0.52.0
|
||||
@@ -18,7 +18,7 @@ require (
|
||||
github.com/alecthomas/chroma/v2 v2.27.0
|
||||
github.com/alexsnet/go-vnc v0.1.0
|
||||
github.com/alitto/pond v1.9.2
|
||||
github.com/andybalholm/brotli v1.2.7
|
||||
github.com/andybalholm/brotli v1.2.0
|
||||
github.com/andygrunwald/go-jira v1.16.1
|
||||
github.com/antchfx/htmlquery v1.3.7
|
||||
github.com/antchfx/xmlquery v1.4.4
|
||||
@@ -46,6 +46,7 @@ require (
|
||||
github.com/go-pdf/fpdf v0.9.0
|
||||
github.com/go-pg/pg/v10 v10.15.0
|
||||
github.com/go-playground/validator/v10 v10.26.0
|
||||
github.com/go-rod/rod v0.116.2
|
||||
github.com/go-sql-driver/mysql v1.10.0
|
||||
github.com/gobwas/ws v1.4.0
|
||||
github.com/goccy/go-json v0.10.5
|
||||
@@ -62,7 +63,7 @@ require (
|
||||
github.com/jhump/protoreflect v1.17.0
|
||||
github.com/julienschmidt/httprouter v1.3.0
|
||||
github.com/kitabisa/go-ci v1.0.3
|
||||
github.com/klauspost/compress v1.20.2
|
||||
github.com/klauspost/compress v1.18.7
|
||||
github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa
|
||||
github.com/lib/pq v1.12.3
|
||||
github.com/logrusorgru/aurora/v4 v4.0.0
|
||||
@@ -101,7 +102,6 @@ require (
|
||||
github.com/projectdiscovery/rdap v0.9.0
|
||||
github.com/projectdiscovery/retryabledns v1.0.116
|
||||
github.com/projectdiscovery/retryablehttp-go v1.3.31
|
||||
github.com/projectdiscovery/rod v0.116.3-0.20261113150229-d775599ff53d
|
||||
github.com/projectdiscovery/sarif v0.1.0
|
||||
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75
|
||||
github.com/projectdiscovery/uncover v1.2.1
|
||||
@@ -129,14 +129,14 @@ require (
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
go.uber.org/multierr v1.11.0
|
||||
golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3
|
||||
golang.org/x/net v0.62.0
|
||||
golang.org/x/net v0.59.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sync v0.24.0
|
||||
golang.org/x/sys v0.49.0
|
||||
golang.org/x/term v0.47.0
|
||||
golang.org/x/text v0.43.0
|
||||
golang.org/x/sync v0.23.0
|
||||
golang.org/x/sys v0.48.0
|
||||
golang.org/x/term v0.46.0
|
||||
golang.org/x/text v0.42.0
|
||||
golang.org/x/time v0.15.0
|
||||
golang.org/x/tools v0.51.0
|
||||
golang.org/x/tools v0.49.0
|
||||
google.golang.org/grpc v1.83.2
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
@@ -427,7 +427,11 @@ require (
|
||||
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
github.com/ysmood/fetchup v0.2.3 // indirect
|
||||
github.com/ysmood/goob v0.4.0 // indirect
|
||||
github.com/ysmood/got v0.40.0 // indirect
|
||||
github.com/ysmood/gson v0.7.3 // indirect
|
||||
github.com/ysmood/leakless v0.9.0 // indirect
|
||||
github.com/yuin/goldmark v1.7.17 // indirect
|
||||
github.com/yuin/goldmark-emoji v1.0.6 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
@@ -451,7 +455,7 @@ require (
|
||||
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
||||
goftp.io/server/v2 v2.0.1 // indirect
|
||||
golang.org/x/arch v0.3.0 // indirect
|
||||
golang.org/x/crypto v0.58.0 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/mod v0.41.0 // indirect
|
||||
google.golang.org/api v0.264.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
|
||||
|
||||
@@ -170,8 +170,8 @@ github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e
|
||||
github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||
github.com/alitto/pond v1.9.2 h1:9Qb75z/scEZVCoSU+osVmQ0I0JOeLfdTDafrbcJ8CLs=
|
||||
github.com/alitto/pond v1.9.2/go.mod h1:xQn3P/sHTYcU/1BR3i86IGIrilcrGC2LiS+E2+CJWsI=
|
||||
github.com/andybalholm/brotli v1.2.7 h1:UgafRPGMam5ySHfRkhthMqOFA6IFoeF3D7mWIDLjG5A=
|
||||
github.com/andybalholm/brotli v1.2.7/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
||||
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q38Qufg=
|
||||
github.com/andybalholm/cascadia v1.3.4/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41GBe/5O+G93kM=
|
||||
github.com/andygrunwald/go-jira v1.16.1 h1:WoQEar5XoDRAibOgKzTFELlPNlKAtnfWr296R9zdFLA=
|
||||
@@ -493,6 +493,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k=
|
||||
github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
|
||||
github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA=
|
||||
github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg=
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
|
||||
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
|
||||
@@ -747,8 +749,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
|
||||
github.com/kitabisa/go-ci v1.0.3 h1:JmIUIvcercRQc/9x/v02ydCCqU4MadSHaNaOF8T2pGA=
|
||||
github.com/kitabisa/go-ci v1.0.3/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U=
|
||||
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo=
|
||||
github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
|
||||
github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
@@ -983,8 +985,6 @@ github.com/projectdiscovery/retryabledns v1.0.116 h1:tK28PJ4tLO7bgM89RpIFjkfTPMS
|
||||
github.com/projectdiscovery/retryabledns v1.0.116/go.mod h1:r3vH0AKYtJ9fculcqRFrdVYe2ABcHwZeR4C6j3rqCbY=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.3.31 h1:0hSEZ5+jj6Oo1IL8qrytD36TK/qCxfe543jEaRqSTjs=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.3.31/go.mod h1:/WEa7YejxTCXEMLCGfGYaC3JsgXgzuqw8rv8cb4buEc=
|
||||
github.com/projectdiscovery/rod v0.116.3-0.20261113150229-d775599ff53d h1:6mwNBdlYWe0hu1/QLLhIrZiz+G1JYsU7OIVmCOaF+q0=
|
||||
github.com/projectdiscovery/rod v0.116.3-0.20261113150229-d775599ff53d/go.mod h1:FLVLMgXw6OFcOU5nd4Il8cdSWgDfAlDeRZMol3drmGs=
|
||||
github.com/projectdiscovery/sarif v0.1.0 h1:O541T+a448nSJsmIMnXXSOeDQEzpnCAYvRfe0eG5h74=
|
||||
github.com/projectdiscovery/sarif v0.1.0/go.mod h1:LBC+reM3bkI3qIIhE0rZaINaYX6VG+En6u2hHa5mA7E=
|
||||
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
|
||||
@@ -1212,14 +1212,20 @@ github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+Seva
|
||||
github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
|
||||
github.com/ysmood/gop v0.4.2 h1:KtW1/RugapWs1iW9ynQdk52BK6TWdP9zC0qDOhW8EDE=
|
||||
github.com/ysmood/gop v0.4.2/go.mod h1:K4LAB/BdLArknXrVSy+DyXcfb8P85JwAtwYM8zlb0mU=
|
||||
github.com/ysmood/got v0.43.0 h1:bcHlKg+qnhFz5JxNZ+j4fIfxxB8WWjTh6qdqeS8FfZw=
|
||||
github.com/ysmood/got v0.43.0/go.mod h1:rsfH2KJ5rcS/5n9pmybIGodoHlT/NTYxjgh3ob7UdSU=
|
||||
github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ=
|
||||
github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
|
||||
github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
|
||||
github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
|
||||
github.com/ysmood/gop v0.2.0 h1:+tFrG0TWPxT6p9ZaZs+VY+opCvHU8/3Fk6BaNv6kqKg=
|
||||
github.com/ysmood/gop v0.2.0/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
|
||||
github.com/ysmood/got v0.40.0 h1:ZQk1B55zIvS7zflRrkGfPDrPG3d7+JOza1ZkNxcc74Q=
|
||||
github.com/ysmood/got v0.40.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg=
|
||||
github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
|
||||
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
|
||||
github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
|
||||
github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
|
||||
github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU=
|
||||
github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
@@ -1329,8 +1335,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -1418,8 +1424,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY=
|
||||
golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
|
||||
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
|
||||
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -1446,8 +1452,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
|
||||
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -1512,8 +1518,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
@@ -1526,8 +1532,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
|
||||
golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY=
|
||||
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
|
||||
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -1544,8 +1550,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
|
||||
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
|
||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
@@ -1597,8 +1603,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI=
|
||||
golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA=
|
||||
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
||||
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
||||
@@ -25,7 +25,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/csv"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
|
||||
@@ -64,8 +63,7 @@ func ConfigureOptions() error {
|
||||
// would hold the scan until it does.
|
||||
func readsTargetsFromStdin(options *types.Options, stdinIsPipe bool) bool {
|
||||
hasOtherTargets := len(options.Targets) > 0 || options.TargetsFilePath != "" || options.Uncover || len(options.UncoverQuery) > 0
|
||||
// a replay runs over stored events and has no targets to wait for
|
||||
return stdinIsPipe && !options.DisableStdin && !hasOtherTargets && options.ReplayFile == ""
|
||||
return stdinIsPipe && !options.DisableStdin && !hasOtherTargets
|
||||
}
|
||||
|
||||
// ParseOptions parses the command line flags provided by a user
|
||||
@@ -121,14 +119,6 @@ func ParseOptions(options *types.Options) {
|
||||
options.Logger.Debug().Msgf("Store response directory specified, enabling \"store-resp\" flag automatically\n")
|
||||
options.StoreResponse = true
|
||||
}
|
||||
// Derived before validation: -uncover-query turns uncover on, and -plan
|
||||
// has to reject that before any target lookup starts.
|
||||
if options.UncoverQuery != nil {
|
||||
options.Uncover = true
|
||||
if len(options.UncoverEngine) == 0 {
|
||||
options.UncoverEngine = append(options.UncoverEngine, "shodan")
|
||||
}
|
||||
}
|
||||
// Validate the options passed by the user and if any
|
||||
// invalid options have been used, exit.
|
||||
if err := ValidateOptions(options); err != nil {
|
||||
@@ -148,6 +138,13 @@ func ParseOptions(options *types.Options) {
|
||||
_ = os.Setenv("GITHUB_TOKEN", options.GitHubToken)
|
||||
}
|
||||
|
||||
if options.UncoverQuery != nil {
|
||||
options.Uncover = true
|
||||
if len(options.UncoverEngine) == 0 {
|
||||
options.UncoverEngine = append(options.UncoverEngine, "shodan")
|
||||
}
|
||||
}
|
||||
|
||||
if options.OfflineHTTP {
|
||||
options.DisableHTTPProbe = true
|
||||
}
|
||||
@@ -165,9 +162,6 @@ func ValidateOptions(options *types.Options) error {
|
||||
}
|
||||
return errors.Wrap(errors.New(strings.Join(errs, ", ")), "validation failed for these fields")
|
||||
}
|
||||
if err := validatePlanOptions(options); err != nil {
|
||||
return err
|
||||
}
|
||||
if options.Verbose && options.Silent {
|
||||
return errors.New("both verbose and silent mode specified")
|
||||
}
|
||||
@@ -373,18 +367,6 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
if options.CSVExport != "" {
|
||||
// Combine the CLI options with the config file options with the CLI options taking precedence
|
||||
if reportingOptions.CSVExporter != nil {
|
||||
reportingOptions.CSVExporter.File = options.CSVExport
|
||||
reportingOptions.CSVExporter.OmitRaw = options.OmitRawRequests
|
||||
} else {
|
||||
reportingOptions.CSVExporter = &csv.Options{
|
||||
File: options.CSVExport,
|
||||
OmitRaw: options.OmitRawRequests,
|
||||
}
|
||||
}
|
||||
}
|
||||
if options.PDFExport != "" {
|
||||
if reportingOptions.PDFExporter != nil {
|
||||
reportingOptions.PDFExporter.File = options.PDFExport
|
||||
@@ -554,35 +536,3 @@ func getBoolEnvValue(key string) bool {
|
||||
value := os.Getenv(key)
|
||||
return strings.EqualFold(value, "true")
|
||||
}
|
||||
|
||||
// validatePlanOptions rejects options that change the templates or targets
|
||||
// only once scanning starts, which a plan written beforehand cannot show.
|
||||
func validatePlanOptions(options *types.Options) error {
|
||||
if options.RunPlanFile != "" && options.PlanFile != "" {
|
||||
return errors.New("-plan cannot be used with -run-plan")
|
||||
}
|
||||
if options.RunPlanFile != "" && options.Resume != "" {
|
||||
return errors.New("-run-plan cannot be used with -resume, the plan's done log resumes it")
|
||||
}
|
||||
if options.PlanFile == "" && options.RunPlanFile == "" {
|
||||
return nil
|
||||
}
|
||||
// -automatic-scan builds its own engine, so a plan set on the main engine
|
||||
// would never be consulted. The other flags change the targets only after
|
||||
// a plan has been written.
|
||||
flag := "-plan"
|
||||
if options.PlanFile == "" {
|
||||
flag = "-run-plan"
|
||||
}
|
||||
switch {
|
||||
case options.AutomaticScan:
|
||||
return fmt.Errorf("%s cannot be used with -automatic-scan, which picks templates while scanning", flag)
|
||||
case options.Uncover || len(options.UncoverQuery) > 0:
|
||||
return fmt.Errorf("%s cannot be used with -uncover, which finds targets over the network", flag)
|
||||
case options.PreflightPortScan:
|
||||
return fmt.Errorf("%s cannot be used with -preflight-portscan, which drops targets after scanning their ports", flag)
|
||||
case options.DASTServer || options.DASTProxy:
|
||||
return fmt.Errorf("%s cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan", flag)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -84,7 +84,6 @@ func TestReadsTargetsFromStdin(t *testing.T) {
|
||||
{"targets given with -l", &types.Options{TargetsFilePath: "targets.txt"}, true, false},
|
||||
{"uncover query", &types.Options{UncoverQuery: goflags.StringSlice{"example"}}, true, false},
|
||||
{"uncover enabled", &types.Options{Uncover: true}, true, false},
|
||||
{"replay", &types.Options{ReplayFile: "events.jsonl"}, true, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
require.Equal(t, tc.want, readsTargetsFromStdin(tc.options, tc.stdinIsPipe))
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"slices"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/core"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
)
|
||||
|
||||
// httpRequestSeconds is the time of one http request the plan costs are
|
||||
// weighted against (see core.PlanEntry.Cost).
|
||||
const httpRequestSeconds = 0.015
|
||||
|
||||
// planSummary is the last line of a plan file, written once every entry is
|
||||
// known.
|
||||
type planSummary struct {
|
||||
Entries int `json:"entries"`
|
||||
// Alternatives are entries that run only when another of their origin
|
||||
// fails, so they are left out of every other total.
|
||||
Alternatives int `json:"alternatives,omitempty"`
|
||||
Requests int `json:"requests"`
|
||||
Cost int `json:"cost"`
|
||||
// EstimatedSeconds is the largest of the time the rate limit allows, the
|
||||
// cost spread over every concurrent slot, and the longest entry, whose
|
||||
// requests run one after another. Hosts slower than the measured request
|
||||
// time make the scan take longer.
|
||||
EstimatedSeconds int `json:"estimated-seconds"`
|
||||
}
|
||||
|
||||
// writePlan writes one JSON line per unit of work the scan would run,
|
||||
// followed by the summary. Targets are listed as given: inputs that the scan
|
||||
// would probe to an http url first are not probed here, since probing sends
|
||||
// requests.
|
||||
func (r *Runner) writePlan(store *loader.Store, engine *core.Engine) error {
|
||||
if !r.options.DisableHTTPProbe && loader.IsHTTPBasedProtocolUsed(store) && r.isInputNonHTTP() {
|
||||
r.Logger.Info().Msgf("Targets without a scheme are planned as given; the scan probes them for http(s) first")
|
||||
}
|
||||
file, err := os.Create(r.options.PlanFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
writer := bufio.NewWriter(file)
|
||||
encoder := json.NewEncoder(writer)
|
||||
|
||||
var summary planSummary
|
||||
var headlessCost, longest int
|
||||
var encodeErr error
|
||||
engine.Plan(slices.Concat(store.Templates(), store.Workflows()), r.inputProvider, r.options.DisableClustering, func(entry core.PlanEntry) {
|
||||
if encodeErr != nil {
|
||||
return
|
||||
}
|
||||
encodeErr = encoder.Encode(entry)
|
||||
if entry.AlternativeTo != "" {
|
||||
summary.Alternatives++
|
||||
return
|
||||
}
|
||||
summary.Entries++
|
||||
summary.Requests += entry.Requests
|
||||
summary.Cost += entry.Cost
|
||||
longest = max(longest, entry.Cost)
|
||||
if entry.Protocol == types.HeadlessProtocol.String() {
|
||||
headlessCost += entry.Cost
|
||||
}
|
||||
})
|
||||
if encodeErr != nil {
|
||||
return encodeErr
|
||||
}
|
||||
summary.EstimatedSeconds = r.estimateSeconds(summary.Requests, summary.Cost-headlessCost, headlessCost, longest)
|
||||
if err := encoder.Encode(map[string]planSummary{"summary": summary}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writer.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
r.Logger.Info().Msgf("Plan written to %s: %d template runs, up to %d requests, about %ds", r.options.PlanFile, summary.Entries, summary.Requests, summary.EstimatedSeconds)
|
||||
return nil
|
||||
}
|
||||
|
||||
// estimateSeconds estimates the scan duration. Headless templates run in
|
||||
// their own, smaller pool, so their cost is spread over it separately.
|
||||
func (r *Runner) estimateSeconds(requests, cost, headlessCost, longest int) int {
|
||||
seconds := float64(longest) * httpRequestSeconds
|
||||
if r.options.RateLimit > 0 && r.options.RateLimitDuration > 0 {
|
||||
seconds = max(seconds, float64(requests)/(float64(r.options.RateLimit)/r.options.RateLimitDuration.Seconds()))
|
||||
}
|
||||
if slots := r.options.TemplateThreads * r.options.BulkSize; slots > 0 {
|
||||
seconds = max(seconds, float64(cost)*httpRequestSeconds/float64(slots))
|
||||
}
|
||||
if slots := r.options.HeadlessTemplateThreads * r.options.HeadlessBulkSize; slots > 0 {
|
||||
seconds = max(seconds, float64(headlessCost)*httpRequestSeconds/float64(slots))
|
||||
}
|
||||
return int(seconds + 0.5)
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPlanRejectsOptionsThatChangeTargetsLater(t *testing.T) {
|
||||
for name, options := range map[string]*types.Options{
|
||||
"automatic scan": {PlanFile: "p", AutomaticScan: true},
|
||||
"uncover": {PlanFile: "p", Uncover: true},
|
||||
"uncover query": {PlanFile: "p", UncoverQuery: goflags.StringSlice{"example"}},
|
||||
"preflight": {PlanFile: "p", PreflightPortScan: true},
|
||||
"dast server": {PlanFile: "p", DASTServer: true},
|
||||
"dast proxy": {PlanFile: "p", DASTProxy: true},
|
||||
} {
|
||||
require.Error(t, validatePlanOptions(options), name)
|
||||
}
|
||||
require.NoError(t, validatePlanOptions(&types.Options{PlanFile: "p"}))
|
||||
require.NoError(t, validatePlanOptions(&types.Options{Uncover: true}), "only -plan restricts them")
|
||||
require.NoError(t, validatePlanOptions(&types.Options{UncoverQuery: goflags.StringSlice{"example"}}))
|
||||
require.NoError(t, validatePlanOptions(&types.Options{DASTServer: true}))
|
||||
|
||||
require.Error(t, validatePlanOptions(&types.Options{RunPlanFile: "p", PlanFile: "q"}))
|
||||
require.Error(t, validatePlanOptions(&types.Options{RunPlanFile: "p", Resume: "r"}))
|
||||
require.NoError(t, validatePlanOptions(&types.Options{RunPlanFile: "p"}))
|
||||
|
||||
for name, options := range map[string]*types.Options{
|
||||
"run automatic scan": {RunPlanFile: "p", AutomaticScan: true},
|
||||
"run uncover": {RunPlanFile: "p", Uncover: true},
|
||||
"run uncover query": {RunPlanFile: "p", UncoverQuery: goflags.StringSlice{"example"}},
|
||||
"run preflight": {RunPlanFile: "p", PreflightPortScan: true},
|
||||
"run dast server": {RunPlanFile: "p", DASTServer: true},
|
||||
"run dast proxy": {RunPlanFile: "p", DASTProxy: true},
|
||||
} {
|
||||
require.Error(t, validatePlanOptions(options), name)
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestEstimateSecondsTakesTheSlowestBound(t *testing.T) {
|
||||
r := &Runner{options: &types.Options{
|
||||
RateLimit: 150, RateLimitDuration: time.Second,
|
||||
TemplateThreads: 25, BulkSize: 25,
|
||||
HeadlessTemplateThreads: 10, HeadlessBulkSize: 10,
|
||||
}}
|
||||
|
||||
require.Equal(t, 20, r.estimateSeconds(3000, 3000, 0, 12), "rate limit bound")
|
||||
require.Equal(t, 10, r.estimateSeconds(52, 41, 660, 660), "one long entry runs its requests in sequence")
|
||||
require.Equal(t, 9, r.estimateSeconds(100, 0, 60000, 60), "headless cost spreads over the headless pool")
|
||||
|
||||
r.options.RateLimit = 0
|
||||
require.Equal(t, 0, r.estimateSeconds(3000, 3000, 0, 1), "no rate limit, cost spread over 625 slots")
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"slices"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/replay"
|
||||
)
|
||||
|
||||
// replay writes the findings of the loaded templates over a stored corpus
|
||||
// through the usual output, so -o, -j and the other output options apply.
|
||||
func (r *Runner) replay(store *loader.Store) error {
|
||||
results, err := replay.Replay(r.options.ReplayFile, slices.Concat(store.Templates(), store.Workflows()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, result := range results {
|
||||
if err := r.output.Write(result); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
r.Logger.Info().Msgf("Replayed %s: %d findings", r.options.ReplayFile, len(results))
|
||||
return nil
|
||||
}
|
||||
@@ -1,215 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
)
|
||||
|
||||
// planRun limits a scan to the entries of a plan file and appends the id of
|
||||
// each finished entry to the plan's done log, so running the plan again skips
|
||||
// them. Ids are derived from what an entry runs, not from its position, so the
|
||||
// log still applies after the plan is written again for changed targets or
|
||||
// templates.
|
||||
type planRun struct {
|
||||
mu sync.Mutex
|
||||
entries map[uint64]plannedEntry
|
||||
// done holds finished origin groups by the id of their first entry; an
|
||||
// entry outside a group is its own group
|
||||
done map[uint64]struct{}
|
||||
log *os.File
|
||||
logErr error
|
||||
// resumed counts the entries finished before this run
|
||||
resumed int
|
||||
}
|
||||
|
||||
type plannedEntry struct {
|
||||
group uint64
|
||||
seen bool
|
||||
}
|
||||
|
||||
func planDoneLog(planFile string) string {
|
||||
return planFile + ".done"
|
||||
}
|
||||
|
||||
func openPlanRun(planFile string) (*planRun, error) {
|
||||
run := &planRun{entries: make(map[uint64]plannedEntry), done: make(map[uint64]struct{})}
|
||||
if err := run.readPlan(planFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := run.readDoneLog(planDoneLog(planFile)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log, err := os.OpenFile(planDoneLog(planFile), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// an interrupted write leaves a partial line, which the next id must not
|
||||
// extend
|
||||
if info, err := log.Stat(); err == nil && info.Size() > 0 {
|
||||
last := make([]byte, 1)
|
||||
if _, err := log.ReadAt(last, info.Size()-1); err == nil && last[0] != '\n' {
|
||||
_, _ = log.WriteString("\n")
|
||||
}
|
||||
}
|
||||
run.log = log
|
||||
return run, nil
|
||||
}
|
||||
|
||||
func (run *planRun) readPlan(planFile string) error {
|
||||
file, err := os.Open(planFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
reader := bufio.NewReader(file)
|
||||
for number := 1; ; number++ {
|
||||
line, readErr := reader.ReadBytes('\n')
|
||||
if line = bytes.TrimSpace(line); len(line) > 0 {
|
||||
var entry struct {
|
||||
ID string `json:"id"`
|
||||
AlternativeTo string `json:"alternative-to"`
|
||||
}
|
||||
if err := json.Unmarshal(line, &entry); err != nil {
|
||||
return fmt.Errorf("%s:%d: %w", planFile, number, err)
|
||||
}
|
||||
// the summary line has no id
|
||||
if entry.ID != "" {
|
||||
id, ok := parsePlanID(entry.ID)
|
||||
if !ok {
|
||||
return fmt.Errorf("%s:%d: invalid id %q", planFile, number, entry.ID)
|
||||
}
|
||||
group := id
|
||||
if entry.AlternativeTo != "" {
|
||||
if group, ok = parsePlanID(entry.AlternativeTo); !ok {
|
||||
return fmt.Errorf("%s:%d: invalid alternative-to %q", planFile, number, entry.AlternativeTo)
|
||||
}
|
||||
}
|
||||
run.entries[id] = plannedEntry{group: group}
|
||||
}
|
||||
}
|
||||
if readErr == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (run *planRun) readDoneLog(path string) error {
|
||||
file, err := os.Open(path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
// a partial line from an interrupted write does not parse and is
|
||||
// run again
|
||||
id, ok := parsePlanID(scanner.Text())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if entry, planned := run.entries[id]; planned {
|
||||
run.done[entry.group] = struct{}{}
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, entry := range run.entries {
|
||||
if _, done := run.done[entry.group]; done {
|
||||
run.resumed++
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parsePlanID keeps ids as integers, which a plan of millions of entries
|
||||
// holds in a fraction of the memory of strings.
|
||||
func parsePlanID(id string) (uint64, bool) {
|
||||
if len(id) != 16 {
|
||||
return 0, false
|
||||
}
|
||||
value, err := strconv.ParseUint(id, 16, 64)
|
||||
return value, err == nil
|
||||
}
|
||||
|
||||
func (run *planRun) Skip(id string) bool {
|
||||
key, ok := parsePlanID(id)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
run.mu.Lock()
|
||||
defer run.mu.Unlock()
|
||||
entry, planned := run.entries[key]
|
||||
if !planned {
|
||||
return true
|
||||
}
|
||||
if !entry.seen {
|
||||
entry.seen = true
|
||||
run.entries[key] = entry
|
||||
}
|
||||
_, done := run.done[entry.group]
|
||||
return done
|
||||
}
|
||||
|
||||
func (run *planRun) Finished(id string) {
|
||||
key, ok := parsePlanID(id)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
run.mu.Lock()
|
||||
defer run.mu.Unlock()
|
||||
entry, planned := run.entries[key]
|
||||
if !planned {
|
||||
return
|
||||
}
|
||||
run.done[entry.group] = struct{}{}
|
||||
// written straight to the file: ctrl+c exits without running defers
|
||||
if _, err := run.log.WriteString(id + "\n"); err != nil && run.logErr == nil {
|
||||
run.logErr = err
|
||||
}
|
||||
}
|
||||
|
||||
// unreached counts unfinished entries the scan did not get to: those of a
|
||||
// host it found unresponsive, and those it does not have because it runs
|
||||
// with other templates, targets or options than the plan.
|
||||
func (run *planRun) unreached() int {
|
||||
run.mu.Lock()
|
||||
defer run.mu.Unlock()
|
||||
count := 0
|
||||
for _, entry := range run.entries {
|
||||
if _, done := run.done[entry.group]; !done && !entry.seen {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (run *planRun) close() error {
|
||||
if err := run.log.Close(); err != nil && run.logErr == nil {
|
||||
run.logErr = err
|
||||
}
|
||||
return run.logErr
|
||||
}
|
||||
|
||||
// finishPlanRun closes the done log and reports the entries left to run.
|
||||
func (r *Runner) finishPlanRun(run *planRun) {
|
||||
if err := run.close(); err != nil {
|
||||
r.Logger.Error().Msgf("Could not write %s: %s", planDoneLog(r.options.RunPlanFile), err)
|
||||
}
|
||||
if unreached := run.unreached(); unreached > 0 {
|
||||
r.Logger.Info().Msgf("%d plan entries did not run; running the plan again retries them. Entries run only when the scan has the templates, targets and options the plan was written with", unreached)
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPlanRunResumesFromItsDoneLog(t *testing.T) {
|
||||
planFile := filepath.Join(t.TempDir(), "plan.jsonl")
|
||||
require.NoError(t, os.WriteFile(planFile, []byte(`{"id":"00000000000000a1","target":"https://a.example/x","templates":["root"]}
|
||||
{"id":"00000000000000a2","target":"https://a.example/y","templates":["root"],"alternative-to":"00000000000000a1"}
|
||||
{"id":"00000000000000b1","target":"https://b.example","templates":["generic"]}
|
||||
{"id":"00000000000000c1","target":"https://c.example","templates":["generic"]}
|
||||
{"summary":{"entries":3,"alternatives":1}}
|
||||
`), 0o644))
|
||||
// the last line was cut off by an interrupted write
|
||||
require.NoError(t, os.WriteFile(planDoneLog(planFile), []byte("00000000000000b1\n00000000"), 0o644))
|
||||
|
||||
run, err := openPlanRun(planFile)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, run.entries, 4)
|
||||
require.Equal(t, 1, run.resumed)
|
||||
|
||||
require.True(t, run.Skip("00000000000000b1"), "finished in the previous run")
|
||||
require.True(t, run.Skip("00000000000000ff"), "not in the plan")
|
||||
require.False(t, run.Skip("00000000000000a1"))
|
||||
run.Finished("00000000000000a1")
|
||||
require.True(t, run.Skip("00000000000000a2"), "its origin is covered")
|
||||
require.Equal(t, 1, run.unreached(), "c1 never ran")
|
||||
require.NoError(t, run.close())
|
||||
|
||||
log, err := os.ReadFile(planDoneLog(planFile))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "00000000000000b1\n00000000\n00000000000000a1\n", string(log), "the partial line is not extended")
|
||||
|
||||
again, err := openPlanRun(planFile)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 3, again.resumed, "b1, and a1 with its alternative")
|
||||
require.True(t, again.Skip("00000000000000a2"), "a finished origin stays covered")
|
||||
require.False(t, again.Skip("00000000000000c1"))
|
||||
require.NoError(t, again.close())
|
||||
}
|
||||
|
||||
func TestPlanRunRejectsAMalformedPlan(t *testing.T) {
|
||||
planFile := filepath.Join(t.TempDir(), "plan.jsonl")
|
||||
require.NoError(t, os.WriteFile(planFile, []byte("{\"id\":\"00000000000000a1\"}\nnot json\n"), 0o644))
|
||||
_, err := openPlanRun(planFile)
|
||||
require.ErrorContains(t, err, "plan.jsonl:2")
|
||||
}
|
||||
@@ -57,7 +57,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/honeypotdetector"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
|
||||
@@ -70,12 +69,10 @@ import (
|
||||
httpProtocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/catchall"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/replay"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
@@ -149,18 +146,10 @@ func New(options *types.Options) (*Runner, error) {
|
||||
|
||||
// Version check by default
|
||||
if config.DefaultConfig.CanCheckForUpdates() {
|
||||
// -plan sends no request it can avoid, so it skips the version and
|
||||
// update checks; it still installs templates on a fresh setup, which
|
||||
// would otherwise plan nothing
|
||||
checkUpdates := options.PlanFile == ""
|
||||
var latestIgnoreHash string
|
||||
if checkUpdates {
|
||||
var versionCheckErr error
|
||||
latestIgnoreHash, versionCheckErr = installer.NucleiVersionCheck()
|
||||
if versionCheckErr != nil {
|
||||
if options.Verbose || options.Debug {
|
||||
runner.Logger.Error().Msgf("nuclei version check failed got: %s\n", versionCheckErr)
|
||||
}
|
||||
latestIgnoreHash, versionCheckErr := installer.NucleiVersionCheck()
|
||||
if versionCheckErr != nil {
|
||||
if options.Verbose || options.Debug {
|
||||
runner.Logger.Error().Msgf("nuclei version check failed got: %s\n", versionCheckErr)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,19 +170,17 @@ func New(options *types.Options) (*Runner, error) {
|
||||
if err := tm.FreshInstallIfNotExists(); err != nil {
|
||||
runner.Logger.Warning().Msgf("Failed to install nuclei templates: %s\n", err)
|
||||
}
|
||||
if checkUpdates {
|
||||
if err := tm.UpdateIfOutdated(); err != nil {
|
||||
runner.Logger.Warning().Msgf("Failed to update nuclei templates: %s\n", err)
|
||||
}
|
||||
if err := tm.UpdateIfOutdated(); err != nil {
|
||||
runner.Logger.Warning().Msgf("Failed to update nuclei templates: %s\n", err)
|
||||
}
|
||||
|
||||
if checkUpdates && config.DefaultConfig.IgnoreFileNeedsUpdate(latestIgnoreHash) {
|
||||
if config.DefaultConfig.IgnoreFileNeedsUpdate(latestIgnoreHash) {
|
||||
if err := installer.UpdateIgnoreFile(); err != nil {
|
||||
runner.Logger.Warning().Msgf("Failed to update .nuclei-ignore file: %s\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
if checkUpdates && options.UpdateTemplates {
|
||||
if options.UpdateTemplates {
|
||||
// we automatically check for updates unless explicitly disabled
|
||||
// this print statement is only to inform the user that there are no updates
|
||||
noUpdates := !config.DefaultConfig.NeedsTemplateUpdate()
|
||||
@@ -604,13 +591,8 @@ func (r *Runner) RunEnumeration() error {
|
||||
httpclientpool.ResetConnectionStats()
|
||||
|
||||
// If the user has asked for DAST server mode, run the live
|
||||
// DAST fuzzing server. -plan never reaches this with those flags set:
|
||||
// validation rejects the combination, and this check covers a caller
|
||||
// that skipped it.
|
||||
// DAST fuzzing server.
|
||||
if r.options.DASTServer || r.options.DASTProxy {
|
||||
if err := validatePlanOptions(r.options); err != nil {
|
||||
return err
|
||||
}
|
||||
execurOpts := &server.NucleiExecutorOptions{
|
||||
Options: r.options,
|
||||
Output: r.output,
|
||||
@@ -694,7 +676,6 @@ func (r *Runner) RunEnumeration() error {
|
||||
Catalog: r.catalog,
|
||||
IssuesClient: r.issuesClient,
|
||||
RateLimiter: r.rateLimiter,
|
||||
RequestBodyBudget: httpbody.NewBudget(int64(r.options.MaxRequestBodyInFlight)),
|
||||
Interactsh: r.interactsh,
|
||||
ProjectFile: r.projectFile,
|
||||
Browser: r.browser,
|
||||
@@ -834,36 +815,6 @@ func (r *Runner) RunEnumeration() error {
|
||||
profiles.Prepare(slices.Concat(store.Templates(), store.Workflows()))
|
||||
executorOpts.TargetScope = profiles
|
||||
}
|
||||
// before uncover, preflight, auth prefetch and probing below, all of which
|
||||
// reach the network
|
||||
if r.options.PlanFile != "" {
|
||||
return r.writePlan(store, executorEngine)
|
||||
}
|
||||
if r.options.ReplayFile != "" {
|
||||
return r.replay(store)
|
||||
}
|
||||
if r.options.StoreEvents != "" {
|
||||
recorder, err := replay.NewRecorder(r.options.StoreEvents, slices.Concat(store.Templates(), store.Workflows()))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not create events file")
|
||||
}
|
||||
previous := eventcreator.SetRecord(recorder.Record)
|
||||
defer func() {
|
||||
eventcreator.SetRecord(previous)
|
||||
if err := recorder.Close(); err != nil {
|
||||
r.Logger.Error().Msgf("Could not write %s: %s", r.options.StoreEvents, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
if r.options.RunPlanFile != "" {
|
||||
run, err := openPlanRun(r.options.RunPlanFile)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not read plan")
|
||||
}
|
||||
r.Logger.Info().Msgf("Running %d plan entries from %s, %d already finished", len(run.entries), r.options.RunPlanFile, run.resumed)
|
||||
executorEngine.SetPlanRun(run)
|
||||
defer r.finishPlanRun(run)
|
||||
}
|
||||
// TODO: remove below functions after v3 or update warning messages
|
||||
templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
|
||||
|
||||
|
||||
@@ -38,35 +38,6 @@ func TestCreateReportingOptions(t *testing.T) {
|
||||
require.Equal(t, resultOptions2.DenyList.Severities, resultOptions.DenyList.Severities)
|
||||
}
|
||||
|
||||
func TestCreateReportingOptionsCSVExport(t *testing.T) {
|
||||
t.Run("flag only", func(t *testing.T) {
|
||||
resultOptions, err := createReportingOptions(&types.Options{CSVExport: "out.csv", OmitRawRequests: true})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resultOptions.CSVExporter)
|
||||
require.Equal(t, "out.csv", resultOptions.CSVExporter.File)
|
||||
require.True(t, resultOptions.CSVExporter.OmitRaw)
|
||||
})
|
||||
|
||||
t.Run("flag is not set", func(t *testing.T) {
|
||||
resultOptions, err := createReportingOptions(&types.Options{})
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, resultOptions.CSVExporter)
|
||||
})
|
||||
|
||||
t.Run("flag takes precedence over config file", func(t *testing.T) {
|
||||
config := filepath.Join(t.TempDir(), "report.yaml")
|
||||
require.NoError(t, os.WriteFile(config, []byte("csv:\n file: from-config.csv\n"), 0o600))
|
||||
|
||||
resultOptions, err := createReportingOptions(&types.Options{ReportingConfig: config})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "from-config.csv", resultOptions.CSVExporter.File)
|
||||
|
||||
resultOptions, err = createReportingOptions(&types.Options{ReportingConfig: config, CSVExport: "from-flag.csv"})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "from-flag.csv", resultOptions.CSVExporter.File)
|
||||
})
|
||||
}
|
||||
|
||||
type TestStruct1 struct {
|
||||
A string `yaml:"a"`
|
||||
Struct *TestStruct2 `yaml:"b"`
|
||||
|
||||
@@ -46,35 +46,6 @@ func TestNetwork(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ReplayStoredEvents", func(t *testing.T) {
|
||||
server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
|
||||
if err == nil && string(data) == "PING" {
|
||||
_, _ = conn.Write([]byte("PONG"))
|
||||
}
|
||||
})
|
||||
events := filepath.Join(t.TempDir(), "events.jsonl")
|
||||
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/basic.yaml", server.URL, suite.debug, "-store-events", events)
|
||||
server.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("recording network events failed: %v", err)
|
||||
}
|
||||
if err := expectResultsCount(results, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// the server is gone, so the finding can only come from the stored event
|
||||
results, err = testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil, "-t", "protocols/network/basic.yaml", "-replay", events)
|
||||
if err != nil {
|
||||
t.Fatalf("replaying network events failed: %v", err)
|
||||
}
|
||||
if err := expectResultsCount(results, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("PinnedTargetLine", func(t *testing.T) {
|
||||
server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
+6
-8
@@ -39,14 +39,12 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
|
||||
Catalog: base.catalog,
|
||||
IssuesClient: base.rc,
|
||||
RateLimiter: base.rateLimiter,
|
||||
|
||||
RequestBodyBudget: base.executerOpts.RequestBodyBudget,
|
||||
Interactsh: base.interactshClient,
|
||||
Colorizer: aurora.New(aurora.WithColors(true)),
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
Parser: base.parser,
|
||||
Browser: base.browserInstance,
|
||||
DoNotCache: opts.DoNotCacheTemplates,
|
||||
Interactsh: base.interactshClient,
|
||||
Colorizer: aurora.New(aurora.WithColors(true)),
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
Parser: base.parser,
|
||||
Browser: base.browserInstance,
|
||||
DoNotCache: opts.DoNotCacheTemplates,
|
||||
}
|
||||
if base.interactshClient != nil {
|
||||
u.executerOpts.InteractshScope = base.interactshClient.NewRequestScope()
|
||||
|
||||
@@ -36,7 +36,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
nucleiUtils "github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
|
||||
)
|
||||
@@ -280,7 +279,6 @@ func (e *NucleiEngine) init(ctx context.Context) error {
|
||||
Catalog: e.catalog,
|
||||
IssuesClient: e.rc,
|
||||
RateLimiter: e.rateLimiter,
|
||||
RequestBodyBudget: httpbody.NewBudget(int64(e.opts.MaxRequestBodyInFlight)),
|
||||
Interactsh: e.interactshClient,
|
||||
Colorizer: aurora.New(aurora.WithColors(true)),
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
|
||||
@@ -2,7 +2,6 @@ package loader
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
@@ -20,7 +19,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
templatesigner "github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -687,135 +685,9 @@ http:
|
||||
loaded := loadSingleTemplateForTest(t, templatePath, "loader-unsigned-flow-template")
|
||||
require.Len(t, loaded, 1)
|
||||
require.Equal(t, "unsigned-flow-template", loaded[0].ID)
|
||||
require.False(t, loaded[0].Verified)
|
||||
require.Equal(t, initialUnverifiedJavascript, stats.GetValue(templates.SkippedUnverifiedJavascriptTemplateStats))
|
||||
}
|
||||
|
||||
func contentDigestForLoaderTest(data []byte) [32]byte {
|
||||
dataDigest := sha256.Sum256(data)
|
||||
return sha256.Sum256(dataDigest[:])
|
||||
}
|
||||
|
||||
func loadTemplateWithPoisonedMetadata(t *testing.T, templatePath, executionID string, source []byte, disableUnsigned bool) []*templates.Template {
|
||||
t.Helper()
|
||||
|
||||
fileInfo, err := os.Stat(templatePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
metadataIndex, err := metadataindex.NewIndex(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
metadataIndex.Set(templatePath, &metadataindex.Metadata{
|
||||
ID: "poisoned",
|
||||
FilePath: templatePath,
|
||||
ModTime: fileInfo.ModTime(),
|
||||
Name: "Poisoned",
|
||||
Authors: []string{"pdteam"},
|
||||
Severity: "info",
|
||||
Verified: true,
|
||||
TemplateVerifier: "projectdiscovery/nuclei-templates",
|
||||
VerifierFingerprint: templatesigner.DefaultTemplateVerifiers[0].Fingerprint(),
|
||||
ContentDigest: contentDigestForLoaderTest(source),
|
||||
Validation: metadataindex.ValidationStrict,
|
||||
})
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.Logger = &gologger.Logger{}
|
||||
options.ExecutionId = executionID
|
||||
options.DisableUnsignedTemplates = disableUnsigned
|
||||
options.TemplateLoadingConcurrency = 1
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() {
|
||||
testutils.Cleanup(options)
|
||||
})
|
||||
|
||||
catalog := disk.NewCatalog("")
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, nil)
|
||||
executerOpts.Catalog = catalog
|
||||
executerOpts.Parser = templates.NewParser()
|
||||
executerOpts.Logger = options.Logger
|
||||
|
||||
workflowLoader, err := workflow.NewLoader(executerOpts)
|
||||
require.NoError(t, err)
|
||||
executerOpts.WorkflowLoader = workflowLoader
|
||||
|
||||
loaderConfig := NewConfig(options, catalog, executerOpts)
|
||||
loaderConfig.MetadataIndex = metadataIndex
|
||||
store, err := New(loaderConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
loaded, err := store.LoadTemplates([]string{templatePath})
|
||||
require.NoError(t, err)
|
||||
return loaded
|
||||
}
|
||||
|
||||
func TestLoadTemplatesIgnoresSelfConsistentCacheForJavascript(t *testing.T) {
|
||||
templatePath := filepath.Join(t.TempDir(), "poisoned-javascript.yaml")
|
||||
source := []byte(`id: poisoned-cached-javascript
|
||||
|
||||
info:
|
||||
name: Poisoned Cached Javascript
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
javascript:
|
||||
- init: |
|
||||
set("init-status", "executed")
|
||||
code: |
|
||||
Export("poisoned-cached-javascript")
|
||||
`)
|
||||
require.NoError(t, os.WriteFile(templatePath, source, 0o600))
|
||||
|
||||
loaded := loadTemplateWithPoisonedMetadata(t, templatePath, "loader-poisoned-javascript", source, false)
|
||||
require.Empty(t, loaded)
|
||||
}
|
||||
|
||||
func TestLoadTemplatesAllowsUnsignedFlowDespitePoisonedCache(t *testing.T) {
|
||||
templatePath := filepath.Join(t.TempDir(), "poisoned-flow.yaml")
|
||||
source := []byte(`id: poisoned-cached-flow
|
||||
|
||||
info:
|
||||
name: Poisoned Cached Flow
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
flow: http(1)
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
`)
|
||||
require.NoError(t, os.WriteFile(templatePath, source, 0o600))
|
||||
|
||||
loaded := loadTemplateWithPoisonedMetadata(t, templatePath, "loader-poisoned-flow", source, false)
|
||||
require.Len(t, loaded, 1)
|
||||
require.Equal(t, "poisoned-cached-flow", loaded[0].ID)
|
||||
require.False(t, loaded[0].Verified)
|
||||
require.True(t, loaded[0].IsFlowTemplate())
|
||||
}
|
||||
|
||||
func TestLoadTemplatesSkipsPoisonedHttpWhenUnsignedDisabled(t *testing.T) {
|
||||
templatePath := filepath.Join(t.TempDir(), "poisoned-http.yaml")
|
||||
source := []byte(`id: poisoned-cached-http
|
||||
|
||||
info:
|
||||
name: Poisoned Cached HTTP
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
`)
|
||||
require.NoError(t, os.WriteFile(templatePath, source, 0o600))
|
||||
|
||||
initialUnverified := stats.GetValue(templates.SkippedUnverifiedTemplateStats)
|
||||
loaded := loadTemplateWithPoisonedMetadata(t, templatePath, "loader-poisoned-http-dut", source, true)
|
||||
require.Empty(t, loaded)
|
||||
require.Equal(t, initialUnverified+1, stats.GetValue(templates.SkippedUnverifiedTemplateStats))
|
||||
}
|
||||
|
||||
func TestLoadTemplatesDoesNotRequireGlobalMatchersFlagToLoadTemplate(t *testing.T) {
|
||||
templatePath := filepath.Join(t.TempDir(), "global-matchers.yaml")
|
||||
err := os.WriteFile(templatePath, []byte(`id: global-matchers-template
|
||||
|
||||
@@ -51,7 +51,6 @@ type Engine struct {
|
||||
executerOpts *protocols.ExecutorOptions
|
||||
Callback func(*output.ResultEvent) // Executed on results
|
||||
templateExecutionCallback TemplateExecutionCallback
|
||||
planRun PlanRun
|
||||
Logger *gologger.Logger
|
||||
}
|
||||
|
||||
|
||||
+13
-17
@@ -9,7 +9,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
|
||||
@@ -36,11 +35,22 @@ func (e *Engine) ExecuteWithResults(ctx context.Context, templatesList []*templa
|
||||
func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider, noCluster bool) *atomic.Bool {
|
||||
results := &atomic.Bool{}
|
||||
selfcontainedWg := &sync.WaitGroup{}
|
||||
httpclientpool.ResetSharedResponses(e.options)
|
||||
|
||||
totalReqBeforeCluster := getRequestCount(templatesList) * int(target.Count())
|
||||
|
||||
finalTemplates, clusterCount := e.clusterTemplates(templatesList, noCluster)
|
||||
// attempt to cluster templates if noCluster is false
|
||||
var finalTemplates []*templates.Template
|
||||
clusterCount := 0
|
||||
if !noCluster {
|
||||
var clusterMappings map[string][]string
|
||||
finalTemplates, clusterCount, clusterMappings = templates.ClusterTemplates(templatesList, e.executerOpts)
|
||||
// Store cluster mappings in executerOpts for SDK access (thread-safe)
|
||||
if clusterMappings != nil {
|
||||
e.executerOpts.ClusterMappings = types.NewClusterMappingsMap(clusterMappings)
|
||||
}
|
||||
} else {
|
||||
finalTemplates = templatesList
|
||||
}
|
||||
|
||||
totalReqAfterClustering := getRequestCount(finalTemplates) * int(target.Count())
|
||||
|
||||
@@ -176,20 +186,6 @@ func (e *Engine) executeHostSpray(ctx context.Context, templatesList []*template
|
||||
return results
|
||||
}
|
||||
|
||||
// clusterTemplates merges templates that send identical requests unless
|
||||
// noCluster is set, and returns the templates to run with the number merged.
|
||||
func (e *Engine) clusterTemplates(templatesList []*templates.Template, noCluster bool) ([]*templates.Template, int) {
|
||||
if noCluster {
|
||||
return templatesList, 0
|
||||
}
|
||||
finalTemplates, clusterCount, clusterMappings := templates.ClusterTemplates(templatesList, e.executerOpts)
|
||||
// Store cluster mappings in executerOpts for SDK access (thread-safe)
|
||||
if clusterMappings != nil {
|
||||
e.executerOpts.ClusterMappings = types.NewClusterMappingsMap(clusterMappings)
|
||||
}
|
||||
return finalTemplates, clusterCount
|
||||
}
|
||||
|
||||
// expectedWork returns the requests and the tasks the target-bound templates
|
||||
// will run. It counts only the template and target pairs that per-target
|
||||
// profiles select, so progress reflects the work that will be done. A task is
|
||||
|
||||
+7
-32
@@ -36,13 +36,9 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
|
||||
if sharedBudget {
|
||||
defer e.options.ReleaseTemplateThread()
|
||||
}
|
||||
defer e.tasksDone(1)
|
||||
planned, skip := e.claimPlanned(template, &contextargs.MetaInput{})
|
||||
if skip {
|
||||
return
|
||||
}
|
||||
finished := e.templateExecutionStarted(template, "")
|
||||
defer func() { finished(ctx.Err()) }()
|
||||
defer e.tasksDone(1)
|
||||
var err error
|
||||
var match bool
|
||||
ctx := scan.NewScanContext(ctx, contextargs.New(ctx))
|
||||
@@ -59,8 +55,6 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
|
||||
}
|
||||
if err != nil {
|
||||
e.options.Logger.Warning().Msgf("[%s] Could not execute step (self-contained): %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
|
||||
} else if ctx.Context().Err() == nil {
|
||||
planned()
|
||||
}
|
||||
results.CompareAndSwap(false, match)
|
||||
}(v, usesSharedTemplateBudget)
|
||||
@@ -336,16 +330,6 @@ func (e *Engine) tasksDone(count int64) {
|
||||
func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates.Template, value *contextargs.MetaInput) (matched bool, err error) {
|
||||
// a pair skipped as a repeat of its origin is done as well
|
||||
defer e.tasksDone(1)
|
||||
planned, skip := e.claimPlanned(template, value)
|
||||
if skip {
|
||||
return false, nil
|
||||
}
|
||||
// an origin a sibling already covered is settled too, so it is recorded
|
||||
defer func() {
|
||||
if err == nil && ctx.Err() == nil {
|
||||
planned()
|
||||
}
|
||||
}()
|
||||
// A template fixed to the target's origin makes the same requests for every
|
||||
// target that shares one, which a crawled list produces by the hundred.
|
||||
// Running it once per origin sends exactly the same traffic, minus the
|
||||
@@ -398,19 +382,6 @@ func newOriginAttempt() *originAttempt {
|
||||
return attempt
|
||||
}
|
||||
|
||||
// originScopeKey returns the key an origin-scoped template runs once per, and
|
||||
// false when the template runs on every target.
|
||||
func originScopeKey(template *templates.Template, value *contextargs.MetaInput) (string, bool) {
|
||||
if template == nil || value == nil || !template.IsOriginScoped() {
|
||||
return "", false
|
||||
}
|
||||
origin := normalize.Origin(value.Input)
|
||||
if origin == "" {
|
||||
return "", false
|
||||
}
|
||||
return template.ID + "\x00" + origin + "\x00" + value.CustomIP, true
|
||||
}
|
||||
|
||||
// claimOriginScoped reports whether this caller should execute an origin-scoped
|
||||
// template. finish(true) records a successful run. finish(false) releases the
|
||||
// claim so a later target, including one that waited, can try again. Templates
|
||||
@@ -418,10 +389,14 @@ func originScopeKey(template *templates.Template, value *contextargs.MetaInput)
|
||||
// key so each address selected for one URL is tested on its own.
|
||||
func (e *Engine) claimOriginScoped(template *templates.Template, value *contextargs.MetaInput) (finish func(bool), run bool) {
|
||||
noop := func(bool) {}
|
||||
key, scoped := originScopeKey(template, value)
|
||||
if !scoped {
|
||||
if template == nil || value == nil || !template.IsOriginScoped() {
|
||||
return noop, true
|
||||
}
|
||||
origin := normalize.Origin(value.Input)
|
||||
if origin == "" {
|
||||
return noop, true
|
||||
}
|
||||
key := template.ID + "\x00" + origin + "\x00" + value.CustomIP
|
||||
actual, _ := e.originScoped.LoadOrStore(key, newOriginAttempt())
|
||||
slot := actual.(*originAttempt)
|
||||
|
||||
|
||||
@@ -1,197 +0,0 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
)
|
||||
|
||||
// PlanEntry is one unit of work a scan runs: a template, a cluster of
|
||||
// templates sharing one request, or a workflow, on one target.
|
||||
type PlanEntry struct {
|
||||
// ID identifies the entry across runs of the same plan.
|
||||
ID string `json:"id"`
|
||||
// Target is empty for self-contained templates, which run once.
|
||||
Target string `json:"target,omitempty"`
|
||||
// IP is the address the target is pinned to, when it is.
|
||||
IP string `json:"ip,omitempty"`
|
||||
Templates []string `json:"templates"`
|
||||
Protocol string `json:"protocol"`
|
||||
// Requests is what the templates declare, the count -stats starts from.
|
||||
// The scan sends fewer when a request needs a value the target lacks,
|
||||
// such as {{DN}} on an ip.
|
||||
Requests int `json:"requests"`
|
||||
// Cost is Requests weighted by how long one request of the protocol
|
||||
// takes, in http requests.
|
||||
Cost int `json:"cost"`
|
||||
// Dynamic marks entries whose requests depend on responses: workflows
|
||||
// (only the root is counted), flows and fuzzing.
|
||||
Dynamic bool `json:"dynamic,omitempty"`
|
||||
// AlternativeTo is the id of an entry running the same origin-scoped
|
||||
// template on the same origin. Workers race for an origin, so the scan
|
||||
// runs whichever of the group claims it first, and another only when
|
||||
// that run fails. Totals count the group once.
|
||||
AlternativeTo string `json:"alternative-to,omitempty"`
|
||||
}
|
||||
|
||||
// Plan reports the work ExecuteScanWithOpts would do with the same arguments,
|
||||
// without sending anything. It applies the same clustering, per-target
|
||||
// profiles and once-per-origin rule, so a pair is listed only if the scan
|
||||
// would run it. Later targets of an origin-scoped template are listed as
|
||||
// alternatives to the first, since the scan retries them when a run fails.
|
||||
func (e *Engine) Plan(templatesList []*templates.Template, target provider.InputProvider, noCluster bool, emit func(PlanEntry)) {
|
||||
finalTemplates, _ := e.clusterTemplates(templatesList, noCluster)
|
||||
// clustering does not keep an order; a plan is diffed and edited, so it does
|
||||
finalTemplates = slices.SortedFunc(slices.Values(finalTemplates), func(a, b *templates.Template) int {
|
||||
return strings.Compare(a.ID, b.ID)
|
||||
})
|
||||
|
||||
var targetBound []*templates.Template
|
||||
for _, template := range finalTemplates {
|
||||
if template.SelfContained {
|
||||
emit(withPlanID(planEntry(template, &contextargs.MetaInput{}, nil)))
|
||||
} else {
|
||||
targetBound = append(targetBound, template)
|
||||
}
|
||||
}
|
||||
|
||||
// entries are emitted as they are planned, so memory does not grow with
|
||||
// the plan; only the first id per origin is kept
|
||||
originIDs := make(map[string]string)
|
||||
target.Iterate(func(value *contextargs.MetaInput) bool {
|
||||
selection := e.selectionFor(value)
|
||||
for _, template := range targetBound {
|
||||
if selection != nil && !template.SelectedBy(selection) {
|
||||
continue
|
||||
}
|
||||
entry := withPlanID(planEntry(template, value, selection))
|
||||
if key, scoped := originScopeKey(template, value); scoped {
|
||||
if id, seen := originIDs[key]; seen {
|
||||
entry.AlternativeTo = id
|
||||
} else {
|
||||
originIDs[key] = entry.ID
|
||||
}
|
||||
}
|
||||
emit(entry)
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func withPlanID(entry PlanEntry) PlanEntry {
|
||||
entry.ID = planEntryID(entry)
|
||||
return entry
|
||||
}
|
||||
|
||||
// protocolCost is how many http requests one request of a protocol is worth
|
||||
// in time. Measured per call against a CDN-fronted site, where an http
|
||||
// request took about 15ms: a javascript call adds about 150ms of runtime, a
|
||||
// headless page load with waitload took about 1s. Unlisted protocols cost
|
||||
// about one round trip, like http.
|
||||
var protocolCost = map[types.ProtocolType]int{
|
||||
types.JavascriptProtocol: 10,
|
||||
types.HeadlessProtocol: 60,
|
||||
}
|
||||
|
||||
func planEntry(template *templates.Template, value *contextargs.MetaInput, selection protocols.TemplateSelection) PlanEntry {
|
||||
entry := PlanEntry{
|
||||
Target: value.Input,
|
||||
IP: value.CustomIP,
|
||||
Templates: planTemplates(template, selection),
|
||||
Protocol: template.Type().String(),
|
||||
Requests: template.TotalRequests,
|
||||
Dynamic: template.Flow != "" || template.IsFuzzableRequest(),
|
||||
}
|
||||
weight := func(protocol types.ProtocolType) int { return max(protocolCost[protocol], 1) }
|
||||
|
||||
switch {
|
||||
case template.CompiledWorkflow != nil:
|
||||
// only the root of a workflow is certain to run; subtemplates depend
|
||||
// on what it matches
|
||||
entry.Requests = 0
|
||||
for _, workflow := range template.CompiledWorkflow.Workflows {
|
||||
for _, executer := range workflow.Executers {
|
||||
requests := executer.Executer.Requests()
|
||||
entry.Requests += requests
|
||||
entry.Cost += requests * weight(executer.TemplateType)
|
||||
}
|
||||
}
|
||||
entry.Dynamic = true
|
||||
case isCluster(template):
|
||||
cluster := template.Executer.(*templates.ClusterExecuter)
|
||||
// TotalRequests of a cluster counts only its http and dns requests,
|
||||
// so an ssl cluster would show none
|
||||
entry.Requests = cluster.Requests()
|
||||
entry.Cost = entry.Requests * weight(template.Type())
|
||||
case len(template.RequestsQueue) > 0:
|
||||
// each block is weighted by its own protocol, so a template mixing
|
||||
// http with javascript is not priced as all http
|
||||
for _, request := range template.RequestsQueue {
|
||||
entry.Cost += request.Requests() * weight(request.Type())
|
||||
}
|
||||
default:
|
||||
entry.Cost = entry.Requests * weight(template.Type())
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
// planTemplates lists the templates an entry runs, sorted so the entry id does
|
||||
// not depend on clustering order.
|
||||
func planTemplates(template *templates.Template, selection protocols.TemplateSelection) []string {
|
||||
if cluster, ok := template.Executer.(*templates.ClusterExecuter); ok {
|
||||
return slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
|
||||
}
|
||||
return []string{template.ID}
|
||||
}
|
||||
|
||||
func isCluster(template *templates.Template) bool {
|
||||
_, ok := template.Executer.(*templates.ClusterExecuter)
|
||||
return ok
|
||||
}
|
||||
|
||||
// planEntryID derives the id from what the entry runs, so planning the same
|
||||
// scan again yields the same ids.
|
||||
func planEntryID(entry PlanEntry) string {
|
||||
sum := sha256.Sum256([]byte(entry.Target + "\x00" + entry.IP + "\x00" + strings.Join(entry.Templates, "\x00")))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// PlanRun limits a scan to the entries of a plan and records the ones that
|
||||
// finish, so running the plan again resumes it.
|
||||
type PlanRun interface {
|
||||
// Skip reports whether the entry is not in the plan or already finished.
|
||||
Skip(id string) bool
|
||||
// Finished records an entry that ran without error.
|
||||
Finished(id string)
|
||||
}
|
||||
|
||||
// SetPlanRun makes the scan run only the entries run allows.
|
||||
func (e *Engine) SetPlanRun(run PlanRun) {
|
||||
e.planRun = run
|
||||
}
|
||||
|
||||
// claimPlanned reports whether the pair is skipped under a plan run, and
|
||||
// otherwise returns the func that records it finished. Without a plan run
|
||||
// every pair runs.
|
||||
func (e *Engine) claimPlanned(template *templates.Template, value *contextargs.MetaInput) (finished func(), skip bool) {
|
||||
if e.planRun == nil {
|
||||
return func() {}, false
|
||||
}
|
||||
// as in Plan, a self-contained template is not narrowed by any target
|
||||
var selection protocols.TemplateSelection
|
||||
if !template.SelfContained {
|
||||
selection = e.selectionFor(value)
|
||||
}
|
||||
id := planEntryID(PlanEntry{Target: value.Input, IP: value.CustomIP, Templates: planTemplates(template, selection)})
|
||||
if e.planRun.Skip(id) {
|
||||
return nil, true
|
||||
}
|
||||
return func() { e.planRun.Finished(id) }, false
|
||||
}
|
||||
@@ -1,215 +0,0 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
tmpltypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPlanEntryCostWeightsByProtocol(t *testing.T) {
|
||||
headless := &templates.Template{ID: "h", RequestsHeadless: []*headless.Request{{}}, TotalRequests: 2}
|
||||
require.Equal(t, 120, planEntry(headless, &contextargs.MetaInput{Input: "t"}, nil).Cost)
|
||||
|
||||
plain := &templates.Template{ID: "p", TotalRequests: 2}
|
||||
require.Equal(t, 2, planEntry(plain, &contextargs.MetaInput{Input: "t"}, nil).Cost, "unlisted protocols cost one request each")
|
||||
}
|
||||
|
||||
// countedRequest is a request block with a fixed protocol and count.
|
||||
type countedRequest struct {
|
||||
protocols.Request
|
||||
protocol tmpltypes.ProtocolType
|
||||
requests int
|
||||
}
|
||||
|
||||
func (r countedRequest) Type() tmpltypes.ProtocolType { return r.protocol }
|
||||
func (r countedRequest) Requests() int { return r.requests }
|
||||
|
||||
func TestPlanEntryWeightsEachProtocolBlock(t *testing.T) {
|
||||
// one http request and ten javascript calls: Type() reports http for the
|
||||
// whole template, which would price it as 11
|
||||
mixed := &templates.Template{
|
||||
ID: "mixed",
|
||||
RequestsHTTP: []*http.Request{{}},
|
||||
TotalRequests: 11,
|
||||
RequestsQueue: []protocols.Request{
|
||||
countedRequest{protocol: tmpltypes.HTTPProtocol, requests: 1},
|
||||
countedRequest{protocol: tmpltypes.JavascriptProtocol, requests: 10},
|
||||
},
|
||||
}
|
||||
entry := planEntry(mixed, &contextargs.MetaInput{Input: "t"}, nil)
|
||||
require.Equal(t, 11, entry.Requests)
|
||||
require.Equal(t, 101, entry.Cost)
|
||||
}
|
||||
|
||||
func TestPlanListsThePairsTheScanRuns(t *testing.T) {
|
||||
options := &types.Options{BulkSize: 2, TemplateThreads: 2, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
|
||||
engine := New(options)
|
||||
engine.SetExecuterOptions(&protocols.ExecutorOptions{
|
||||
Logger: engine.Logger,
|
||||
Options: options,
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
ProtocolType: tmpltypes.HTTPProtocol,
|
||||
Progress: &progressRecorder{},
|
||||
TargetScope: scopeByInput{
|
||||
"php": {paths: []string{"thinkphp.yaml"}},
|
||||
"none": {},
|
||||
},
|
||||
})
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
runs []string
|
||||
)
|
||||
var templatesList []*templates.Template
|
||||
for _, path := range []string{"thinkphp.yaml", "generic.yaml"} {
|
||||
templatesList = append(templatesList, &templates.Template{
|
||||
ID: path,
|
||||
Path: path,
|
||||
TotalRequests: 3,
|
||||
Executer: &recordingExecuter{path: path, mu: &mu, runs: &runs},
|
||||
})
|
||||
}
|
||||
selfContained := &templates.Template{ID: "self.yaml", Path: "self.yaml", SelfContained: true, TotalRequests: 1}
|
||||
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "php"}, {Input: "none"}, {Input: "plain"}}}
|
||||
|
||||
var plan []PlanEntry
|
||||
engine.Plan(append(templatesList, selfContained), targets, true, func(entry PlanEntry) { plan = append(plan, entry) })
|
||||
|
||||
require.Empty(t, plan[0].Target, "a self-contained template runs once, with no target")
|
||||
require.Equal(t, []string{"self.yaml"}, plan[0].Templates)
|
||||
var planned []string
|
||||
ids := make(map[string]struct{})
|
||||
for _, entry := range plan[1:] {
|
||||
require.Equal(t, 3, entry.Requests)
|
||||
planned = append(planned, entry.Templates[0]+"@"+entry.Target)
|
||||
ids[entry.ID] = struct{}{}
|
||||
}
|
||||
require.Len(t, ids, len(plan)-1, "every entry has its own id")
|
||||
|
||||
var again []PlanEntry
|
||||
engine.Plan(append(templatesList, selfContained), targets, true, func(entry PlanEntry) { again = append(again, entry) })
|
||||
require.Equal(t, plan, again, "planning the same scan again yields the same entries and ids")
|
||||
|
||||
engine.ExecuteScanWithOpts(context.Background(), templatesList, targets, true)
|
||||
require.ElementsMatch(t, runs, planned, "the plan lists exactly the pairs the scan runs")
|
||||
require.ElementsMatch(t, []string{"thinkphp.yaml@php", "thinkphp.yaml@plain", "generic.yaml@plain"}, planned)
|
||||
}
|
||||
|
||||
func TestPlanListsOriginAlternatives(t *testing.T) {
|
||||
options := &types.Options{BulkSize: 1, TemplateThreads: 1, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
|
||||
engine := New(options)
|
||||
engine.SetExecuterOptions(&protocols.ExecutorOptions{
|
||||
Logger: engine.Logger,
|
||||
Options: options,
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
ProtocolType: tmpltypes.HTTPProtocol,
|
||||
Progress: &progressRecorder{},
|
||||
})
|
||||
|
||||
root := &templates.Template{
|
||||
ID: "root.yaml",
|
||||
RequestsHTTP: []*http.Request{{Path: []string{"{{RootURL}}/health"}}},
|
||||
TotalRequests: 1,
|
||||
}
|
||||
require.True(t, root.IsOriginScoped())
|
||||
generic := &templates.Template{ID: "generic.yaml", TotalRequests: 1}
|
||||
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{
|
||||
{Input: "https://example.com/a"},
|
||||
{Input: "https://example.com/b"},
|
||||
{Input: "https://other.example/a"},
|
||||
{Input: "https://example.com/a", CustomIP: "203.0.113.5"},
|
||||
}}
|
||||
|
||||
var plan []PlanEntry
|
||||
engine.Plan([]*templates.Template{generic, root}, targets, true, func(entry PlanEntry) {
|
||||
plan = append(plan, entry)
|
||||
})
|
||||
|
||||
var roots []PlanEntry
|
||||
for _, entry := range plan {
|
||||
if entry.Templates[0] == "root.yaml" {
|
||||
roots = append(roots, entry)
|
||||
}
|
||||
}
|
||||
require.Equal(t, []PlanEntry{
|
||||
{ID: roots[0].ID, Target: "https://example.com/a", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
|
||||
{ID: roots[1].ID, Target: "https://example.com/b", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1, AlternativeTo: roots[0].ID},
|
||||
{ID: roots[2].ID, Target: "https://other.example/a", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
|
||||
{ID: roots[3].ID, Target: "https://example.com/a", IP: "203.0.113.5", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
|
||||
}, roots)
|
||||
require.NotEqual(t, roots[0].ID, roots[3].ID, "a pinned address is its own attempt")
|
||||
|
||||
var again []PlanEntry
|
||||
engine.Plan([]*templates.Template{generic, root}, targets, true, func(entry PlanEntry) {
|
||||
again = append(again, entry)
|
||||
})
|
||||
require.Equal(t, plan, again)
|
||||
}
|
||||
|
||||
// fixedPlanRun is a plan run over fixed ids that records the finished ones.
|
||||
type fixedPlanRun struct {
|
||||
mu sync.Mutex
|
||||
allowed map[string]bool
|
||||
finished []string
|
||||
}
|
||||
|
||||
func (a *fixedPlanRun) Skip(id string) bool { return !a.allowed[id] }
|
||||
|
||||
func (a *fixedPlanRun) Finished(id string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.finished = append(a.finished, id)
|
||||
}
|
||||
|
||||
func TestPlanRunRunsOnlyThePlannedPairs(t *testing.T) {
|
||||
options := &types.Options{BulkSize: 2, TemplateThreads: 2, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
|
||||
engine := New(options)
|
||||
engine.SetExecuterOptions(&protocols.ExecutorOptions{
|
||||
Logger: engine.Logger,
|
||||
Options: options,
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
ProtocolType: tmpltypes.HTTPProtocol,
|
||||
Progress: &progressRecorder{},
|
||||
TargetScope: scopeByInput{"php": {paths: []string{"thinkphp.yaml"}}},
|
||||
})
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
runs []string
|
||||
)
|
||||
var templatesList []*templates.Template
|
||||
for _, path := range []string{"thinkphp.yaml", "generic.yaml"} {
|
||||
templatesList = append(templatesList, &templates.Template{
|
||||
ID: path,
|
||||
Path: path,
|
||||
TotalRequests: 1,
|
||||
Executer: &recordingExecuter{path: path, mu: &mu, runs: &runs},
|
||||
})
|
||||
}
|
||||
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "php"}, {Input: "plain"}}}
|
||||
|
||||
var plan []PlanEntry
|
||||
engine.Plan(templatesList, targets, true, func(entry PlanEntry) { plan = append(plan, entry) })
|
||||
require.Len(t, plan, 3)
|
||||
|
||||
// the first entry is left out of the plan, as if an agent removed it
|
||||
run := &fixedPlanRun{allowed: map[string]bool{plan[1].ID: true, plan[2].ID: true}}
|
||||
engine.SetPlanRun(run)
|
||||
engine.ExecuteScanWithOpts(context.Background(), templatesList, targets, true)
|
||||
|
||||
var want []string
|
||||
for _, entry := range plan[1:] {
|
||||
want = append(want, entry.Templates[0]+"@"+entry.Target)
|
||||
}
|
||||
require.ElementsMatch(t, want, runs, "only the planned pairs run")
|
||||
require.ElementsMatch(t, []string{plan[1].ID, plan[2].ID}, run.finished, "the scan derives the ids the plan wrote")
|
||||
}
|
||||
@@ -38,11 +38,14 @@ func NewEntityParser(dir string) (*EntityParser, error) {
|
||||
return parser.ParseFile(fset, filename, src, parser.ParseComments)
|
||||
},
|
||||
}
|
||||
|
||||
pkg, err := loadPackage(cfg, ".")
|
||||
pkgs, err := packages.Load(cfg, ".")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(pkgs) == 0 {
|
||||
return nil, errors.New("no packages found")
|
||||
}
|
||||
pkg := pkgs[0]
|
||||
|
||||
return &EntityParser{
|
||||
syntax: pkg.Syntax,
|
||||
@@ -284,8 +287,8 @@ func (p *EntityParser) extractClassProperties(node *ast.StructType) []Property {
|
||||
Description: field.Doc.Text(),
|
||||
}
|
||||
if strings.Contains(property.Type, ".") {
|
||||
// external type found
|
||||
property.Type = p.handleExternalType(property.Type)
|
||||
// external struct found
|
||||
property.Type = p.handleExternalStruct(property.Type)
|
||||
}
|
||||
properties = append(properties, property)
|
||||
}
|
||||
@@ -349,7 +352,7 @@ func (p *EntityParser) extractReturnType(fn *ast.FuncDecl) (out string) {
|
||||
for _, result := range fn.Type.Results.List {
|
||||
tmp := exprToString(result.Type)
|
||||
if strings.Contains(tmp, ".") && !strings.HasPrefix(tmp, "goja.") {
|
||||
tmp = p.handleExternalType(tmp) + " | null" // external object interfaces can always return null
|
||||
tmp = p.handleExternalStruct(tmp) + " | null" // external object interfaces can always return null
|
||||
}
|
||||
returns = append(returns, tmp)
|
||||
}
|
||||
@@ -374,10 +377,8 @@ func (p *EntityParser) extractReturnType(fn *ast.FuncDecl) (out string) {
|
||||
}
|
||||
}
|
||||
if sliceutil.Contains(returns, "error") {
|
||||
// external types already record that they can be null
|
||||
if !strings.HasSuffix(out, "| null") {
|
||||
out += " | null"
|
||||
}
|
||||
// add | null to the return type
|
||||
out = out + " | null"
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -415,18 +416,10 @@ func (p *EntityParser) extractParameters(fn *ast.FuncDecl) []Parameter {
|
||||
return parameters
|
||||
}
|
||||
|
||||
// handleExternalType returns the TypeScript type for typeName, a type from
|
||||
// another package such as ssh.ClientConfig. A struct is queued for
|
||||
// scrapeAndCreate, which describes its fields in an interface of the same name.
|
||||
// Any other type, such as the net.Conn interface, has no fields to describe,
|
||||
// so it becomes any. A typeName that does not resolve is queued too, so that
|
||||
// scrapeAndCreate reports it.
|
||||
func (p *EntityParser) handleExternalType(typeName string) string {
|
||||
obj, err := p.lookupType(typeName)
|
||||
if err == nil && !isStruct(obj.Type()) {
|
||||
return "any"
|
||||
}
|
||||
|
||||
// typeName is in format ssh.ClientConfig
|
||||
// it first fetches all fields from the struct and creates a new object
|
||||
// with that name and returns name of that object as type
|
||||
func (p *EntityParser) handleExternalStruct(typeName string) string {
|
||||
baseType := typeName[strings.LastIndex(typeName, ".")+1:]
|
||||
p.newObjects[typeName] = &Entity{
|
||||
Name: baseType,
|
||||
@@ -520,7 +513,6 @@ func (p *EntityParser) extractVarsNConstants() {
|
||||
|
||||
// loadImportedPackages loads all imported packages
|
||||
func (p *EntityParser) loadImportedPackages() error {
|
||||
cfg := &packages.Config{Mode: packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesInfo}
|
||||
// get all import statements
|
||||
// iterate over all imports
|
||||
for _, file := range p.syntax {
|
||||
@@ -530,7 +522,7 @@ func (p *EntityParser) loadImportedPackages() error {
|
||||
// remove the quotes from the path
|
||||
path = path[1 : len(path)-1]
|
||||
// load the package
|
||||
pkg, err := loadPackage(cfg, path)
|
||||
pkg, err := loadPackage(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -551,31 +543,18 @@ func (p *EntityParser) loadImportedPackages() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadPackage loads the package that pattern names.
|
||||
//
|
||||
// packages.Load reports parse, type, and list errors in Package.Errors
|
||||
// rather than in its returned error, so loadPackage converts them into an
|
||||
// error. The parser assumes a complete syntax tree and type information.
|
||||
func loadPackage(cfg *packages.Config, pattern string) (*packages.Package, error) {
|
||||
pkgs, err := packages.Load(cfg, pattern)
|
||||
// Load the package containing the type definition
|
||||
// TODO: we don't support named imports yet
|
||||
func loadPackage(pkgPath string) (*packages.Package, error) {
|
||||
cfg := &packages.Config{Mode: packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesInfo}
|
||||
pkgs, err := packages.Load(cfg, pkgPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(pkgs) == 0 {
|
||||
return nil, errors.New("no packages found")
|
||||
}
|
||||
|
||||
pkg := pkgs[0]
|
||||
if len(pkg.Errors) > 0 {
|
||||
errs := make([]error, len(pkg.Errors))
|
||||
for i, e := range pkg.Errors {
|
||||
errs[i] = e
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("package %s: %w", pkg.ID, errors.Join(errs...))
|
||||
}
|
||||
|
||||
return pkg, nil
|
||||
return pkgs[0], nil
|
||||
}
|
||||
|
||||
// exprToString converts an expression to a string
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
package tsgen_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/js/devtools/tsgen"
|
||||
)
|
||||
|
||||
func TestParseExternalTypes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ep, err := tsgen.NewEntityParser(filepath.Join("testdata", "external"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, ep.Parse())
|
||||
|
||||
entities := make(map[string]tsgen.Entity)
|
||||
for _, e := range ep.GetEntities() {
|
||||
entities[e.Name] = e
|
||||
}
|
||||
|
||||
// net.Conn is an interface, so it has no fields to describe.
|
||||
require.Equal(t, "any | null", entities["Dial"].Function.Returns)
|
||||
require.Equal(t, []tsgen.Property{{Name: "Conn", Type: "any", Description: ""}}, entities["Session"].Object.Properties)
|
||||
require.NotContains(t, entities, "Conn")
|
||||
|
||||
// url.URL is a struct, so its fields become an interface.
|
||||
require.Equal(t, "URL | null", entities["Parse"].Function.Returns)
|
||||
require.Equal(t, "interface", entities["URL"].Type)
|
||||
require.NotEmpty(t, entities["URL"].Object.Properties)
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
package tsgen
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/tools/go/packages"
|
||||
)
|
||||
|
||||
// writeModule writes src as the only package of a new Go module and
|
||||
// returns the module directory.
|
||||
func writeModule(t *testing.T, src string) string {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
files := map[string]string{
|
||||
"go.mod": "module example.com/tsgentest\n\ngo 1.21\n",
|
||||
"example.go": src,
|
||||
}
|
||||
|
||||
for name, content := range files {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600))
|
||||
}
|
||||
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestNewEntityParser(t *testing.T) {
|
||||
const src = `package example
|
||||
|
||||
import "strings"
|
||||
|
||||
// Upper returns s in upper case.
|
||||
func Upper(s string) string { return strings.ToUpper(s) }
|
||||
`
|
||||
|
||||
p, err := NewEntityParser(writeModule(t, src))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, p.Parse())
|
||||
|
||||
entities := p.GetEntities()
|
||||
|
||||
names := make([]string, 0, len(entities))
|
||||
for _, e := range entities {
|
||||
names = append(names, e.Name)
|
||||
}
|
||||
|
||||
require.Contains(t, names, "Upper")
|
||||
}
|
||||
|
||||
func TestNewEntityParserPackageErrors(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
src string
|
||||
}{
|
||||
{name: "syntax error", src: "package example\n\nfunc Broken( {\n\treturn 42\n}\n"},
|
||||
{name: "type error", src: "package example\n\nfunc Broken() int { return undefined }\n"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := NewEntityParser(writeModule(t, tt.src))
|
||||
require.ErrorContains(t, err, "example.go:")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadImportedPackagesMissing(t *testing.T) {
|
||||
const src = `package example; import "example.com/tsgentest/missing"`
|
||||
|
||||
file, err := parser.ParseFile(token.NewFileSet(), "example.go", src, parser.ImportsOnly)
|
||||
require.NoError(t, err)
|
||||
p := &EntityParser{
|
||||
syntax: []*ast.File{file},
|
||||
imports: map[string]*packages.Package{},
|
||||
}
|
||||
|
||||
var pkgErr packages.Error
|
||||
|
||||
require.ErrorAs(t, p.loadImportedPackages(), &pkgErr)
|
||||
require.Contains(t, pkgErr.Msg, "example.com/tsgentest/missing")
|
||||
}
|
||||
@@ -1,15 +1,13 @@
|
||||
package tsgen
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"go/types"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// errUnresolvedType is returned by lookupType for a name it cannot resolve.
|
||||
var errUnresolvedType = errors.New("unresolved type")
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
)
|
||||
|
||||
// scrape.go scrapes all information of exported type from different package
|
||||
|
||||
@@ -17,13 +15,24 @@ func (p *EntityParser) scrapeAndCreate(typeName string) error {
|
||||
if p.newObjects[typeName] == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
typeNameObj, err := p.lookupType(typeName)
|
||||
if err != nil {
|
||||
return err
|
||||
// get package name
|
||||
pkgName := strings.Split(typeName, ".")[0]
|
||||
baseTypeName := strings.Split(typeName, ".")[1]
|
||||
// get package
|
||||
pkg, ok := p.imports[pkgName]
|
||||
if !ok {
|
||||
return errkit.Newf("package %v for type %v not found", pkgName, typeName)
|
||||
}
|
||||
// get type
|
||||
obj := pkg.Types.Scope().Lookup(baseTypeName)
|
||||
if obj == nil {
|
||||
return errkit.Newf("type %v not found in package %+v", typeName, pkg)
|
||||
}
|
||||
// Ensure the object is a type name
|
||||
typeNameObj, ok := obj.(*types.TypeName)
|
||||
if !ok {
|
||||
return errkit.Newf("%v is not a type name", typeName)
|
||||
}
|
||||
|
||||
baseTypeName := typeNameObj.Name()
|
||||
// Ensure the type is a named struct type
|
||||
namedStruct, ok := typeNameObj.Type().Underlying().(*types.Struct)
|
||||
if !ok {
|
||||
@@ -63,36 +72,6 @@ func (p *EntityParser) scrapeAndCreate(typeName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// lookupType resolves typeName, such as net.Conn, in the packages imported by
|
||||
// the parsed files.
|
||||
func (p *EntityParser) lookupType(typeName string) (*types.TypeName, error) {
|
||||
pkgName, baseTypeName, _ := strings.Cut(typeName, ".")
|
||||
|
||||
pkg := p.imports[pkgName]
|
||||
if pkg == nil {
|
||||
return nil, fmt.Errorf("%w: package %v for type %v not found", errUnresolvedType, pkgName, typeName)
|
||||
}
|
||||
|
||||
obj := pkg.Types.Scope().Lookup(baseTypeName)
|
||||
if obj == nil {
|
||||
return nil, fmt.Errorf("%w: type %v not found in package %+v", errUnresolvedType, typeName, pkg)
|
||||
}
|
||||
|
||||
typeNameObj, ok := obj.(*types.TypeName)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: %v is not a type name", errUnresolvedType, typeName)
|
||||
}
|
||||
|
||||
return typeNameObj, nil
|
||||
}
|
||||
|
||||
// isStruct reports whether t is a struct type.
|
||||
func isStruct(t types.Type) bool {
|
||||
_, ok := t.Underlying().(*types.Struct)
|
||||
|
||||
return ok
|
||||
}
|
||||
|
||||
type ExtObject struct {
|
||||
builtIn map[string]string
|
||||
nested map[string]map[string]*ExtObject // Changed to map of field names to ExtObject
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
// Package external is a tsgen fixture that references types from other
|
||||
// packages, both structs and non-structs.
|
||||
package external
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
// Dial returns an external interface type.
|
||||
func Dial(network, address string) (net.Conn, error) {
|
||||
return net.Dial(network, address)
|
||||
}
|
||||
|
||||
// Parse returns an external struct type.
|
||||
func Parse(raw string) (*url.URL, error) {
|
||||
return url.Parse(raw)
|
||||
}
|
||||
|
||||
// Session holds an external interface type in an exported field.
|
||||
type Session struct {
|
||||
Conn net.Conn
|
||||
}
|
||||
|
||||
type dialer struct{}
|
||||
|
||||
// DialContext is exported, but its receiver is not.
|
||||
func (dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, network, address)
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
shared adapter package; not a public JavaScript module
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
|
||||
"fmt"
|
||||
|
||||
_ "github.com/microsoft/go-mssqldb"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
)
|
||||
|
||||
|
||||
+16
-87
@@ -9,58 +9,11 @@ import (
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
gomssql "github.com/microsoft/go-mssqldb"
|
||||
_ "github.com/microsoft/go-mssqldb"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
)
|
||||
|
||||
// mssqlCustomDialer routes connection dialing through Nuclei's fastdialer.
|
||||
// Implementing HostDialer ensures the driver delegates hostname resolution to the dialer
|
||||
// rather than using the local OS net.LookupIP, keeping DNS resolution, scan proxies,
|
||||
// and dial-time network policies strictly enforced.
|
||||
type mssqlCustomDialer struct {
|
||||
executionId string
|
||||
host string
|
||||
}
|
||||
|
||||
var _ gomssql.HostDialer = (*mssqlCustomDialer)(nil)
|
||||
|
||||
func (d *mssqlCustomDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
dialers := protocolstate.GetDialersWithId(d.executionId)
|
||||
if dialers == nil {
|
||||
return nil, fmt.Errorf("dialers not initialized for %s", d.executionId)
|
||||
}
|
||||
targetHost, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
targetHost = addr
|
||||
}
|
||||
if !protocolstate.IsHostAllowed(d.executionId, targetHost) {
|
||||
return nil, protocolstate.ErrHostDenied.Msgf(targetHost)
|
||||
}
|
||||
return dialers.Fastdialer.Dial(ctx, network, addr)
|
||||
}
|
||||
|
||||
func (d *mssqlCustomDialer) HostName() string {
|
||||
return d.host
|
||||
}
|
||||
|
||||
func openDB(executionId, target, username, password, dbName string) (*sql.DB, error) {
|
||||
host, _, err := net.SplitHostPort(target)
|
||||
if err != nil {
|
||||
host = target
|
||||
}
|
||||
connString := mssqlConnString(target, username, password, dbName)
|
||||
connector, err := gomssql.NewConnector(connString)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
connector.Dialer = &mssqlCustomDialer{
|
||||
executionId: executionId,
|
||||
host: host,
|
||||
}
|
||||
return sql.OpenDB(connector), nil
|
||||
}
|
||||
|
||||
type (
|
||||
// Client is a client for MS SQL database.
|
||||
// Internally client uses microsoft/go-mssqldb driver.
|
||||
@@ -114,7 +67,9 @@ func connect(ctx context.Context, executionId string, host string, port int, use
|
||||
|
||||
target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
|
||||
|
||||
db, err := openDB(executionId, target, username, password, dbName)
|
||||
connString := mssqlConnString(target, username, password, dbName)
|
||||
|
||||
db, err := sql.Open("sqlserver", connString)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -124,7 +79,16 @@ func connect(ctx context.Context, executionId string, host string, port int, use
|
||||
|
||||
_, err = db.ExecContext(ctx, "select 1")
|
||||
if err != nil {
|
||||
if isNetworkError(err) {
|
||||
switch {
|
||||
case strings.Contains(err.Error(), "connect: connection refused"):
|
||||
fallthrough
|
||||
case strings.Contains(err.Error(), "no pg_hba.conf entry for host"):
|
||||
fallthrough
|
||||
case strings.Contains(err.Error(), "network unreachable"):
|
||||
fallthrough
|
||||
case strings.Contains(err.Error(), "reset"):
|
||||
fallthrough
|
||||
case strings.Contains(err.Error(), "i/o timeout"):
|
||||
return false, err
|
||||
}
|
||||
return false, nil
|
||||
@@ -132,42 +96,6 @@ func connect(ctx context.Context, executionId string, host string, port int, use
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// isNetworkError reports whether an error indicates a network, dialer,
|
||||
// timeout, or policy failure as opposed to a server-side authentication rejection.
|
||||
func isNetworkError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return true
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) {
|
||||
return true
|
||||
}
|
||||
errStr := strings.ToLower(err.Error())
|
||||
for _, marker := range []string{
|
||||
"connection refused",
|
||||
"actively refused",
|
||||
"port closed",
|
||||
"unable to open tcp connection",
|
||||
"network unreachable",
|
||||
"connection reset",
|
||||
"i/o timeout",
|
||||
"connection timed out",
|
||||
"context canceled",
|
||||
"deadline exceeded",
|
||||
"dialers not initialized",
|
||||
"network policy",
|
||||
"host denied",
|
||||
} {
|
||||
if strings.Contains(errStr, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsMssql checks if the given host is running MS SQL database.
|
||||
// If the host is running MS SQL database, it returns true.
|
||||
// If the host is not running MS SQL database, it returns false.
|
||||
@@ -228,8 +156,9 @@ func (c *MSSQLClient) ExecuteQuery(ctx context.Context, host string, port int, u
|
||||
}
|
||||
|
||||
target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
|
||||
connString := mssqlConnString(target, username, password, dbName)
|
||||
|
||||
db, err := openDB(executionId, target, username, password, dbName)
|
||||
db, err := sql.Open("sqlserver", connString)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -1,18 +1,9 @@
|
||||
package mssql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
gomssql "github.com/microsoft/go-mssqldb"
|
||||
"github.com/microsoft/go-mssqldb/msdsn"
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -42,114 +33,3 @@ func TestConnectionStringKeepsPlainDatabaseName(t *testing.T) {
|
||||
|
||||
require.Equal(t, "master", cfg.Database)
|
||||
}
|
||||
|
||||
func TestOpenDBUsesCustomFastdialer(t *testing.T) {
|
||||
execID := "mssql-opendb-test"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
|
||||
db, err := openDB(execID, "127.0.0.1:1433", "user", "pass", "master")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, db)
|
||||
_ = db.Close()
|
||||
}
|
||||
|
||||
func TestMssqlDialerEnforcesNetworkPolicy(t *testing.T) {
|
||||
execID := "mssql-policy-test"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{
|
||||
ExecutionId: execID,
|
||||
ExcludeTargets: goflags.StringSlice{"denied.host"},
|
||||
}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
|
||||
dialer := &mssqlCustomDialer{
|
||||
executionId: execID,
|
||||
host: "denied.host",
|
||||
}
|
||||
require.Equal(t, "denied.host", dialer.HostName())
|
||||
var _ gomssql.HostDialer = dialer
|
||||
|
||||
_, err := dialer.DialContext(context.Background(), "tcp", "denied.host:1433")
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "network policy")
|
||||
|
||||
uninitDialer := &mssqlCustomDialer{
|
||||
executionId: "nonexistent-exec-id",
|
||||
host: "example.com",
|
||||
}
|
||||
_, err = uninitDialer.DialContext(context.Background(), "tcp", "example.com:1433")
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "dialers not initialized")
|
||||
}
|
||||
|
||||
func TestConnectFailsFastOnClosedPort(t *testing.T) {
|
||||
execID := "mssql-closed-port-test"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
_, portStr, err := net.SplitHostPort(ln.Addr().String())
|
||||
require.NoError(t, err)
|
||||
port, err := strconv.Atoi(portStr)
|
||||
require.NoError(t, err)
|
||||
_ = ln.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
ok, err := connect(ctx, execID, "127.0.0.1", port, "user", "pass", "master")
|
||||
require.Error(t, err)
|
||||
require.False(t, ok)
|
||||
}
|
||||
|
||||
func TestConnectWithCanceledContext(t *testing.T) {
|
||||
execID := "mssql-canceled-ctx-test"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // Cancel immediately
|
||||
|
||||
ok, err := connect(ctx, execID, "127.0.0.1", 1433, "user", "pass", "master")
|
||||
require.Error(t, err)
|
||||
require.False(t, ok)
|
||||
}
|
||||
|
||||
func TestIsNetworkError(t *testing.T) {
|
||||
require.True(t, isNetworkError(errors.New("connect: connection refused")))
|
||||
require.True(t, isNetworkError(errors.New("No connection could be made because the target machine actively refused it.")))
|
||||
require.True(t, isNetworkError(errors.New("unable to open tcp connection with host '127.0.0.1:1433': cause=\"port closed or filtered\"")))
|
||||
require.True(t, isNetworkError(errors.New("network unreachable")))
|
||||
require.True(t, isNetworkError(errors.New("connection reset by peer")))
|
||||
require.True(t, isNetworkError(errors.New("i/o timeout")))
|
||||
require.True(t, isNetworkError(context.Canceled))
|
||||
require.True(t, isNetworkError(context.DeadlineExceeded))
|
||||
require.True(t, isNetworkError(protocolstate.ErrHostDenied.Msgf("127.0.0.1")))
|
||||
|
||||
// Server-side responses / authentication failures are not network errors
|
||||
require.False(t, isNetworkError(nil))
|
||||
require.False(t, isNetworkError(errors.New("mssql: Login failed for user 'sa'")))
|
||||
require.False(t, isNetworkError(errors.New("mssql: Cannot open database \"secret\" requested by the login")))
|
||||
require.False(t, isNetworkError(errors.New("mssql: Password has expired")))
|
||||
require.False(t, isNetworkError(errors.New("mssql: Permission denied to database")))
|
||||
require.False(t, isNetworkError(errors.New("mssql: Password must be reset")))
|
||||
require.False(t, isNetworkError(errors.New("mssql: The query timed out")))
|
||||
}
|
||||
|
||||
func TestExecuteQueryUsesFastdialer(t *testing.T) {
|
||||
execID := "mssql-execute-query-test"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
|
||||
client := &MSSQLClient{}
|
||||
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck // SA1029: executionId string key matches goja runtime injection
|
||||
|
||||
// Denied host should fail policy check immediately
|
||||
_, err := client.ExecuteQuery(ctx, "denied.host", 1433, "user", "pass", "master", "SELECT 1")
|
||||
require.Error(t, err)
|
||||
|
||||
// Closed port should fail
|
||||
_, err = client.ExecuteQuery(ctx, "127.0.0.1", 65530, "user", "pass", "master", "SELECT 1")
|
||||
require.Error(t, err)
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
shared adapter package; not a public JavaScript module
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -20,6 +21,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan/techgraph"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
@@ -62,6 +64,14 @@ type Service struct {
|
||||
techTemplates []*templates.Template
|
||||
ServiceOpts Options
|
||||
hasResults *atomic.Bool
|
||||
|
||||
// tech-graph based selection (automatic scan v2). When selector is nil the
|
||||
// service falls back to the legacy tag-overlap selection.
|
||||
selector *techgraph.Selector
|
||||
tier techgraph.Tier
|
||||
universe map[string]*templates.Template
|
||||
universeErr error
|
||||
universeOnce sync.Once
|
||||
}
|
||||
|
||||
// New takes options and returns a new automatic scan service
|
||||
@@ -103,7 +113,7 @@ func New(opts Options) (*Service, error) {
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not get http client")
|
||||
}
|
||||
return &Service{
|
||||
service := &Service{
|
||||
opts: opts.ExecuterOpts,
|
||||
store: opts.Store,
|
||||
engine: opts.Engine,
|
||||
@@ -115,7 +125,22 @@ func New(opts Options) (*Service, error) {
|
||||
techTemplates: techDetectTemplates,
|
||||
ServiceOpts: opts,
|
||||
hasResults: &atomic.Bool{},
|
||||
}, nil
|
||||
}
|
||||
tier, err := techgraph.ParseTier(opts.ExecuterOpts.Options.AutomaticScanCoverage)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
service.tier = tier
|
||||
|
||||
// load the embedded tech-graph for v2 selection; on failure we silently fall
|
||||
// back to legacy tag-overlap so behaviour never regresses.
|
||||
if graph, err := techgraph.Embedded(); err == nil {
|
||||
service.selector = techgraph.NewSelector(graph)
|
||||
} else {
|
||||
gologger.Warning().Msgf("automatic scan: tech-graph unavailable, using legacy selection: %s", err)
|
||||
}
|
||||
|
||||
return service, nil
|
||||
}
|
||||
|
||||
// Close closes the service
|
||||
@@ -179,7 +204,9 @@ func (s *Service) executeAutomaticScanOnTarget(ctx context.Context, input *conte
|
||||
finalTags = append(finalTags, s.opts.Options.Tags...)
|
||||
finalTags = sliceutil.Dedupe(finalTags)
|
||||
|
||||
if len(finalTags) == 0 {
|
||||
// Lean has no baseline, so an empty signal set has nothing to run. Balanced
|
||||
// and thorough still run the tech-agnostic baseline.
|
||||
if len(finalTags) == 0 && (s.selector == nil || s.tier == techgraph.TierLean) {
|
||||
gologger.Warning().Msgf("Skipping automatic scan since no tags were found on %v\n", input.Input)
|
||||
return
|
||||
}
|
||||
@@ -187,9 +214,19 @@ func (s *Service) executeAutomaticScanOnTarget(ctx context.Context, input *conte
|
||||
gologger.Print().Msgf("Final tags identified for %v: %+v\n", input.Input, finalTags)
|
||||
}
|
||||
|
||||
finalTemplates, err := LoadTemplatesWithTags(s.ServiceOpts, s.templateDirs, finalTags, false)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("%v Error loading templates: %s\n", input.Input, err)
|
||||
var finalTemplates []*templates.Template
|
||||
if s.selector != nil {
|
||||
finalTemplates = s.selectWithGraph(input, finalTags)
|
||||
} else {
|
||||
var err error
|
||||
finalTemplates, err = LoadTemplatesWithTags(s.ServiceOpts, s.templateDirs, finalTags, false)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("%v Error loading templates: %s\n", input.Input, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if len(finalTemplates) == 0 {
|
||||
gologger.Warning().Msgf("Skipping automatic scan since no templates were selected on %v\n", input.Input)
|
||||
return
|
||||
}
|
||||
gologger.Info().Msgf("Executing %d templates on %v", len(finalTemplates), input.Input)
|
||||
@@ -205,12 +242,137 @@ func (s *Service) executeAutomaticScanOnTarget(ctx context.Context, input *conte
|
||||
s.hasResults.Store(tmp.Load())
|
||||
}
|
||||
|
||||
// ensureUniverse lazily loads every selectable template once and indexes it by
|
||||
// id. This is the pool from which graph-based selection draws (precise + fuzzy),
|
||||
// avoiding a per-target loader pass.
|
||||
func (s *Service) ensureUniverse() (map[string]*templates.Template, error) {
|
||||
s.universeOnce.Do(func() {
|
||||
all, err := s.store.LoadTemplates(s.templateDirs)
|
||||
if err != nil {
|
||||
s.universeErr = err
|
||||
return
|
||||
}
|
||||
s.universe = make(map[string]*templates.Template, len(all))
|
||||
for _, t := range all {
|
||||
if t != nil {
|
||||
s.universe[t.ID] = t
|
||||
}
|
||||
}
|
||||
})
|
||||
return s.universe, s.universeErr
|
||||
}
|
||||
|
||||
// selectWithGraph resolves detected signals to a precise fire-set via the
|
||||
// tech-graph, augmented (except at the lean tier) by legacy tag-overlap for any
|
||||
// signals not present in the graph, so recall never falls below legacy behaviour.
|
||||
func (s *Service) selectWithGraph(input *contextargs.MetaInput, finalTags []string) []*templates.Template {
|
||||
universe, err := s.ensureUniverse()
|
||||
if err != nil || len(universe) == 0 {
|
||||
if len(finalTags) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("automatic scan: template load failed, using tag selection: %s", err)
|
||||
}
|
||||
tpls, loadErr := LoadTemplatesWithTags(s.ServiceOpts, s.templateDirs, finalTags, false)
|
||||
if loadErr != nil {
|
||||
gologger.Error().Msgf("%v Error loading templates: %s\n", input.Input, loadErr)
|
||||
return nil
|
||||
}
|
||||
return tpls
|
||||
}
|
||||
|
||||
techIDs, unresolved := s.selector.Resolve(finalTags)
|
||||
selected := map[string]*templates.Template{}
|
||||
|
||||
graph := 0
|
||||
for _, id := range s.selector.FireSet(s.tier, techIDs) {
|
||||
if t, ok := universe[id]; ok {
|
||||
selected[id] = t
|
||||
graph++
|
||||
}
|
||||
}
|
||||
|
||||
added := 0
|
||||
if s.tier != techgraph.TierLean && len(techIDs) > 0 {
|
||||
candidates := make([]techgraph.Candidate, 0, len(universe))
|
||||
for id, t := range universe {
|
||||
if _, ok := selected[id]; ok {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, techgraph.Candidate{
|
||||
ID: id,
|
||||
Tags: t.Info.Tags.ToSlice(),
|
||||
Category: categoryFromPath(t.Path),
|
||||
})
|
||||
}
|
||||
for _, id := range s.selector.AbsentDependents(techIDs, candidates) {
|
||||
if t, ok := universe[id]; ok {
|
||||
selected[id] = t
|
||||
added++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fuzzy := 0
|
||||
if s.tier != techgraph.TierLean && len(unresolved) > 0 {
|
||||
want := map[string]struct{}{}
|
||||
for _, tag := range unresolved {
|
||||
want[strings.ToLower(tag)] = struct{}{}
|
||||
}
|
||||
for id, t := range universe {
|
||||
if _, ok := selected[id]; ok {
|
||||
continue
|
||||
}
|
||||
for _, tag := range t.Info.Tags.ToSlice() {
|
||||
if _, ok := want[strings.ToLower(tag)]; ok {
|
||||
selected[id] = t
|
||||
fuzzy++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]*templates.Template, 0, len(selected))
|
||||
for _, t := range selected {
|
||||
out = append(out, t)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
|
||||
|
||||
if s.opts.Options.VerboseVerbose {
|
||||
gologger.Print().Msgf("%v selection [tier=%s]: %d techs, %d graph + %d new + %d fuzzy = %d templates\n",
|
||||
input.Input, s.tier, len(techIDs), graph, added, fuzzy, len(out))
|
||||
}
|
||||
|
||||
if !s.opts.Options.DisableClustering {
|
||||
out, _, _ = templates.ClusterTemplates(out, s.opts)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type sharedProgress struct {
|
||||
progress.Progress
|
||||
}
|
||||
|
||||
func (p *sharedProgress) Init(hostCount int64, rulesCount int, requestCount int64) {}
|
||||
|
||||
// categoryFromPath returns the nuclei category directory for a template path
|
||||
// (http/cves/2024/x.yaml -> cves). An empty result means the category is unknown
|
||||
// and the template is not eligible for the post-graph supplement.
|
||||
func categoryFromPath(path string) string {
|
||||
parts := strings.Split(filepath.ToSlash(path), "/")
|
||||
for i, part := range parts {
|
||||
switch part {
|
||||
case "http", "network", "dns", "ssl", "file", "headless", "javascript", "code", "cloud", "dast":
|
||||
if i+1 < len(parts) {
|
||||
return parts[i+1]
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// getTagsUsingWappalyzer returns tags using wappalyzer by fingerprinting target
|
||||
// and utilizing the mapping data
|
||||
func (s *Service) getTagsUsingWappalyzer(input *contextargs.MetaInput) []string {
|
||||
|
||||
@@ -13,3 +13,9 @@ func TestNormalizeAppName(t *testing.T) {
|
||||
appName = normalizeAppName("JBoss:2.3.5")
|
||||
require.Equal(t, "jboss", appName, "could not get normalized name")
|
||||
}
|
||||
|
||||
func TestCategoryFromPath(t *testing.T) {
|
||||
require.Equal(t, "cves", categoryFromPath("/home/templates/http/cves/2024/CVE-2024-1.yaml"))
|
||||
require.Equal(t, "fuzzing", categoryFromPath("http/fuzzing/xss.yaml"))
|
||||
require.Equal(t, "", categoryFromPath("README.md"))
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package techgraph
|
||||
|
||||
import "strings"
|
||||
|
||||
// excludedCategories never participate in automatic scan (opt-in/specialized).
|
||||
var excludedCategories = map[string]struct{}{
|
||||
"fuzzing": {},
|
||||
"osint": {},
|
||||
"token-spray": {},
|
||||
"malware": {},
|
||||
"cloud": {},
|
||||
}
|
||||
|
||||
// baselineCategories are the only categories whose product-less templates may be
|
||||
// considered for the tech-agnostic baseline. Product-specific categories (cves,
|
||||
// vulnerabilities, exposed-panels, default-logins, cnvd) are intentionally absent:
|
||||
// an unmappable template there must go to unmapped (fallback), never baseline.
|
||||
var baselineCategories = map[string]struct{}{
|
||||
"exposures": {},
|
||||
"misconfiguration": {},
|
||||
"miscellaneous": {},
|
||||
"iot": {},
|
||||
"takeovers": {}, // subdomain takeover signatures are run host-agnostically
|
||||
}
|
||||
|
||||
// genericBaselineTags positively identify a tech-agnostic check.
|
||||
var genericBaselineTags = map[string]struct{}{
|
||||
"cors": {}, "crlf": {}, "open-redirect": {}, "ds_store": {}, "dsstore": {},
|
||||
"phpinfo": {}, "directory-listing": {}, "listing": {}, "debug": {},
|
||||
"robots": {}, "sitemap": {}, "git": {}, "svn": {}, "dotenv": {},
|
||||
"host-header-injection": {}, "http-trace": {}, "options-method": {},
|
||||
"well-known": {}, "trace": {},
|
||||
}
|
||||
|
||||
// genericBaselineIDSubstrings positively identify generic checks by id.
|
||||
var genericBaselineIDSubstrings = []string{
|
||||
"git-config", "gitignore", "git-credentials", "svn-", ".env", "dotenv",
|
||||
"phpinfo", "directory-listing", "dir-listing", "ds-store", "ds_store",
|
||||
"robots-txt", "sitemap-", "cors-", "crlf-", "open-redirect", "trace-method",
|
||||
"options-method", "well-known", "backup-files", "sql-dump",
|
||||
}
|
||||
|
||||
// classifyBaseline decides whether an otherwise-unmapped template is a genuinely
|
||||
// generic, tech-agnostic baseline check. It is a positive allowlist: anything not
|
||||
// recognised as generic is left for the caller to mark unmapped.
|
||||
func classifyBaseline(info templateInfo) (Tier, bool) {
|
||||
// tech-agnostic "generic" buckets (e.g. vulnerabilities/generic/* holds
|
||||
// cors/sqli/xss probes that apply to any target) qualify regardless of
|
||||
// category. The logs/files/backups subdir heuristic was removed: it leaked
|
||||
// product-specific exposures (bitrix, magento, oracle-ebs, ...) into baseline.
|
||||
if info.Subdir == "generic" {
|
||||
return TierThorough, true
|
||||
}
|
||||
if _, ok := baselineCategories[info.Category]; !ok {
|
||||
return "", false
|
||||
}
|
||||
// takeovers are wholesale host-agnostic.
|
||||
if info.Category == "takeovers" {
|
||||
return TierBalanced, true
|
||||
}
|
||||
for _, tag := range info.Tags {
|
||||
if _, ok := genericBaselineTags[normToken(tag)]; ok {
|
||||
return TierBalanced, true
|
||||
}
|
||||
}
|
||||
idl := strings.ToLower(info.ID)
|
||||
for _, sub := range genericBaselineIDSubstrings {
|
||||
if strings.Contains(idl, sub) {
|
||||
return TierBalanced, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
package techgraph
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Build walks the templates root and produces the tech-graph artifact.
|
||||
func Build(root string) (*Graph, error) {
|
||||
root = filepath.Clean(root)
|
||||
g := &Graph{
|
||||
Version: ArtifactVersion,
|
||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
Techs: map[string]*Tech{},
|
||||
Synonyms: map[string]string{},
|
||||
}
|
||||
|
||||
infos, hash, parseErr, err := collect(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g.Stats.ParseError = parseErr
|
||||
g.SourceHash = fmt.Sprintf("sha256:%x", hash)
|
||||
|
||||
var deferred []templateInfo
|
||||
|
||||
// Pass 1: build tech nodes from authoritative signals (detection anchors,
|
||||
// cpe/product, platform). Everything else is deferred so it can be resolved
|
||||
// against the fully-built node set in pass 2.
|
||||
for _, info := range infos {
|
||||
g.Stats.Total++
|
||||
|
||||
// Excluded categories never enter detection or the fire-set. Gating
|
||||
// this before detection classification keeps opt-in categories such as
|
||||
// cloud out of the automatic-scan detector list.
|
||||
if _, ok := excludedCategories[info.Category]; ok {
|
||||
g.Stats.Excluded++
|
||||
continue
|
||||
}
|
||||
if isDetection(info) && isDetectionSeverity(info) {
|
||||
id := techID(info.Vendor, info.Product)
|
||||
if id != "" {
|
||||
g.anchorTech(id, info)
|
||||
}
|
||||
g.Detection = append(g.Detection, DetectionEntry{ID: info.ID, Tech: id})
|
||||
g.Stats.Detection++
|
||||
continue
|
||||
}
|
||||
if p, ok := detectPlatform(info); ok {
|
||||
g.addDependent(p, info, "platform")
|
||||
g.Stats.Dependents++
|
||||
g.Stats.DependentsPlat++
|
||||
continue
|
||||
}
|
||||
if id := techID(info.Vendor, info.Product); id != "" {
|
||||
g.addDependent(id, info, "cpe")
|
||||
g.Stats.Dependents++
|
||||
g.Stats.DependentsCPE++
|
||||
continue
|
||||
}
|
||||
deferred = append(deferred, info)
|
||||
}
|
||||
|
||||
// Pass 2: resolve deferred templates to a known tech BEFORE considering
|
||||
// baseline. Order: directory grouping -> tag -> id token. Only genuinely
|
||||
// generic leftovers become baseline; the rest are unmapped (runtime fallback).
|
||||
aliasIdx := g.buildAliasIndex()
|
||||
for _, info := range deferred {
|
||||
if id, ok := g.resolveToken(info.Subdir, aliasIdx); ok {
|
||||
g.addDependent(id, info, "dir")
|
||||
g.Stats.Dependents++
|
||||
g.Stats.DependentsDir++
|
||||
continue
|
||||
}
|
||||
if id, ok := reconcileByTag(g, info, aliasIdx); ok {
|
||||
g.addDependent(id, info, "tag")
|
||||
g.Stats.Dependents++
|
||||
g.Stats.DependentsTag++
|
||||
continue
|
||||
}
|
||||
if id, ok := reconcileByID(g, info, aliasIdx); ok {
|
||||
g.addDependent(id, info, "id")
|
||||
g.Stats.Dependents++
|
||||
g.Stats.DependentsID++
|
||||
continue
|
||||
}
|
||||
if tier, ok := classifyBaseline(info); ok {
|
||||
g.Baseline = append(g.Baseline, BaselineEntry{
|
||||
ID: info.ID, Category: info.Category, Severity: info.Severity, Tier: tier,
|
||||
})
|
||||
g.Stats.Baseline++
|
||||
continue
|
||||
}
|
||||
g.Stats.Unmapped++
|
||||
}
|
||||
|
||||
// ship validated seed synonyms in the artifact so the runtime selector (which
|
||||
// only reads the artifact) can resolve acronyms like aem -> adobe:experience_manager.
|
||||
for alias, id := range seedSynonyms {
|
||||
if _, ok := g.Techs[id]; ok {
|
||||
g.Synonyms[alias] = id
|
||||
}
|
||||
}
|
||||
|
||||
g.Stats.Techs = len(g.Techs)
|
||||
g.sortStable()
|
||||
return g, nil
|
||||
}
|
||||
|
||||
// collect walks the tree once and returns parsed infos plus a content hash.
|
||||
func collect(root string) ([]templateInfo, []byte, int, error) {
|
||||
var infos []templateInfo
|
||||
var parseErr int
|
||||
hasher := sha256.New()
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() {
|
||||
if skipDir(d.Name()) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(path, ".yaml") && !strings.HasSuffix(path, ".yml") {
|
||||
return nil
|
||||
}
|
||||
info, ok := extractTemplate(path, root)
|
||||
if !ok {
|
||||
parseErr++
|
||||
return nil
|
||||
}
|
||||
if _, err := fmt.Fprintf(hasher, "%s|%s|%s|%s\n", info.ID, info.Vendor, info.Product, info.CPE); err != nil {
|
||||
return err
|
||||
}
|
||||
infos = append(infos, info)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
sort.Slice(infos, func(i, j int) bool { return infos[i].ID < infos[j].ID })
|
||||
return infos, hasher.Sum(nil), parseErr, nil
|
||||
}
|
||||
|
||||
// anchorTech creates/updates a tech node's metadata without adding a fireable
|
||||
// template (used by detector templates that define a tech).
|
||||
func (g *Graph) anchorTech(id string, info templateInfo) *Tech {
|
||||
tech, ok := g.Techs[id]
|
||||
if !ok {
|
||||
tech = &Tech{ID: id, Vendor: normToken(info.Vendor), Product: normToken(info.Product), CPE: info.CPE}
|
||||
g.Techs[id] = tech
|
||||
}
|
||||
if tech.CPE == "" && info.CPE != "" {
|
||||
tech.CPE = info.CPE
|
||||
}
|
||||
mergeAliases(tech, aliasesFor(info.Vendor, info.Product, info.Tags))
|
||||
for _, a := range tech.Aliases {
|
||||
if existing, ok := g.Synonyms[a]; !ok || existing == id {
|
||||
g.Synonyms[a] = id
|
||||
}
|
||||
}
|
||||
return tech
|
||||
}
|
||||
|
||||
// addDependent registers a fireable template under its canonical tech.
|
||||
//
|
||||
// Only "cpe" attachments contribute aliases (the template's own vendor/product
|
||||
// genuinely define the tech). Platform/dir/tag/id attachments must NOT pollute
|
||||
// the node's aliases with the sub-product's tokens (e.g. a wp-plugin's name must
|
||||
// not become a wordpress alias), which would corrupt reconciliation.
|
||||
func (g *Graph) addDependent(id string, info templateInfo, source string) {
|
||||
var tech *Tech
|
||||
if source == "cpe" {
|
||||
tech = g.anchorTech(id, info)
|
||||
} else {
|
||||
tech = g.ensureTech(id, info)
|
||||
}
|
||||
ref := TemplateRef{ID: info.ID, Category: info.Category, Severity: info.Severity, Source: source}
|
||||
if source == "platform" || source == "dir" {
|
||||
ref.Product = normToken(info.Product)
|
||||
}
|
||||
tech.Templates = append(tech.Templates, ref)
|
||||
}
|
||||
|
||||
// ensureTech returns an existing node, or creates a bare one without merging the
|
||||
// template's aliases (used for non-cpe attachments).
|
||||
func (g *Graph) ensureTech(id string, info templateInfo) *Tech {
|
||||
if tech, ok := g.Techs[id]; ok {
|
||||
return tech
|
||||
}
|
||||
vendor, product := "", ""
|
||||
if v, p, ok := splitTechID(id); ok {
|
||||
vendor, product = v, p
|
||||
}
|
||||
tech := &Tech{ID: id, Vendor: vendor, Product: product}
|
||||
g.Techs[id] = tech
|
||||
return tech
|
||||
}
|
||||
|
||||
// splitTechID reverses techID for bare node creation.
|
||||
func splitTechID(id string) (vendor, product string, ok bool) {
|
||||
if i := strings.IndexByte(id, ':'); i > 0 {
|
||||
return id[:i], id[i+1:], true
|
||||
}
|
||||
return "", id, true
|
||||
}
|
||||
|
||||
// buildAliasIndex maps a single alias/product token to a unique tech id. Tokens
|
||||
// that resolve to more than one tech, are too short, or are generic are dropped
|
||||
// so tag reconciliation stays conservative.
|
||||
func (g *Graph) buildAliasIndex() map[string]string {
|
||||
counts := map[string]map[string]struct{}{}
|
||||
add := func(tok, id string) {
|
||||
tok = normToken(tok)
|
||||
if len(tok) < 5 || genericTokens[tok] {
|
||||
return
|
||||
}
|
||||
if counts[tok] == nil {
|
||||
counts[tok] = map[string]struct{}{}
|
||||
}
|
||||
counts[tok][id] = struct{}{}
|
||||
}
|
||||
for id, tech := range g.Techs {
|
||||
add(id, id)
|
||||
add(tech.Product, id)
|
||||
for _, a := range tech.Aliases {
|
||||
add(a, id)
|
||||
}
|
||||
}
|
||||
idx := map[string]string{}
|
||||
for tok, ids := range counts {
|
||||
if len(ids) == 1 {
|
||||
for id := range ids {
|
||||
idx[tok] = id
|
||||
}
|
||||
}
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
// resolveToken maps a single token to a tech id via the seed synonym table
|
||||
// first (validated against existing nodes), then the alias index.
|
||||
func (g *Graph) resolveToken(tok string, idx map[string]string) (string, bool) {
|
||||
tok = normToken(tok)
|
||||
if tok == "" || genericTokens[tok] {
|
||||
return "", false
|
||||
}
|
||||
if id, ok := seedSynonyms[tok]; ok {
|
||||
if _, exists := g.Techs[id]; exists {
|
||||
return id, true
|
||||
}
|
||||
}
|
||||
if id, ok := idx[tok]; ok {
|
||||
return id, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// reconcileByTag attaches a product-less template to a known tech when its tags
|
||||
// resolve to exactly one unique tech node.
|
||||
func reconcileByTag(g *Graph, info templateInfo, idx map[string]string) (string, bool) {
|
||||
matches := map[string]struct{}{}
|
||||
for _, tag := range info.Tags {
|
||||
if id, ok := g.resolveToken(tag, idx); ok {
|
||||
matches[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(matches) == 1 {
|
||||
for id := range matches {
|
||||
return id, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// reconcileByID attaches a template when its id tokens resolve to exactly one
|
||||
// unique tech node (e.g. "aem-jcr-exposure" -> adobe:experience_manager).
|
||||
//
|
||||
// This is the riskiest signal (id tokens are common English words), so it is
|
||||
// guarded: a token must be >=6 chars AND also appear in the template name, which
|
||||
// requires the product to be the template's actual subject rather than an
|
||||
// incidental id suffix like "-dashboard"/"-gateway".
|
||||
func reconcileByID(g *Graph, info templateInfo, idx map[string]string) (string, bool) {
|
||||
name := strings.ToLower(info.Name)
|
||||
matches := map[string]struct{}{}
|
||||
for _, tok := range strings.Split(strings.ToLower(info.ID), "-") {
|
||||
if len(tok) < 6 || !strings.Contains(name, tok) {
|
||||
continue
|
||||
}
|
||||
if id, ok := g.resolveToken(tok, idx); ok {
|
||||
matches[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(matches) == 1 {
|
||||
for id := range matches {
|
||||
return id, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func mergeAliases(tech *Tech, aliases []string) {
|
||||
have := map[string]struct{}{}
|
||||
for _, a := range tech.Aliases {
|
||||
have[a] = struct{}{}
|
||||
}
|
||||
for _, a := range aliases {
|
||||
if a == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := have[a]; ok {
|
||||
continue
|
||||
}
|
||||
have[a] = struct{}{}
|
||||
tech.Aliases = append(tech.Aliases, a)
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Graph) sortStable() {
|
||||
for _, tech := range g.Techs {
|
||||
sort.Strings(tech.Aliases)
|
||||
sort.Slice(tech.Templates, func(i, j int) bool {
|
||||
return tech.Templates[i].ID < tech.Templates[j].ID
|
||||
})
|
||||
}
|
||||
sort.Slice(g.Baseline, func(i, j int) bool { return g.Baseline[i].ID < g.Baseline[j].ID })
|
||||
sort.Slice(g.Detection, func(i, j int) bool { return g.Detection[i].ID < g.Detection[j].ID })
|
||||
}
|
||||
|
||||
// WriteFile serialises the graph as indented JSON.
|
||||
func (g *Graph) WriteFile(path string) error {
|
||||
data, err := json.MarshalIndent(g, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, append(data, '\n'), 0o644)
|
||||
}
|
||||
|
||||
func skipDir(name string) bool {
|
||||
switch name {
|
||||
case ".git", ".github", "helpers", "profiles", "workflows":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package techgraph
|
||||
|
||||
import "strings"
|
||||
|
||||
// seedSynonyms maps common acronyms/aliases to a canonical tech id. This is the
|
||||
// curated seed of the synonym table that the offline AI step later expands.
|
||||
// Targets are validated against existing nodes at resolve time, so a stale entry
|
||||
// simply has no effect.
|
||||
var seedSynonyms = map[string]string{
|
||||
"aem": "adobe:experience_manager",
|
||||
"coldfusion": "adobe:coldfusion",
|
||||
"iis": "microsoft:iis",
|
||||
"k8s": "kubernetes:kubernetes",
|
||||
"springboot": "vmware:spring_boot",
|
||||
"spring-boot": "vmware:spring_boot",
|
||||
"wls": "oracle:weblogic_server",
|
||||
"weblogic": "oracle:weblogic_server",
|
||||
"adfs": "microsoft:active_directory_federation_services",
|
||||
// platform roots: ambiguous in the alias index (collide with sub-product
|
||||
// nodes like keydatas:wordpress) so pin them to the canonical platform node.
|
||||
"wordpress": "wordpress",
|
||||
"drupal": "drupal",
|
||||
"joomla": "joomla",
|
||||
"magento": "magento",
|
||||
}
|
||||
|
||||
// genericTokens are tags/aliases that are not products and must never be used as
|
||||
// a tag/id/dir-reconciliation key (they would incorrectly attach templates to random
|
||||
// techs). This list intentionally includes common English words that also happen
|
||||
// to be CPE product names (dashboard, gateway, console, ...), platform/OS tokens
|
||||
// (linux, windows, android, ...), and category labels (generic, audit, ...),
|
||||
// since those were the dominant source of false attachments.
|
||||
var genericTokens = map[string]bool{
|
||||
// vuln classes / scan vocabulary
|
||||
"cve": true, "cves": true, "vuln": true, "vulnerability": true, "tech": true,
|
||||
"detect": true, "detection": true, "panel": true, "panels": true, "login": true, "cms": true,
|
||||
"exposure": true, "exposures": true, "misconfig": true, "misconfiguration": true,
|
||||
"rce": true, "lfi": true, "xss": true, "sqli": true, "ssrf": true, "ssti": true,
|
||||
"redirect": true, "disclosure": true, "default": true, "auth": true, "bypass": true,
|
||||
"injection": true, "traversal": true, "config": true, "files": true, "file": true,
|
||||
"backup": true, "takeover": true, "oast": true, "intrusive": true, "unauth": true,
|
||||
"router": true, "iot": true, "network": true, "http": true, "json": true, "api": true,
|
||||
"server": true, "service": true, "framework": true, "plugin": true, "theme": true,
|
||||
"webserver": true, "database": true, "cloud": true, "devops": true, "tools": true,
|
||||
"edb": true, "kev": true, "packetstorm": true, "seclists": true, "osint": true,
|
||||
// category labels
|
||||
"generic": true, "audit": true, "miscellaneous": true, "misc": true, "enumeration": true,
|
||||
"vulnerabilities": true, "exposed": true, "keys": true, "secret": true, "secrets": true,
|
||||
// platform / OS / cloud-provider tokens (not products)
|
||||
"linux": true, "windows": true, "macos": true, "unix": true, "android": true, "ios": true,
|
||||
"aws": true, "azure": true, "gcp": true, "alibaba": true, "kubernetes": true, "k8s": true,
|
||||
"docker": true, "mobile": true,
|
||||
// common English words that collide with product names
|
||||
"dashboard": true, "gateway": true, "path": true, "console": true, "firewall": true,
|
||||
"monitor": true, "control": true, "permissions": true, "oauth": true, "analytics": true,
|
||||
"enterprise": true, "report": true, "reports": true, "help": true, "printer": true,
|
||||
"core": true, "commerce": true, "anchor": true, "nova": true, "element": true,
|
||||
"webclient": true, "memcached": true, "manager": true, "management": true, "admin": true,
|
||||
"client": true, "agent": true, "portal": true, "gateway2": true, "status": true,
|
||||
"user": true, "users": true, "account": true, "token": true, "tokens": true,
|
||||
"web": true, "app": true, "application": true, "system": true, "platform": true,
|
||||
}
|
||||
|
||||
// normToken lowercases and trims a vendor/product token. CPE uses '_' for
|
||||
// spaces; we keep it as-is so the id stays stable and machine-joinable.
|
||||
func normToken(s string) string {
|
||||
s = strings.ToLower(strings.TrimSpace(s))
|
||||
// CPE wildcards / NA markers are not real tokens.
|
||||
if s == "*" || s == "-" {
|
||||
return ""
|
||||
}
|
||||
// URLs / paths / multi-word strings are not clean product tokens (guards
|
||||
// against junk nodes like "https://apisix.apache.org").
|
||||
if strings.ContainsAny(s, "/ ") {
|
||||
return ""
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// techID builds the canonical technology id from vendor/product. When vendor is
|
||||
// empty or equal to product we collapse to a single token to avoid duplicate
|
||||
// nodes like "wordpress:wordpress" vs "wordpress".
|
||||
func techID(vendor, product string) string {
|
||||
vendor = normToken(vendor)
|
||||
product = normToken(product)
|
||||
switch {
|
||||
case product == "":
|
||||
return ""
|
||||
case vendor == "" || vendor == product:
|
||||
return product
|
||||
default:
|
||||
return vendor + ":" + product
|
||||
}
|
||||
}
|
||||
|
||||
// aliasesFor returns alias tokens that runtime detection (wappalyzer names,
|
||||
// tags) might surface for a tech, so they can be reconciled to the canonical id.
|
||||
// Pillar 1 derives these from product/vendor tokens; AI augments this offline.
|
||||
func aliasesFor(vendor, product string, tags []string) []string {
|
||||
seen := map[string]struct{}{}
|
||||
var out []string
|
||||
add := func(s string) {
|
||||
s = normToken(s)
|
||||
if s == "" {
|
||||
return
|
||||
}
|
||||
if _, ok := seen[s]; ok {
|
||||
return
|
||||
}
|
||||
seen[s] = struct{}{}
|
||||
out = append(out, s)
|
||||
}
|
||||
// Only whole product/vendor tokens are safe aliases. Splitting on separators
|
||||
// produces ambiguous fragments (manager, server, token, web, ...) that cause
|
||||
// false tag/id reconciliation, so we deliberately avoid it.
|
||||
add(product)
|
||||
if vendor != product {
|
||||
add(vendor)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
// Command techgraph-gen builds the offline tech-graph.json artifact from a local
|
||||
// nuclei-templates checkout. See pkg/protocols/common/automaticscan/AUTOMATIC_SCAN_V2.md.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan/techgraph"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var (
|
||||
templates = flag.String("templates", "", "path to nuclei-templates directory (required)")
|
||||
output = flag.String("output", "tech-graph.json", "output artifact path")
|
||||
htmlOut = flag.String("html", "", "also write a standalone visualizer HTML with data inlined")
|
||||
)
|
||||
flag.Parse()
|
||||
|
||||
if *templates == "" {
|
||||
fmt.Fprintln(os.Stderr, "error: -templates is required")
|
||||
flag.Usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
graph, err := techgraph.Build(*templates)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: build failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := graph.WriteFile(*output); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: write failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if *htmlOut != "" {
|
||||
html, err := techgraph.StandaloneHTML(graph)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: html render failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.WriteFile(*htmlOut, []byte(html), 0o644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: html write failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("standalone visualizer written to %s\n", *htmlOut)
|
||||
}
|
||||
|
||||
s := graph.Stats
|
||||
fmt.Printf("tech-graph written to %s\n", *output)
|
||||
fmt.Printf(" templates: %d (parse-skipped %d)\n", s.Total, s.ParseError)
|
||||
fmt.Printf(" dependents: %d (cpe %d, platform %d, dir %d, tag %d, id %d) across %d techs\n",
|
||||
s.Dependents, s.DependentsCPE, s.DependentsPlat, s.DependentsDir, s.DependentsTag, s.DependentsID, s.Techs)
|
||||
fmt.Printf(" baseline: %d\n", s.Baseline)
|
||||
fmt.Printf(" detection: %d\n", s.Detection)
|
||||
fmt.Printf(" excluded: %d\n", s.Excluded)
|
||||
fmt.Printf(" unmapped: %d\n", s.Unmapped)
|
||||
fmt.Printf(" source: %s\n", graph.SourceHash)
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package techgraph
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// embeddedGraph is the prebuilt tech-graph artifact shipped with nuclei. Run the
|
||||
// techgraph-gen command to regenerate it against a nuclei-templates checkout.
|
||||
//
|
||||
//go:embed tech-graph.json
|
||||
var embeddedGraph []byte
|
||||
|
||||
//go:embed visualizer.html
|
||||
var visualizerHTML string
|
||||
|
||||
// Embedded returns the tech-graph artifact compiled into the binary.
|
||||
func Embedded() (*Graph, error) {
|
||||
g := &Graph{}
|
||||
if err := json.Unmarshal(embeddedGraph, g); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
|
||||
// EmbeddedRaw returns the raw embedded artifact bytes.
|
||||
func EmbeddedRaw() []byte { return embeddedGraph }
|
||||
|
||||
// StandaloneHTML returns the visualizer with the given graph inlined, so it can
|
||||
// be opened directly in a browser without a server or sidecar file.
|
||||
func StandaloneHTML(g *Graph) (string, error) {
|
||||
data, err := json.Marshal(g)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
inject := "<script>window.__TECHGRAPH__=" + string(data) + ";</script>\n</head>"
|
||||
return strings.Replace(visualizerHTML, "</head>", inject, 1), nil
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package techgraph
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// templateInfo is the subset of a nuclei template we need for graph building.
|
||||
// We deliberately avoid the full template parser: only the info block is read,
|
||||
// which keeps the generator fast and dependency-light over ~13k files.
|
||||
type templateInfo struct {
|
||||
ID string
|
||||
Name string
|
||||
Severity string
|
||||
Category string // derived from directory (cves, exposed-panels, ...)
|
||||
Subdir string // directory under the category (e.g. "aem" in misconfiguration/aem/)
|
||||
Tags []string
|
||||
Vendor string
|
||||
Product string
|
||||
CPE string
|
||||
}
|
||||
|
||||
// rawTemplate mirrors the YAML structure for lenient decoding.
|
||||
type rawTemplate struct {
|
||||
ID string `yaml:"id"`
|
||||
Info struct {
|
||||
Name string `yaml:"name"`
|
||||
Severity string `yaml:"severity"`
|
||||
Tags any `yaml:"tags"`
|
||||
Metadata map[string]any `yaml:"metadata"`
|
||||
Classification struct {
|
||||
CPE string `yaml:"cpe"`
|
||||
} `yaml:"classification"`
|
||||
} `yaml:"info"`
|
||||
}
|
||||
|
||||
// extractTemplate reads a single template file and returns its info subset.
|
||||
// ok is false when the file is not a parseable template (e.g. workflow, helper).
|
||||
func extractTemplate(path, root string) (templateInfo, bool) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return templateInfo{}, false
|
||||
}
|
||||
var raw rawTemplate
|
||||
if err := yaml.Unmarshal(data, &raw); err != nil {
|
||||
return templateInfo{}, false
|
||||
}
|
||||
if raw.ID == "" {
|
||||
return templateInfo{}, false
|
||||
}
|
||||
|
||||
category, subdir := derivePath(path, root)
|
||||
info := templateInfo{
|
||||
ID: raw.ID,
|
||||
Name: raw.Info.Name,
|
||||
Severity: strings.ToLower(strings.TrimSpace(raw.Info.Severity)),
|
||||
Category: category,
|
||||
Subdir: subdir,
|
||||
Tags: coerceStringSlice(raw.Info.Tags),
|
||||
CPE: strings.TrimSpace(raw.Info.Classification.CPE),
|
||||
}
|
||||
|
||||
if raw.Info.Metadata != nil {
|
||||
info.Vendor = firstNonEmpty(coerceStringSlice(raw.Info.Metadata["vendor"]))
|
||||
info.Product = firstNonEmpty(coerceStringSlice(raw.Info.Metadata["product"]))
|
||||
}
|
||||
|
||||
// Prefer CPE for vendor/product when metadata is missing.
|
||||
if cv, cp, ok := parseCPE(info.CPE); ok {
|
||||
if info.Vendor == "" {
|
||||
info.Vendor = cv
|
||||
}
|
||||
if info.Product == "" {
|
||||
info.Product = cp
|
||||
}
|
||||
}
|
||||
return info, true
|
||||
}
|
||||
|
||||
// derivePath returns the nuclei category and the immediate subdirectory under
|
||||
// it, e.g. http/misconfiguration/aem/x.yaml -> ("misconfiguration", "aem").
|
||||
// The subdir is often a product grouping and is used as a tech signal.
|
||||
func derivePath(path, root string) (category, subdir string) {
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
rel = path
|
||||
}
|
||||
parts := strings.Split(filepath.ToSlash(rel), "/")
|
||||
switch {
|
||||
case len(parts) >= 4:
|
||||
// protocol/category/subdir/.../file
|
||||
return parts[1], parts[2]
|
||||
case len(parts) == 3:
|
||||
return parts[1], ""
|
||||
case len(parts) == 2:
|
||||
return parts[0], ""
|
||||
default:
|
||||
return "", ""
|
||||
}
|
||||
}
|
||||
|
||||
// parseCPE extracts vendor and product from CPE 2.3 (cpe:2.3:a:vendor:product:..)
|
||||
// or CPE 2.2 URI (cpe:/a:vendor:product:..) forms.
|
||||
func parseCPE(cpe string) (vendor, product string, ok bool) {
|
||||
cpe = strings.TrimSpace(cpe)
|
||||
if cpe == "" {
|
||||
return "", "", false
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(cpe, "cpe:2.3:"):
|
||||
parts := strings.Split(cpe, ":")
|
||||
// cpe:2.3:part:vendor:product:...
|
||||
if len(parts) >= 5 {
|
||||
return normToken(parts[3]), normToken(parts[4]), true
|
||||
}
|
||||
case strings.HasPrefix(cpe, "cpe:/"):
|
||||
parts := strings.Split(strings.TrimPrefix(cpe, "cpe:/"), ":")
|
||||
// part:vendor:product:...
|
||||
if len(parts) >= 3 {
|
||||
return normToken(parts[1]), normToken(parts[2]), true
|
||||
}
|
||||
}
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
// coerceStringSlice normalises tags/metadata values that may be a string, a
|
||||
// comma-separated string, or a YAML list into a clean slice.
|
||||
func coerceStringSlice(v any) []string {
|
||||
var out []string
|
||||
add := func(s string) {
|
||||
for _, part := range strings.Split(s, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
out = append(out, part)
|
||||
}
|
||||
}
|
||||
}
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
add(t)
|
||||
case []any:
|
||||
for _, item := range t {
|
||||
if s, ok := item.(string); ok {
|
||||
add(s)
|
||||
}
|
||||
}
|
||||
case []string:
|
||||
for _, s := range t {
|
||||
add(s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func firstNonEmpty(values []string) string {
|
||||
for _, v := range values {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package techgraph
|
||||
|
||||
// platforms are technologies with large plugin/extension ecosystems whose
|
||||
// add-on templates carry the add-on as `product` (often with the platform as
|
||||
// `vendor` or only as a tag). Runtime detection sees the platform, not each
|
||||
// plugin, so those templates must be reachable from the platform node.
|
||||
var platforms = map[string]string{
|
||||
"wordpress": "wordpress",
|
||||
"joomla": "joomla",
|
||||
"drupal": "drupal",
|
||||
"magento": "magento",
|
||||
"prestashop": "prestashop",
|
||||
"opencart": "opencart",
|
||||
"moodle": "moodle",
|
||||
"typo3": "typo3",
|
||||
"dotnetnuke": "dotnetnuke",
|
||||
"concrete5": "concrete5",
|
||||
"shopware": "shopware",
|
||||
"vbulletin": "vbulletin",
|
||||
"mybb": "mybb",
|
||||
"phpbb": "phpbb",
|
||||
}
|
||||
|
||||
// platformTagAliases maps secondary tag tokens to a platform key.
|
||||
var platformTagAliases = map[string]string{
|
||||
"wp": "wordpress",
|
||||
"wp-plugin": "wordpress",
|
||||
"wp-theme": "wordpress",
|
||||
}
|
||||
|
||||
// detectPlatform returns the platform node id when a template belongs to a
|
||||
// platform's add-on ecosystem (and is not the platform core itself).
|
||||
func detectPlatform(info templateInfo) (string, bool) {
|
||||
vendor := normToken(info.Vendor)
|
||||
product := normToken(info.Product)
|
||||
|
||||
if p, ok := platforms[vendor]; ok && product != "" && product != p {
|
||||
return p, true
|
||||
}
|
||||
for _, tag := range info.Tags {
|
||||
t := normToken(tag)
|
||||
if p, ok := platforms[t]; ok {
|
||||
if product != "" && product != p {
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
if p, ok := platformTagAliases[t]; ok {
|
||||
if product != p {
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// isDetection reports whether a template is a phase-1 detector.
|
||||
func isDetection(info templateInfo) bool {
|
||||
for _, tag := range info.Tags {
|
||||
switch normToken(tag) {
|
||||
case "tech", "detect", "favicon":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isDetectionSeverity guards the detection bucket against vuln/exposure templates
|
||||
// that merely carry a "tech"/"detect" tag. Real fingerprint templates are
|
||||
// informational; medium+ severity indicates an actual finding, not detection.
|
||||
func isDetectionSeverity(info templateInfo) bool {
|
||||
switch info.Severity {
|
||||
case "", "info", "low":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
package techgraph
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Selector resolves detected technology signals into a concrete fire-set of
|
||||
// template ids using a tech-graph artifact. It is pure and deterministic.
|
||||
type Selector struct {
|
||||
g *Graph
|
||||
byAlias map[string]string // alias/synonym/id/product -> canonical tech id
|
||||
}
|
||||
|
||||
// NewSelector builds a selector with an alias index for fast signal resolution.
|
||||
func NewSelector(g *Graph) *Selector {
|
||||
s := &Selector{g: g, byAlias: make(map[string]string)}
|
||||
// synonyms first (authoritative), then ids/products/aliases without clobbering.
|
||||
for a, id := range g.Synonyms {
|
||||
s.byAlias[normToken(a)] = id
|
||||
}
|
||||
for id, t := range g.Techs {
|
||||
if _, ok := s.byAlias[normToken(id)]; !ok {
|
||||
s.byAlias[normToken(id)] = id
|
||||
}
|
||||
if t.Product != "" {
|
||||
if _, ok := s.byAlias[normToken(t.Product)]; !ok {
|
||||
s.byAlias[normToken(t.Product)] = id
|
||||
}
|
||||
}
|
||||
for _, a := range t.Aliases {
|
||||
if _, ok := s.byAlias[normToken(a)]; !ok {
|
||||
s.byAlias[normToken(a)] = id
|
||||
}
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Resolve maps detected signal tokens (wappalyzer names, detection tags, cpe
|
||||
// products) to canonical tech ids. Tokens that match no node are returned as
|
||||
// unresolved so the caller can apply tag-overlap fallback.
|
||||
func (s *Selector) Resolve(signals []string) (techIDs, unresolved []string) {
|
||||
seenTech := map[string]struct{}{}
|
||||
seenUnres := map[string]struct{}{}
|
||||
for _, sig := range signals {
|
||||
tok := normToken(sig)
|
||||
if tok == "" || genericTokens[tok] {
|
||||
continue
|
||||
}
|
||||
if id, ok := s.byAlias[tok]; ok {
|
||||
if _, dup := seenTech[id]; !dup {
|
||||
seenTech[id] = struct{}{}
|
||||
techIDs = append(techIDs, id)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, dup := seenUnres[tok]; !dup {
|
||||
seenUnres[tok] = struct{}{}
|
||||
unresolved = append(unresolved, tok)
|
||||
}
|
||||
}
|
||||
sort.Strings(techIDs)
|
||||
sort.Strings(unresolved)
|
||||
return techIDs, unresolved
|
||||
}
|
||||
|
||||
// FireSet returns the precise template ids to execute for the given techs at the
|
||||
// requested coverage tier (dependents + tier-gated baseline).
|
||||
func (s *Selector) FireSet(tier Tier, techIDs []string) []string {
|
||||
ids := map[string]struct{}{}
|
||||
for _, id := range techIDs {
|
||||
if t, ok := s.g.Techs[id]; ok {
|
||||
for _, r := range t.Templates {
|
||||
ids[r.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, b := range s.g.Baseline {
|
||||
if tierIncludesBaseline(tier, b.Tier) {
|
||||
ids[b.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(ids))
|
||||
for id := range ids {
|
||||
out = append(out, id)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// BaselineIDs returns just the tier-gated baseline ids.
|
||||
func (s *Selector) BaselineIDs(tier Tier) []string {
|
||||
var out []string
|
||||
for _, b := range s.g.Baseline {
|
||||
if tierIncludesBaseline(tier, b.Tier) {
|
||||
out = append(out, b.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// tierIncludesBaseline gates baseline membership by coverage tier:
|
||||
// - lean: no baseline (dependents only)
|
||||
// - balanced: balanced baseline only
|
||||
// - thorough: balanced + thorough baseline
|
||||
func tierIncludesBaseline(scan, entry Tier) bool {
|
||||
switch scan {
|
||||
case TierLean:
|
||||
return false
|
||||
case TierThorough:
|
||||
return true
|
||||
default: // balanced
|
||||
return entry == TierBalanced
|
||||
}
|
||||
}
|
||||
|
||||
// NormalizeTier coerces arbitrary input to a valid tier, defaulting to balanced.
|
||||
// Case and surrounding whitespace are ignored.
|
||||
func NormalizeTier(s string) Tier {
|
||||
tier, err := ParseTier(s)
|
||||
if err != nil {
|
||||
return TierBalanced
|
||||
}
|
||||
return tier
|
||||
}
|
||||
|
||||
// ParseTier validates a coverage tier. Empty input defaults to balanced.
|
||||
// Unknown values return an error instead of silently changing behavior.
|
||||
func ParseTier(s string) (Tier, error) {
|
||||
switch Tier(strings.ToLower(strings.TrimSpace(s))) {
|
||||
case "", TierBalanced:
|
||||
return TierBalanced, nil
|
||||
case TierLean:
|
||||
return TierLean, nil
|
||||
case TierThorough:
|
||||
return TierThorough, nil
|
||||
default:
|
||||
return "", fmt.Errorf("invalid automatic scan coverage %q (allowed: lean, balanced, thorough)", s)
|
||||
}
|
||||
}
|
||||
|
||||
// Candidate is a loaded template that the artifact may not know about yet.
|
||||
type Candidate struct {
|
||||
ID string
|
||||
Tags []string
|
||||
Category string
|
||||
}
|
||||
|
||||
// AbsentDependents returns ids of templates the graph does not contain whose
|
||||
// tags resolve to exactly one of the detected techs. Ambiguous tags and
|
||||
// excluded categories are left out, so this only recovers templates added
|
||||
// after the artifact was generated.
|
||||
func (s *Selector) AbsentDependents(detected []string, candidates []Candidate) []string {
|
||||
if s == nil || s.g == nil || len(detected) == 0 || len(candidates) == 0 {
|
||||
return nil
|
||||
}
|
||||
want := make(map[string]struct{}, len(detected))
|
||||
for _, id := range detected {
|
||||
want[id] = struct{}{}
|
||||
}
|
||||
known := s.knownIDs()
|
||||
var out []string
|
||||
for _, c := range candidates {
|
||||
if c.ID == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := known[c.ID]; ok {
|
||||
continue
|
||||
}
|
||||
// An unknown category is skipped: excluded categories such as fuzzing
|
||||
// and cloud must not be pulled in just because the path did not parse.
|
||||
if _, excluded := excludedCategories[c.Category]; excluded || c.Category == "" {
|
||||
continue
|
||||
}
|
||||
matches := map[string]struct{}{}
|
||||
for _, tag := range c.Tags {
|
||||
tok := normToken(tag)
|
||||
if len(tok) < 5 || genericTokens[tok] {
|
||||
continue
|
||||
}
|
||||
id, ok := s.byAlias[tok]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
matches[id] = struct{}{}
|
||||
}
|
||||
if len(matches) != 1 {
|
||||
continue
|
||||
}
|
||||
var only string
|
||||
for id := range matches {
|
||||
only = id
|
||||
}
|
||||
if _, ok := want[only]; !ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, c.ID)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// knownIDs is every template id the artifact already classifies.
|
||||
func (s *Selector) knownIDs() map[string]struct{} {
|
||||
ids := make(map[string]struct{})
|
||||
for _, tech := range s.g.Techs {
|
||||
for _, ref := range tech.Templates {
|
||||
ids[ref.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, b := range s.g.Baseline {
|
||||
ids[b.ID] = struct{}{}
|
||||
}
|
||||
for _, d := range s.g.Detection {
|
||||
ids[d.ID] = struct{}{}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package techgraph
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func testGraph() *Graph {
|
||||
return &Graph{
|
||||
Techs: map[string]*Tech{
|
||||
"wordpress": {ID: "wordpress", Product: "wordpress", Aliases: []string{"wordpress"},
|
||||
Templates: []TemplateRef{{ID: "wp-cve-1"}, {ID: "wp-plugin-1", Source: "platform"}}},
|
||||
"atlassian:confluence": {ID: "atlassian:confluence", Product: "confluence", Aliases: []string{"confluence"},
|
||||
Templates: []TemplateRef{{ID: "confluence-cve-1"}}},
|
||||
},
|
||||
Baseline: []BaselineEntry{
|
||||
{ID: "git-config", Tier: TierBalanced},
|
||||
{ID: "noisy-osinty", Tier: TierThorough},
|
||||
},
|
||||
Synonyms: map[string]string{"aem": "adobe:experience_manager"},
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolve(t *testing.T) {
|
||||
s := NewSelector(testGraph())
|
||||
tech, unres := s.Resolve([]string{"WordPress", "confluence", "php", "cve"})
|
||||
if len(tech) != 2 {
|
||||
t.Fatalf("want 2 techs, got %v", tech)
|
||||
}
|
||||
if len(unres) != 1 || unres[0] != "php" {
|
||||
t.Fatalf("want [php] unresolved, got %v", unres)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFireSetTiers(t *testing.T) {
|
||||
s := NewSelector(testGraph())
|
||||
techs := []string{"wordpress"}
|
||||
|
||||
lean := s.FireSet(TierLean, techs)
|
||||
if want := []string{"wp-cve-1", "wp-plugin-1"}; !reflect.DeepEqual(lean, want) {
|
||||
t.Fatalf("lean: want %v, got %v", want, lean)
|
||||
}
|
||||
bal := s.FireSet(TierBalanced, techs)
|
||||
if want := []string{"git-config", "wp-cve-1", "wp-plugin-1"}; !reflect.DeepEqual(bal, want) {
|
||||
t.Fatalf("balanced: want %v, got %v", want, bal)
|
||||
}
|
||||
tho := s.FireSet(TierThorough, techs)
|
||||
if want := []string{"git-config", "noisy-osinty", "wp-cve-1", "wp-plugin-1"}; !reflect.DeepEqual(tho, want) {
|
||||
t.Fatalf("thorough: want %v, got %v", want, tho)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeTier(t *testing.T) {
|
||||
for in, want := range map[string]Tier{"lean": TierLean, " THOROUGH ": TierThorough, "": TierBalanced, "x": TierBalanced} {
|
||||
if got := NormalizeTier(in); got != want {
|
||||
t.Fatalf("NormalizeTier(%q)=%q want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTier(t *testing.T) {
|
||||
if _, err := ParseTier("not-a-tier"); err == nil {
|
||||
t.Fatal("expected error for unknown tier")
|
||||
}
|
||||
got, err := ParseTier(" Lean ")
|
||||
if err != nil || got != TierLean {
|
||||
t.Fatalf("ParseTier(Lean)=%q %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAbsentDependents(t *testing.T) {
|
||||
s := NewSelector(testGraph())
|
||||
candidates := []Candidate{
|
||||
{ID: "new-wp", Tags: []string{"wordpress"}, Category: "cves"},
|
||||
{ID: "ambiguous", Tags: []string{"wordpress", "confluence"}, Category: "cves"},
|
||||
{ID: "fuzzed", Tags: []string{"wordpress"}, Category: "fuzzing"},
|
||||
{ID: "wp-cve-1", Tags: []string{"wordpress"}, Category: "cves"},
|
||||
{ID: "short", Tags: []string{"wp"}, Category: "cves"},
|
||||
{ID: "other-tech", Tags: []string{"wordpress"}, Category: "cves"},
|
||||
}
|
||||
got := s.AbsentDependents([]string{"wordpress"}, candidates[:5])
|
||||
if want := []string{"new-wp"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("want %v, got %v", want, got)
|
||||
}
|
||||
if got := s.AbsentDependents([]string{"atlassian:confluence"}, candidates[5:]); len(got) != 0 {
|
||||
t.Fatalf("wordpress tag must not attach to confluence, got %v", got)
|
||||
}
|
||||
if got := s.AbsentDependents([]string{"wordpress"}, []Candidate{{ID: "no-path", Tags: []string{"wordpress"}}}); len(got) != 0 {
|
||||
t.Fatalf("unknown category must not be added, got %v", got)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,84 @@
|
||||
// Package techgraph builds the offline technology->template mapping artifact
|
||||
// (tech-graph.json) consumed by automatic scan v2. See AUTOMATIC_SCAN_V2.md.
|
||||
package techgraph
|
||||
|
||||
// ArtifactVersion is the schema version of the emitted artifact.
|
||||
const ArtifactVersion = 1
|
||||
|
||||
// Tier gates how aggressively a template is selected at runtime.
|
||||
type Tier string
|
||||
|
||||
const (
|
||||
TierLean Tier = "lean"
|
||||
TierBalanced Tier = "balanced"
|
||||
TierThorough Tier = "thorough"
|
||||
)
|
||||
|
||||
// Graph is the top-level tech-graph artifact.
|
||||
type Graph struct {
|
||||
Version int `json:"version"`
|
||||
GeneratedAt string `json:"generated_at"`
|
||||
SourceHash string `json:"source_hash,omitempty"`
|
||||
Techs map[string]*Tech `json:"techs"`
|
||||
Baseline []BaselineEntry `json:"baseline"`
|
||||
Detection []DetectionEntry `json:"detection"`
|
||||
Synonyms map[string]string `json:"synonyms"`
|
||||
Stats Stats `json:"stats"`
|
||||
}
|
||||
|
||||
// DetectionEntry is a phase-1 detector template (tagged tech/detect/favicon).
|
||||
// Running these fingerprints the target before selection.
|
||||
type DetectionEntry struct {
|
||||
ID string `json:"id"`
|
||||
Tech string `json:"tech,omitempty"` // canonical tech this detector anchors, if any
|
||||
}
|
||||
|
||||
// Tech is a single canonical technology and the templates that depend on it.
|
||||
type Tech struct {
|
||||
ID string `json:"id"`
|
||||
Vendor string `json:"vendor,omitempty"`
|
||||
Product string `json:"product,omitempty"`
|
||||
CPE string `json:"cpe,omitempty"`
|
||||
Aliases []string `json:"aliases,omitempty"`
|
||||
Templates []TemplateRef `json:"templates"`
|
||||
}
|
||||
|
||||
// TemplateRef references a template that depends on a tech.
|
||||
type TemplateRef struct {
|
||||
ID string `json:"id"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Severity string `json:"severity,omitempty"`
|
||||
// Product is the original sub-product when attached to a platform node via
|
||||
// platform grouping (e.g. plugin "wp-optimize" under platform "wordpress").
|
||||
Product string `json:"product,omitempty"`
|
||||
// Source records how the template was attached: "cpe", "platform", "tag".
|
||||
Source string `json:"source,omitempty"`
|
||||
// VersionRange is reserved for the AI version-parsing milestone. Empty means
|
||||
// "fire regardless of detected version".
|
||||
VersionRange string `json:"version_range,omitempty"`
|
||||
}
|
||||
|
||||
// BaselineEntry is a tech-agnostic template that runs on every applicable target.
|
||||
type BaselineEntry struct {
|
||||
ID string `json:"id"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Severity string `json:"severity,omitempty"`
|
||||
Tier Tier `json:"tier"`
|
||||
}
|
||||
|
||||
// Stats summarises a build run.
|
||||
type Stats struct {
|
||||
Total int `json:"total"`
|
||||
Dependents int `json:"dependents"`
|
||||
DependentsCPE int `json:"dependents_cpe"`
|
||||
DependentsPlat int `json:"dependents_platform"`
|
||||
DependentsDir int `json:"dependents_dir"`
|
||||
DependentsTag int `json:"dependents_tag"`
|
||||
DependentsID int `json:"dependents_id"`
|
||||
Baseline int `json:"baseline"`
|
||||
Detection int `json:"detection"`
|
||||
Excluded int `json:"excluded"`
|
||||
Unmapped int `json:"unmapped"`
|
||||
Techs int `json:"techs"`
|
||||
ParseError int `json:"parse_error"`
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>nuclei tech-graph visualizer</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0c0f17; --panel: #141926; --panel2: #1b2233; --line: #263149;
|
||||
--txt: #e6edf6; --muted: #8b97ad; --accent: #5b8cff;
|
||||
--cpe:#36c98a; --platform:#b07cff; --dir:#3bc4c4; --tag:#5b8cff; --id:#f0a23c;
|
||||
--crit:#ff5470; --high:#ff8a3d; --med:#f5c451; --low:#5b8cff; --info:#7d8aa3;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body { margin:0; background:var(--bg); color:var(--txt);
|
||||
font:14px/1.5 ui-sans-serif,system-ui,-apple-system,"Segoe UI",Roboto,sans-serif; }
|
||||
header { padding:18px 24px; border-bottom:1px solid var(--line);
|
||||
display:flex; align-items:center; gap:16px; flex-wrap:wrap;
|
||||
background:linear-gradient(180deg,#10141f,#0c0f17); position:sticky; top:0; z-index:5; }
|
||||
h1 { font-size:16px; margin:0; font-weight:600; letter-spacing:.3px; }
|
||||
h1 span { color:var(--accent); }
|
||||
.meta { color:var(--muted); font-size:12px; }
|
||||
.btn { background:var(--panel2); color:var(--txt); border:1px solid var(--line);
|
||||
padding:7px 13px; border-radius:8px; cursor:pointer; font-size:13px; }
|
||||
.btn:hover { border-color:var(--accent); }
|
||||
main { padding:20px 24px; max-width:1240px; margin:0 auto; }
|
||||
.cards { display:grid; grid-template-columns:repeat(auto-fit,minmax(140px,1fr)); gap:12px; margin-bottom:18px; }
|
||||
.card { background:var(--panel); border:1px solid var(--line); border-radius:12px; padding:14px 16px; }
|
||||
.card .n { font-size:24px; font-weight:700; }
|
||||
.card .l { color:var(--muted); font-size:12px; text-transform:uppercase; letter-spacing:.5px; }
|
||||
.srcbar { display:flex; height:10px; border-radius:6px; overflow:hidden; margin:4px 0 16px; border:1px solid var(--line); }
|
||||
.srcbar div { height:100%; }
|
||||
.legend { display:flex; gap:14px; flex-wrap:wrap; font-size:12px; color:var(--muted); margin-bottom:18px; }
|
||||
.legend i { display:inline-block; width:10px; height:10px; border-radius:3px; margin-right:5px; vertical-align:middle; }
|
||||
.tabs { display:flex; gap:6px; border-bottom:1px solid var(--line); margin-bottom:16px; }
|
||||
.tab { padding:9px 15px; cursor:pointer; color:var(--muted); border-bottom:2px solid transparent; }
|
||||
.tab.active { color:var(--txt); border-bottom-color:var(--accent); }
|
||||
.toolbar { display:flex; gap:10px; margin-bottom:14px; flex-wrap:wrap; align-items:center; }
|
||||
input[type=search], select { background:var(--panel2); border:1px solid var(--line); color:var(--txt);
|
||||
padding:8px 12px; border-radius:8px; font-size:13px; }
|
||||
input[type=search] { flex:1; min-width:220px; }
|
||||
.tech { background:var(--panel); border:1px solid var(--line); border-radius:10px; margin-bottom:8px; overflow:hidden; }
|
||||
.tech > summary { padding:11px 15px; cursor:pointer; display:flex; align-items:center; gap:12px; list-style:none; }
|
||||
.tech > summary::-webkit-details-marker { display:none; }
|
||||
.tech .id { font-weight:600; }
|
||||
.tech .cnt { margin-left:auto; color:var(--muted); font-size:12px; background:var(--panel2);
|
||||
padding:2px 9px; border-radius:20px; }
|
||||
.cpe { color:var(--muted); font-size:11px; font-family:ui-monospace,monospace; }
|
||||
.tpls { border-top:1px solid var(--line); padding:6px 15px 12px; }
|
||||
.row { display:flex; align-items:center; gap:8px; padding:5px 0; border-bottom:1px dashed #1e2740; font-size:13px; }
|
||||
.row:last-child { border-bottom:none; }
|
||||
.row .tid { font-family:ui-monospace,monospace; font-size:12px; }
|
||||
.badge { font-size:10px; padding:1px 7px; border-radius:20px; text-transform:uppercase; letter-spacing:.4px; font-weight:600; }
|
||||
.b-cpe{background:rgba(54,201,138,.16);color:var(--cpe)} .b-platform{background:rgba(176,124,255,.16);color:var(--platform)}
|
||||
.b-dir{background:rgba(59,196,196,.16);color:var(--dir)} .b-tag{background:rgba(91,140,255,.16);color:var(--tag)}
|
||||
.b-id{background:rgba(240,162,60,.16);color:var(--id)}
|
||||
.sev { width:7px; height:7px; border-radius:50%; flex:none; }
|
||||
.s-critical{background:var(--crit)} .s-high{background:var(--high)} .s-medium{background:var(--med)}
|
||||
.s-low{background:var(--low)} .s-info{background:var(--info)} .s-unknown{background:#444}
|
||||
.cat { color:var(--muted); font-size:11px; }
|
||||
.drop { border:2px dashed var(--line); border-radius:14px; padding:60px; text-align:center; color:var(--muted); }
|
||||
.drop.hot { border-color:var(--accent); color:var(--txt); }
|
||||
.muted { color:var(--muted); }
|
||||
.hidden { display:none; }
|
||||
.alias { color:var(--muted); font-size:11px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>nuclei <span>tech-graph</span> visualizer</h1>
|
||||
<span class="meta" id="meta"></span>
|
||||
<button class="btn" id="pick" style="margin-left:auto">Load tech-graph.json</button>
|
||||
<input type="file" id="file" accept="application/json,.json" class="hidden" />
|
||||
</header>
|
||||
<main>
|
||||
<div id="empty" class="drop">
|
||||
Drop <b>tech-graph.json</b> here, or click <b>Load</b>.<br/>
|
||||
<span class="muted">Generate it with <code>techgraph-gen -templates <dir> -output tech-graph.json</code></span>
|
||||
</div>
|
||||
|
||||
<div id="app" class="hidden">
|
||||
<div class="cards" id="cards"></div>
|
||||
<div class="srcbar" id="srcbar"></div>
|
||||
<div class="legend" id="legend"></div>
|
||||
|
||||
<div class="tabs">
|
||||
<div class="tab active" data-tab="techs">Technologies</div>
|
||||
<div class="tab" data-tab="baseline">Baseline</div>
|
||||
<div class="tab" data-tab="detection">Detection</div>
|
||||
<div class="tab" data-tab="synonyms">Synonyms</div>
|
||||
</div>
|
||||
|
||||
<div class="toolbar">
|
||||
<input type="search" id="q" placeholder="Search tech, template id, alias, cpe…" />
|
||||
<select id="sort">
|
||||
<option value="count">Sort: most templates</option>
|
||||
<option value="name">Sort: name</option>
|
||||
</select>
|
||||
<select id="srcFilter">
|
||||
<option value="">Source: all</option>
|
||||
<option value="cpe">cpe</option><option value="platform">platform</option>
|
||||
<option value="dir">dir</option><option value="tag">tag</option><option value="id">id</option>
|
||||
</select>
|
||||
<span class="muted" id="resultCount"></span>
|
||||
</div>
|
||||
|
||||
<div id="view"></div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<script>
|
||||
const SEV = ["critical","high","medium","low","info"];
|
||||
const SRC = ["cpe","platform","dir","tag","id"];
|
||||
const SRC_COLOR = {cpe:"--cpe",platform:"--platform",dir:"--dir",tag:"--tag",id:"--id"};
|
||||
let G = null;
|
||||
|
||||
const $ = s => document.querySelector(s);
|
||||
const el = (t,c,h) => { const e=document.createElement(t); if(c)e.className=c; if(h!=null)e.innerHTML=h; return e; };
|
||||
const esc = s => String(s==null?"":s).replace(/[&<>]/g,m=>({"&":"&","<":"<",">":">"}[m]));
|
||||
|
||||
function load(json){
|
||||
G = json;
|
||||
$("#empty").classList.add("hidden");
|
||||
$("#app").classList.remove("hidden");
|
||||
$("#meta").textContent = `v${json.version} · ${json.generated_at||""} · ${json.source_hash? json.source_hash.slice(0,18)+"…":""}`;
|
||||
renderCards(); renderTabs(); render();
|
||||
}
|
||||
|
||||
function renderCards(){
|
||||
const s = G.stats || {};
|
||||
const cards = [
|
||||
["templates", s.total], ["dependents", s.dependents], ["techs", s.techs],
|
||||
["baseline", s.baseline], ["detection", s.detection], ["excluded", s.excluded], ["unmapped", s.unmapped],
|
||||
];
|
||||
$("#cards").innerHTML = cards.map(([l,n])=>`<div class="card"><div class="n">${n??0}</div><div class="l">${l}</div></div>`).join("");
|
||||
// source bar from per-source dependent counts
|
||||
const bySrc = {cpe:s.dependents_cpe,platform:s.dependents_platform,dir:s.dependents_dir,tag:s.dependents_tag,id:s.dependents_id};
|
||||
const tot = SRC.reduce((a,k)=>a+(bySrc[k]||0),0)||1;
|
||||
$("#srcbar").innerHTML = SRC.map(k=>`<div style="width:${100*(bySrc[k]||0)/tot}%;background:var(${SRC_COLOR[k]})" title="${k}: ${bySrc[k]||0}"></div>`).join("");
|
||||
$("#legend").innerHTML = SRC.map(k=>`<span><i style="background:var(${SRC_COLOR[k]})"></i>${k} ${bySrc[k]||0}</span>`).join("")
|
||||
+ ` <span style="margin-left:auto" class="muted">dependent attachment source</span>`;
|
||||
}
|
||||
|
||||
let TAB = "techs";
|
||||
function renderTabs(){
|
||||
document.querySelectorAll(".tab").forEach(t=>{
|
||||
t.onclick = ()=>{ TAB=t.dataset.tab; document.querySelectorAll(".tab").forEach(x=>x.classList.toggle("active",x===t)); render(); };
|
||||
});
|
||||
}
|
||||
["#q","#sort","#srcFilter"].forEach(s=>document.addEventListener("input",e=>{ if(e.target.matches(s)) render(); }));
|
||||
|
||||
function render(){
|
||||
const q = $("#q").value.trim().toLowerCase();
|
||||
const v = $("#view"); v.innerHTML="";
|
||||
if(TAB==="techs") return renderTechs(q,v);
|
||||
if(TAB==="baseline") return renderList(G.baseline||[], q, v, b=>[b.id,b.category], b=>rowHTML(b.id,null,b.category,b.severity,b.tier));
|
||||
if(TAB==="detection") return renderList(G.detection||[], q, v, d=>[d.id,d.tech], d=>rowHTML(d.id,null,"detector → "+(d.tech||"—")));
|
||||
if(TAB==="synonyms"){
|
||||
const ents=Object.entries(G.synonyms||{}).filter(([a,t])=>!q||a.includes(q)||t.includes(q)).sort();
|
||||
$("#resultCount").textContent = ents.length+" synonyms";
|
||||
ents.forEach(([a,t])=>{ const r=el("div","row"); r.innerHTML=`<span class="tid">${esc(a)}</span><span class="muted">→</span><span class="badge b-tag">${esc(t)}</span>`; v.appendChild(r); });
|
||||
}
|
||||
}
|
||||
|
||||
function renderTechs(q,v){
|
||||
const srcF = $("#srcFilter").value;
|
||||
let techs = Object.values(G.techs||{});
|
||||
techs = techs.filter(t=>{
|
||||
const tpls = t.templates||[];
|
||||
if(srcF && !tpls.some(x=>x.source===srcF)) return false;
|
||||
if(!q) return true;
|
||||
if(t.id.toLowerCase().includes(q) || (t.cpe||"").toLowerCase().includes(q)) return true;
|
||||
if((t.aliases||[]).some(a=>a.includes(q))) return true;
|
||||
return tpls.some(x=>x.id.toLowerCase().includes(q));
|
||||
});
|
||||
techs.sort($("#sort").value==="name"
|
||||
? (a,b)=>a.id.localeCompare(b.id)
|
||||
: (a,b)=>(b.templates||[]).length-(a.templates||[]).length);
|
||||
$("#resultCount").textContent = `${techs.length} techs · ${techs.reduce((n,t)=>n+(t.templates||[]).length,0)} templates`;
|
||||
const frag=document.createDocumentFragment();
|
||||
for(const t of techs.slice(0,400)){
|
||||
const d=el("details","tech");
|
||||
const tpls=(t.templates||[]);
|
||||
d.innerHTML=`<summary><span class="id">${esc(t.id)}</span>`
|
||||
+(t.cpe?`<span class="cpe">${esc(t.cpe)}</span>`:"")
|
||||
+(t.aliases&&t.aliases.length?`<span class="alias">${esc(t.aliases.slice(0,4).join(", "))}</span>`:"")
|
||||
+`<span class="cnt">${tpls.length}</span></summary>`;
|
||||
const body=el("div","tpls");
|
||||
let list=tpls; if(srcF) list=list.filter(x=>x.source===srcF);
|
||||
if(q) list=list.filter(x=>x.id.toLowerCase().includes(q)||t.id.toLowerCase().includes(q)||(t.aliases||[]).some(a=>a.includes(q)));
|
||||
body.innerHTML = list.slice(0,300).map(x=>rowHTML(x.id,x.source,x.category,x.severity)).join("");
|
||||
d.appendChild(body); frag.appendChild(d);
|
||||
}
|
||||
v.appendChild(frag);
|
||||
if(techs.length>400) v.appendChild(el("div","muted",`Showing first 400 of ${techs.length}. Refine your search.`));
|
||||
}
|
||||
|
||||
function renderList(items,q,v,keys,rowFn){
|
||||
const f=items.filter(it=>!q||keys(it).some(k=>String(k||"").toLowerCase().includes(q)));
|
||||
$("#resultCount").textContent = f.length+" items";
|
||||
v.innerHTML = `<div class="tech"><div class="tpls">`+f.slice(0,1000).map(rowFn).join("")+`</div></div>`;
|
||||
}
|
||||
|
||||
function rowHTML(id,source,category,severity,extra){
|
||||
const sev=(severity||"unknown").toLowerCase();
|
||||
return `<div class="row"><span class="sev s-${SEV.includes(sev)?sev:"unknown"}"></span>`
|
||||
+ `<span class="tid">${esc(id)}</span>`
|
||||
+ (source?`<span class="badge b-${source}">${esc(source)}</span>`:"")
|
||||
+ (category?`<span class="cat">${esc(category)}</span>`:"")
|
||||
+ (extra?`<span class="cat">· ${esc(extra)}</span>`:"")
|
||||
+ `</div>`;
|
||||
}
|
||||
|
||||
// ---- loading: file picker, drag/drop, auto-fetch ----
|
||||
$("#pick").onclick=()=>$("#file").click();
|
||||
$("#file").onchange=e=>{ const f=e.target.files[0]; if(f) f.text().then(t=>load(JSON.parse(t))); };
|
||||
const dz=$("#empty");
|
||||
["dragover","dragenter"].forEach(ev=>dz.addEventListener(ev,e=>{e.preventDefault();dz.classList.add("hot");}));
|
||||
["dragleave","drop"].forEach(ev=>dz.addEventListener(ev,e=>{e.preventDefault();dz.classList.remove("hot");}));
|
||||
dz.addEventListener("drop",e=>{ const f=e.dataTransfer.files[0]; if(f) f.text().then(t=>load(JSON.parse(t))); });
|
||||
if(window.__TECHGRAPH__){ load(window.__TECHGRAPH__); }
|
||||
else { fetch("tech-graph.json").then(r=>r.ok?r.json():Promise.reject()).then(load).catch(()=>{}); }
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,8 +1,6 @@
|
||||
package eventcreator
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
@@ -12,29 +10,6 @@ import (
|
||||
"golang.org/x/text/language"
|
||||
)
|
||||
|
||||
// RecordFunc receives the event of a request before its operators run, which
|
||||
// is what a replay needs to run them again.
|
||||
type RecordFunc func(request protocols.Request, event output.InternalEvent)
|
||||
|
||||
// Every protocol builds its result here, so a recorder set here covers all
|
||||
// of them. Scans read it concurrently, hence the atomic.
|
||||
var record atomic.Pointer[RecordFunc]
|
||||
|
||||
// SetRecord installs fn, nil to stop recording, and returns the callback it
|
||||
// replaced so the caller can put it back.
|
||||
func SetRecord(fn RecordFunc) RecordFunc {
|
||||
var previous *RecordFunc
|
||||
if fn == nil {
|
||||
previous = record.Swap(nil)
|
||||
} else {
|
||||
previous = record.Swap(&fn)
|
||||
}
|
||||
if previous == nil {
|
||||
return nil
|
||||
}
|
||||
return *previous
|
||||
}
|
||||
|
||||
// CreateEvent wraps the outputEvent with the result of the operators defined on the request
|
||||
func CreateEvent(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool) *output.InternalWrappedEvent {
|
||||
return CreateEventWithAdditionalOptions(request, outputEvent, isResponseDebug, nil)
|
||||
@@ -44,9 +19,6 @@ func CreateEvent(request protocols.Request, outputEvent output.InternalEvent, is
|
||||
// and enables extending the resulting event with additional attributes or values.
|
||||
func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool,
|
||||
addAdditionalOptions func(internalWrappedEvent *output.InternalWrappedEvent)) *output.InternalWrappedEvent {
|
||||
if fn := record.Load(); fn != nil {
|
||||
(*fn)(request, outputEvent)
|
||||
}
|
||||
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
|
||||
|
||||
// Dump response variables if ran in debug mode
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
package eventcreator
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSetRecordRestoresTheCallbackItReplaced(t *testing.T) {
|
||||
var calls []string
|
||||
first := RecordFunc(func(protocols.Request, output.InternalEvent) { calls = append(calls, "first") })
|
||||
second := RecordFunc(func(protocols.Request, output.InternalEvent) { calls = append(calls, "second") })
|
||||
|
||||
require.Nil(t, SetRecord(first))
|
||||
previous := SetRecord(second)
|
||||
(*record.Load())(nil, nil)
|
||||
// a second runner stopping puts the first one's recorder back, not nil
|
||||
SetRecord(previous)
|
||||
(*record.Load())(nil, nil)
|
||||
require.Equal(t, []string{"second", "first"}, calls)
|
||||
|
||||
SetRecord(nil)
|
||||
require.Nil(t, record.Load())
|
||||
}
|
||||
@@ -16,7 +16,6 @@ type Dialers struct {
|
||||
HTTPClientPool *HTTPPool
|
||||
PerHostRateLimitPool any // *hostratelimit.PerHostRateLimitPool
|
||||
HTTPToHTTPSPortTracker any // *httpclientpool.HTTPToHTTPSPortTracker
|
||||
SharedResponses any // *httpclientpool.sharedResponses
|
||||
HTTPDesyncHosts *ExpiringSet
|
||||
NetworkPolicy *networkpolicy.NetworkPolicy
|
||||
LocalFileAccessAllowed bool
|
||||
|
||||
@@ -7,10 +7,10 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/projectdiscovery/networkpolicy"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
|
||||
@@ -15,7 +15,7 @@ func TestMemGuardianGoroutineLeak(t *testing.T) {
|
||||
defer goleak.VerifyNone(t,
|
||||
goleak.IgnoreAnyContainingPkg("go.opencensus.io/stats/view"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/rod"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/go-rod/rod"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/server"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/ratelimit"),
|
||||
// expirable LRU cache creates a background goroutine for TTL expiration that persists
|
||||
@@ -104,7 +104,7 @@ func TestMemGuardianReset(t *testing.T) {
|
||||
defer goleak.VerifyNone(t,
|
||||
goleak.IgnoreAnyContainingPkg("go.opencensus.io/stats/view"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/rod"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/go-rod/rod"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/server"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/ratelimit"),
|
||||
// expirable LRU cache creates a background goroutine for TTL expiration that persists
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/pkg/errors"
|
||||
@@ -18,7 +16,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
iputil "github.com/projectdiscovery/utils/ip"
|
||||
)
|
||||
|
||||
// Request contains a DNS protocol request to be made from a template
|
||||
@@ -207,48 +204,6 @@ func resolverHost(resolver string) string {
|
||||
return r
|
||||
}
|
||||
|
||||
// pinResolverHost replaces a resolver host name with an address that the
|
||||
// network policy allows. The policy check passes when one address of the name
|
||||
// is allowed, and the dns client resolves the name again when it dials, so it
|
||||
// could otherwise reach a denied address of the same name.
|
||||
// The lookup stops after timeout, so a slow system resolver cannot hold the
|
||||
// request.
|
||||
func pinResolverHost(executionID, resolver, host string, timeout time.Duration) (string, error) {
|
||||
// DoH and DoT resolvers dial through the scan dialer, which checks every
|
||||
// address, and DoT verifies the certificate against the name.
|
||||
if iputil.IsIP(host) || strings.HasPrefix(resolver, "doh:") || strings.HasPrefix(resolver, "dot:") {
|
||||
return resolver, nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
addrs, err := net.DefaultResolver.LookupHost(ctx, host)
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "could not resolve dns resolver %s", resolver)
|
||||
}
|
||||
prefix, address := resolver, ""
|
||||
if index := strings.LastIndex(resolver, host); index != -1 {
|
||||
prefix, address = resolver[:index], resolver[index:]
|
||||
}
|
||||
port := "53"
|
||||
if _, p, err := net.SplitHostPort(address); err == nil && p != "" {
|
||||
port = p
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
if protocolstate.IsHostAllowed(executionID, addr) {
|
||||
return prefix + net.JoinHostPort(addr, port), nil
|
||||
}
|
||||
}
|
||||
return "", errors.Errorf("dns resolver %s is blocked by network policy", resolver)
|
||||
}
|
||||
|
||||
// resolverLookupTimeout is the scan timeout, or ten seconds when it is not set.
|
||||
func resolverLookupTimeout(options *protocols.ExecutorOptions) time.Duration {
|
||||
if options.Options.Timeout > 0 {
|
||||
return time.Duration(options.Options.Timeout) * time.Second
|
||||
}
|
||||
return 10 * time.Second
|
||||
}
|
||||
|
||||
func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadata map[string]interface{}) (*dnsclientpool.Client, error) {
|
||||
dnsClientOptions := &dnsclientpool.Configuration{
|
||||
Retries: request.Retries,
|
||||
@@ -282,11 +237,6 @@ func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadat
|
||||
if !protocolstate.IsHostAllowed(options.Options.ExecutionId, host) {
|
||||
return nil, errors.Errorf("dns resolver %s is blocked by network policy", resolver)
|
||||
}
|
||||
pinned, err := pinResolverHost(options.Options.ExecutionId, resolver, host, resolverLookupTimeout(options))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resolver = pinned
|
||||
}
|
||||
resolvers = append(resolvers, resolver)
|
||||
}
|
||||
|
||||
@@ -93,7 +93,7 @@ func Get(options *types.Options, configuration *Configuration) (*Client, error)
|
||||
if (configuration.Retries <= 1) && len(configuration.Resolvers) == 0 {
|
||||
return cachedClient(resolversFor(options))
|
||||
}
|
||||
if len(options.InternalResolversList) == 0 && hasPolicyResolver(configuration.Resolvers) {
|
||||
if len(options.InternalResolversList) == 0 && hasDohResolver(configuration.Resolvers) {
|
||||
return getPolicyClient(options, configuration)
|
||||
}
|
||||
hash := configuration.Hash()
|
||||
|
||||
@@ -3,14 +3,12 @@ package dnsclientpool
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/pkg/errors"
|
||||
@@ -21,8 +19,8 @@ import (
|
||||
)
|
||||
|
||||
// Client sends the DNS queries of a template. It sends queries for template
|
||||
// DoH and DoT resolvers itself, because the retryabledns clients dial and
|
||||
// follow redirects without the network policy of the scan.
|
||||
// DoH resolvers itself, because the retryabledns DoH client dials and follows
|
||||
// redirects without the network policy of the scan.
|
||||
type Client struct {
|
||||
*retryabledns.Client
|
||||
|
||||
@@ -31,51 +29,31 @@ type Client struct {
|
||||
index uint32
|
||||
doh *doh.Client
|
||||
transport *http.Transport
|
||||
dial func(ctx context.Context, network, address string) (net.Conn, error)
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
// resolver is one template resolver. url is set for a DoH resolver and dot
|
||||
// for a DoT resolver, and both are empty for a resolver that retryabledns
|
||||
// queries over UDP or TCP.
|
||||
// resolver is one template resolver. url is empty for a resolver that
|
||||
// retryabledns queries over UDP, TCP or DoT.
|
||||
type resolver struct {
|
||||
url string
|
||||
method doh.Method
|
||||
dot string
|
||||
}
|
||||
|
||||
var (
|
||||
// policyClients holds the clients with DoH or DoT resolvers of each execution,
|
||||
// policyClients holds the clients with DoH resolvers of each execution,
|
||||
// because each one dials with the dialer of its execution.
|
||||
policyClients = map[string]map[string]*Client{}
|
||||
policyMu sync.Mutex
|
||||
|
||||
// dotRootCAs verifies DoT certificates, and nil uses the system roots.
|
||||
dotRootCAs *x509.CertPool
|
||||
)
|
||||
|
||||
func hasPolicyResolver(resolvers []string) bool {
|
||||
func hasDohResolver(resolvers []string) bool {
|
||||
for _, r := range resolvers {
|
||||
if strings.HasPrefix(r, "doh:") || strings.HasPrefix(r, "dot:") {
|
||||
if strings.HasPrefix(r, "doh:") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// parseDotResolver returns the host and port of a resolver such as
|
||||
// dot:dns.example or dot:dns.example:853, in the same way as retryabledns.
|
||||
func parseDotResolver(r string) (resolver, bool) {
|
||||
if !strings.HasPrefix(r, "dot:") {
|
||||
return resolver{}, false
|
||||
}
|
||||
address := strings.TrimPrefix(r, "dot:")
|
||||
if _, _, err := net.SplitHostPort(address); err != nil {
|
||||
address = net.JoinHostPort(address, "853")
|
||||
}
|
||||
return resolver{dot: address}, true
|
||||
}
|
||||
|
||||
// parseDohResolver returns the URL and method of a resolver such as
|
||||
// doh:https://dns.example/dns-query:get, in the same way as retryabledns.
|
||||
func parseDohResolver(r string) (resolver, bool) {
|
||||
@@ -113,14 +91,10 @@ func getPolicyClient(options *types.Options, configuration *Configuration) (*Cli
|
||||
client.resolvers = append(client.resolvers, parsed)
|
||||
continue
|
||||
}
|
||||
if parsed, ok := parseDotResolver(r); ok {
|
||||
client.resolvers = append(client.resolvers, parsed)
|
||||
continue
|
||||
}
|
||||
client.resolvers = append(client.resolvers, resolver{})
|
||||
networkResolvers = append(networkResolvers, r)
|
||||
}
|
||||
// Trace and the UDP and TCP resolvers still go through retryabledns.
|
||||
// Trace and the non DoH resolvers still go through retryabledns.
|
||||
if len(networkResolvers) == 0 {
|
||||
networkResolvers = defaultResolvers
|
||||
}
|
||||
@@ -137,16 +111,14 @@ func getPolicyClient(options *types.Options, configuration *Configuration) (*Cli
|
||||
// The scan dialer checks every address it connects to, so redirects
|
||||
// and names that resolve to a denied address are refused too. It also
|
||||
// goes through the scan proxy when one is set.
|
||||
client.dial = func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
dialers := protocolstate.GetDialersWithId(executionID)
|
||||
if dialers == nil || dialers.Fastdialer == nil {
|
||||
return nil, fmt.Errorf("dialers not initialized for %s", executionID)
|
||||
}
|
||||
return dialers.Fastdialer.Dial(ctx, network, address)
|
||||
}
|
||||
client.timeout = options.GetTimeouts().DialTimeout
|
||||
client.transport = &http.Transport{
|
||||
DialContext: client.dial,
|
||||
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
dialers := protocolstate.GetDialersWithId(executionID)
|
||||
if dialers == nil || dialers.Fastdialer == nil {
|
||||
return nil, fmt.Errorf("dialers not initialized for %s", executionID)
|
||||
}
|
||||
return dialers.Fastdialer.Dial(ctx, network, address)
|
||||
},
|
||||
// same as the retryabledns DoH client
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec G402
|
||||
}
|
||||
@@ -177,8 +149,6 @@ func (c *Client) Do(msg *dns.Msg) (*dns.Msg, error) {
|
||||
r := c.resolvers[index%uint32(len(c.resolvers))]
|
||||
if r.url != "" {
|
||||
resp, err = c.doh.QueryWithDOHMsg(r.method, doh.Resolver{URL: r.url}, msg)
|
||||
} else if r.dot != "" {
|
||||
resp, err = c.queryDoT(r.dot, msg)
|
||||
} else {
|
||||
resp, err = c.Client.Do(msg)
|
||||
}
|
||||
@@ -189,35 +159,7 @@ func (c *Client) Do(msg *dns.Msg) (*dns.Msg, error) {
|
||||
return resp, retryabledns.ErrRetriesExceeded
|
||||
}
|
||||
|
||||
// queryDoT sends a DNS message over TLS to address. It dials through the scan
|
||||
// dialer and verifies the certificate against the resolver host, as the
|
||||
// retryabledns DoT client does.
|
||||
func (c *Client) queryDoT(address string, msg *dns.Msg) (*dns.Msg, error) {
|
||||
host, _, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
timeout := c.timeout
|
||||
if timeout <= 0 {
|
||||
timeout = retryabledns.DefaultOptions.Timeout
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
conn, err := c.dial(ctx, "tcp", address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tlsConn := tls.Client(conn, &tls.Config{ServerName: host, RootCAs: dotRootCAs, MinVersion: tls.VersionTLS12})
|
||||
defer func() { _ = tlsConn.Close() }()
|
||||
if err := tlsConn.HandshakeContext(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dnsClient := &dns.Client{Net: "tcp-tls", Timeout: timeout}
|
||||
resp, _, err := dnsClient.ExchangeWithConn(msg, &dns.Conn{Conn: tlsConn})
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// Close drops the clients with DoH or DoT resolvers of an execution.
|
||||
// Close drops the clients with DoH resolvers of an execution.
|
||||
func Close(executionID string) {
|
||||
policyMu.Lock()
|
||||
clients := policyClients[executionID]
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
package dnsclientpool
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newDotListener accepts TCP connections on 127.0.0.1, counts them and closes
|
||||
// them, so a test sees each DoT dial that reaches the address.
|
||||
func newDotListener(t *testing.T, hits *atomic.Int32) string {
|
||||
t.Helper()
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = l.Close() })
|
||||
go func() {
|
||||
for {
|
||||
conn, err := l.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
hits.Add(1)
|
||||
_ = conn.Close()
|
||||
}
|
||||
}()
|
||||
_, port, err := net.SplitHostPort(l.Addr().String())
|
||||
require.NoError(t, err)
|
||||
return port
|
||||
}
|
||||
|
||||
func TestDotResolverObeysNetworkPolicy(t *testing.T) {
|
||||
for _, host := range []string{"127.0.0.1", "localhost"} {
|
||||
t.Run("denied address "+host, func(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
port := newDotListener(t, &hits)
|
||||
// localhost passes the resolver check through ::1, while 127.0.0.1 is denied
|
||||
options := dohOptions(t, []string{"127.0.0.1"})
|
||||
|
||||
client, err := Get(options, &Configuration{Retries: 2, Resolvers: []string{"dot:" + net.JoinHostPort(host, port)}})
|
||||
require.NoError(t, err)
|
||||
_, _ = client.Do(txtQuery())
|
||||
require.Zero(t, hits.Load(), "DoT query reached a denied address")
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("allowed address", func(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
port := newDotListener(t, &hits)
|
||||
options := dohOptions(t, nil)
|
||||
|
||||
client, err := Get(options, &Configuration{Retries: 1, Resolvers: []string{"dot:127.0.0.1:" + port}})
|
||||
require.NoError(t, err)
|
||||
_, _ = client.Do(txtQuery())
|
||||
require.Equal(t, int32(1), hits.Load())
|
||||
})
|
||||
}
|
||||
|
||||
func TestDotResolverAnswers(t *testing.T) {
|
||||
// httptest gives a certificate for 127.0.0.1 and a pool that trusts it.
|
||||
certs := httptest.NewTLSServer(http.NotFoundHandler())
|
||||
t.Cleanup(certs.Close)
|
||||
previous := dotRootCAs
|
||||
dotRootCAs = certs.Client().Transport.(*http.Transport).TLSClientConfig.RootCAs
|
||||
t.Cleanup(func() { dotRootCAs = previous })
|
||||
|
||||
l, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: certs.TLS.Certificates, MinVersion: tls.VersionTLS12})
|
||||
require.NoError(t, err)
|
||||
server := &dns.Server{Listener: l, Net: "tcp-tls", Handler: dns.HandlerFunc(func(w dns.ResponseWriter, q *dns.Msg) {
|
||||
m := new(dns.Msg)
|
||||
m.SetReply(q)
|
||||
rr, _ := dns.NewRR(`example.com. 60 IN TXT "dot-answer"`)
|
||||
m.Answer = append(m.Answer, rr)
|
||||
_ = w.WriteMsg(m)
|
||||
})}
|
||||
go func() { _ = server.ActivateAndServe() }()
|
||||
t.Cleanup(func() { _ = server.Shutdown() })
|
||||
|
||||
options := dohOptions(t, nil)
|
||||
client, err := Get(options, &Configuration{Retries: 2, Resolvers: []string{"dot:" + l.Addr().String()}})
|
||||
require.NoError(t, err)
|
||||
resp, err := client.Do(txtQuery())
|
||||
require.NoError(t, err)
|
||||
require.Len(t, resp.Answer, 1)
|
||||
require.Contains(t, resp.Answer[0].String(), "dot-answer")
|
||||
}
|
||||
@@ -195,7 +195,7 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad
|
||||
// perform trace if necessary (excluded from duration — only the query RTT is measured)
|
||||
var traceData *retryabledns.TraceData
|
||||
if request.Trace {
|
||||
traceData, err = trace(request.options.Options.ExecutionId, request.dnsClient, domain, request.question, request.TraceMaxRecursion)
|
||||
traceData, err = request.dnsClient.Trace(domain, request.question, request.TraceMaxRecursion)
|
||||
if err != nil {
|
||||
request.options.Output.Request(request.options.TemplatePath, domain, "dns", err)
|
||||
}
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
)
|
||||
|
||||
// startDNSServer answers every query with a TXT record, or with the A records
|
||||
// in addrs when the question is for name, and counts the queries.
|
||||
func startDNSServer(t *testing.T, name string, addrs []string, hits *atomic.Int32) string {
|
||||
t.Helper()
|
||||
pc, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
server := &dns.Server{PacketConn: pc, Handler: dns.HandlerFunc(func(w dns.ResponseWriter, q *dns.Msg) {
|
||||
hits.Add(1)
|
||||
m := new(dns.Msg)
|
||||
m.SetReply(q)
|
||||
question := q.Question[0]
|
||||
switch {
|
||||
case question.Name == name && question.Qtype == dns.TypeA:
|
||||
for _, addr := range addrs {
|
||||
rr, _ := dns.NewRR(name + " 0 IN A " + addr)
|
||||
m.Answer = append(m.Answer, rr)
|
||||
}
|
||||
case question.Name == name:
|
||||
default:
|
||||
rr, _ := dns.NewRR(question.Name + ` 0 IN TXT "answer"`)
|
||||
m.Answer = append(m.Answer, rr)
|
||||
}
|
||||
_ = w.WriteMsg(m)
|
||||
})}
|
||||
go func() { _ = server.ActivateAndServe() }()
|
||||
t.Cleanup(func() { _ = server.Shutdown() })
|
||||
l, err := net.Listen("tcp", pc.LocalAddr().String())
|
||||
require.NoError(t, err)
|
||||
tcpServer := &dns.Server{Listener: l, Handler: server.Handler}
|
||||
go func() { _ = tcpServer.ActivateAndServe() }()
|
||||
t.Cleanup(func() { _ = tcpServer.Shutdown() })
|
||||
return pc.LocalAddr().String()
|
||||
}
|
||||
|
||||
// useSystemResolver sends the lookups of the process to the DNS server at address.
|
||||
func useSystemResolver(t *testing.T, address string) {
|
||||
t.Helper()
|
||||
previous := net.DefaultResolver
|
||||
net.DefaultResolver = &net.Resolver{
|
||||
PreferGo: true,
|
||||
Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, "udp", address)
|
||||
},
|
||||
}
|
||||
t.Cleanup(func() { net.DefaultResolver = previous })
|
||||
}
|
||||
|
||||
func TestDNSResolverNameObeysNetworkPolicy(t *testing.T) {
|
||||
// The resolver name has an allowed address and a denied address. The
|
||||
// policy check passes because one address is allowed, so the query must
|
||||
// not go to the denied one.
|
||||
var deniedHits, systemHits atomic.Int32
|
||||
denied := startDNSServer(t, "", nil, &deniedHits)
|
||||
_, port, err := net.SplitHostPort(denied)
|
||||
require.NoError(t, err)
|
||||
system := startDNSServer(t, "resolver.nuclei-test.", []string{"127.0.0.1", "127.0.0.2"}, &systemHits)
|
||||
useSystemResolver(t, system)
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.SetExecutionID(t.Name())
|
||||
options.ExcludeTargets = []string{"127.0.0.1"}
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() { testutils.Cleanup(options) })
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: "dns-resolver-policy",
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}, Name: "test"},
|
||||
})
|
||||
|
||||
for _, resolver := range []string{"resolver.nuclei-test:" + port, "tcp:resolver.nuclei-test:" + port} {
|
||||
t.Run(resolver, func(t *testing.T) {
|
||||
deniedHits.Store(0)
|
||||
request := &Request{
|
||||
RequestType: DNSRequestTypeHolder{DNSRequestType: TXT},
|
||||
Class: "INET",
|
||||
Retries: 2,
|
||||
Name: "{{FQDN}}",
|
||||
Resolvers: []string{resolver},
|
||||
}
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
msg := new(dns.Msg)
|
||||
msg.SetQuestion("example.com.", dns.TypeTXT)
|
||||
_, _ = request.dnsClient.Do(msg)
|
||||
require.Zero(t, deniedHits.Load(), "dns query reached a denied address")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSResolverNameLookupStopsAtTimeout(t *testing.T) {
|
||||
// The system resolver never answers, so the lookup that pins the resolver
|
||||
// name must stop at the timeout.
|
||||
silent, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = silent.Close() })
|
||||
useSystemResolver(t, silent.LocalAddr().String())
|
||||
|
||||
start := time.Now()
|
||||
_, err = pinResolverHost(t.Name(), "silent.nuclei-test:53", "silent.nuclei-test", time.Second)
|
||||
require.Error(t, err)
|
||||
require.Less(t, time.Since(start), 4*time.Second, "the resolver lookup did not stop at the timeout")
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math/rand"
|
||||
"net"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool"
|
||||
"github.com/projectdiscovery/retryabledns"
|
||||
sliceutil "github.com/projectdiscovery/utils/slice"
|
||||
)
|
||||
|
||||
// traceNameServerPort is the port of the name servers that a trace queries.
|
||||
var traceNameServerPort = "53"
|
||||
|
||||
// trace follows the delegation of host from the root servers in the same way
|
||||
// as retryabledns.Client.Trace, but it skips name server addresses that the
|
||||
// network policy denies. The name servers come from the answers, so the owner
|
||||
// of a zone could otherwise send the queries of a trace to internal addresses.
|
||||
func trace(executionID string, client *dnsclientpool.Client, host string, requestType uint16, maxRecursion int) (*retryabledns.TraceData, error) {
|
||||
var traceData retryabledns.TraceData
|
||||
host = dns.CanonicalName(host)
|
||||
servers := retryabledns.RootDNSServersIPv4
|
||||
seenNS := make(map[string]struct{})
|
||||
seenCName := make(map[string]int)
|
||||
for i := 1; i < maxRecursion; i++ {
|
||||
responses, err := client.QueryParallel(host, requestType, servers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, server := range servers {
|
||||
seenNS[server] = struct{}{}
|
||||
}
|
||||
if len(responses) == 0 {
|
||||
return &traceData, nil
|
||||
}
|
||||
for _, dnsData := range responses {
|
||||
if dnsData != nil && len(dnsData.Resolver) > 0 {
|
||||
traceData.DNSData = append(traceData.DNSData, dnsData)
|
||||
}
|
||||
}
|
||||
|
||||
var nameServers []string
|
||||
var nextCname string
|
||||
for _, dnsData := range responses {
|
||||
if dnsData == nil {
|
||||
continue
|
||||
}
|
||||
for _, ns := range dnsData.NS {
|
||||
ips, err := net.DefaultResolver.LookupIP(context.Background(), "ip4", ns)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if !protocolstate.IsHostAllowed(executionID, ip.String()) {
|
||||
continue
|
||||
}
|
||||
nameServers = append(nameServers, net.JoinHostPort(ip.String(), traceNameServerPort))
|
||||
}
|
||||
}
|
||||
if nextCname == "" && len(dnsData.CNAME) > 0 {
|
||||
nextCname = dnsData.CNAME[0]
|
||||
}
|
||||
}
|
||||
nameServers = sliceutil.Dedupe(nameServers)
|
||||
if len(nameServers) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
server := nameServers[rand.Intn(len(nameServers))] // #nosec G404
|
||||
if _, ok := seenNS[server]; ok && nextCname == "" {
|
||||
break
|
||||
}
|
||||
servers = []string{server}
|
||||
|
||||
if nextCname != "" {
|
||||
seenCName[nextCname]++
|
||||
if seenCName[nextCname] > retryabledns.DefaultMaxPerCNAMEFollows {
|
||||
break
|
||||
}
|
||||
host = nextCname
|
||||
}
|
||||
}
|
||||
return &traceData, nil
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/retryabledns"
|
||||
)
|
||||
|
||||
func serveTraceDNS(t *testing.T, address string, handler dns.HandlerFunc) string {
|
||||
t.Helper()
|
||||
pc, err := net.ListenPacket("udp", address)
|
||||
require.NoError(t, err)
|
||||
server := &dns.Server{PacketConn: pc, Handler: handler}
|
||||
go func() { _ = server.ActivateAndServe() }()
|
||||
t.Cleanup(func() { _ = server.Shutdown() })
|
||||
return pc.LocalAddr().String()
|
||||
}
|
||||
|
||||
func TestDNSTraceObeysNetworkPolicy(t *testing.T) {
|
||||
// The zone of the traced name delegates to a name server on 127.0.0.1.
|
||||
// The trace must query it when the policy allows the address, so that the
|
||||
// test can see a trace that ignores the policy, and must skip it when the
|
||||
// policy denies the address.
|
||||
t.Run("allowed", func(t *testing.T) {
|
||||
require.NotZero(t, runTraceAgainstLoopbackNameServer(t, nil), "trace did not query the allowed name server")
|
||||
})
|
||||
t.Run("denied", func(t *testing.T) {
|
||||
require.Zero(t, runTraceAgainstLoopbackNameServer(t, []string{"127.0.0.1"}), "trace queried a denied name server")
|
||||
})
|
||||
}
|
||||
|
||||
func runTraceAgainstLoopbackNameServer(t *testing.T, excludeTargets []string) int32 {
|
||||
t.Helper()
|
||||
var internalHits atomic.Int32
|
||||
internal := serveTraceDNS(t, "127.0.0.1:0", func(w dns.ResponseWriter, q *dns.Msg) {
|
||||
internalHits.Add(1)
|
||||
m := new(dns.Msg)
|
||||
m.SetReply(q)
|
||||
rr, _ := dns.NewRR(q.Question[0].Name + ` 0 IN TXT "internal"`)
|
||||
m.Answer = append(m.Answer, rr)
|
||||
_ = w.WriteMsg(m)
|
||||
})
|
||||
_, port, err := net.SplitHostPort(internal)
|
||||
require.NoError(t, err)
|
||||
previousPort := traceNameServerPort
|
||||
traceNameServerPort = port
|
||||
t.Cleanup(func() { traceNameServerPort = previousPort })
|
||||
|
||||
// public stands in for the root servers, the scan resolvers and the system resolver
|
||||
public := serveTraceDNS(t, "127.0.0.1:0", func(w dns.ResponseWriter, q *dns.Msg) {
|
||||
m := new(dns.Msg)
|
||||
m.SetReply(q)
|
||||
question := q.Question[0]
|
||||
var rr dns.RR
|
||||
switch {
|
||||
case question.Name == "ns.nuclei-test." && question.Qtype == dns.TypeA:
|
||||
rr, _ = dns.NewRR("ns.nuclei-test. 0 IN A 127.0.0.1")
|
||||
case question.Name == "ns.nuclei-test.":
|
||||
case question.Name == "trace.nuclei-test.":
|
||||
rr, _ = dns.NewRR("trace.nuclei-test. 0 IN NS ns.nuclei-test.")
|
||||
default:
|
||||
rr, _ = dns.NewRR(question.Name + ` 0 IN TXT "public"`)
|
||||
}
|
||||
if rr != nil {
|
||||
m.Answer = append(m.Answer, rr)
|
||||
}
|
||||
_ = w.WriteMsg(m)
|
||||
})
|
||||
previousRoots := retryabledns.RootDNSServersIPv4
|
||||
retryabledns.RootDNSServersIPv4 = []string{public}
|
||||
t.Cleanup(func() { retryabledns.RootDNSServersIPv4 = previousRoots })
|
||||
previousResolver := net.DefaultResolver
|
||||
net.DefaultResolver = &net.Resolver{
|
||||
PreferGo: true,
|
||||
Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, "udp", public)
|
||||
},
|
||||
}
|
||||
t.Cleanup(func() { net.DefaultResolver = previousResolver })
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.SetExecutionID(t.Name())
|
||||
options.ExcludeTargets = excludeTargets
|
||||
options.InternalResolversList = []string{public}
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() { testutils.Cleanup(options) })
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: "dns-trace-policy",
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}, Name: "test"},
|
||||
})
|
||||
|
||||
request := &Request{
|
||||
RequestType: DNSRequestTypeHolder{DNSRequestType: TXT},
|
||||
Class: "INET",
|
||||
Name: "{{FQDN}}",
|
||||
// One retry keeps the client keyed by this test's resolvers, so a
|
||||
// client cached by an earlier run cannot point at a closed server.
|
||||
Retries: 1,
|
||||
Trace: true,
|
||||
TraceMaxRecursion: 5,
|
||||
}
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
var event output.InternalEvent
|
||||
input := contextargs.NewWithInput(context.Background(), "trace.nuclei-test")
|
||||
err = request.ExecuteWithResults(input, nil, nil, func(e *output.InternalWrappedEvent) {
|
||||
event = e.InternalEvent
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, event["trace"], "ns.nuclei-test", "trace did not query the root servers")
|
||||
return internalHits.Load()
|
||||
}
|
||||
@@ -7,10 +7,10 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/go-rod/rod/lib/launcher/flags"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/launcher"
|
||||
"github.com/projectdiscovery/rod/lib/launcher/flags"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
|
||||
@@ -5,8 +5,8 @@ package engine
|
||||
import (
|
||||
"encoding/base64"
|
||||
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
)
|
||||
|
||||
// NewHijack create hijack from page.
|
||||
|
||||
@@ -5,11 +5,11 @@ import (
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/utils"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/utils"
|
||||
)
|
||||
|
||||
// Instance is an isolated browser instance opened for doing operations with it.
|
||||
|
||||
@@ -5,9 +5,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/cdp"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/cdp"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
@@ -19,8 +21,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
)
|
||||
|
||||
@@ -11,6 +11,10 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/input"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/go-rod/rod/lib/utils"
|
||||
"github.com/kitabisa/go-ci"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
@@ -20,10 +24,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
filepathutil "github.com/projectdiscovery/nuclei/v3/pkg/utils/filepath"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/input"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
"github.com/projectdiscovery/rod/lib/utils"
|
||||
contextutil "github.com/projectdiscovery/utils/context"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
|
||||
@@ -11,12 +11,12 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/cdp"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/cdp"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
||||
@@ -10,12 +10,12 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/projectdiscovery/networkpolicy"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/launcher"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
||||
@@ -6,10 +6,10 @@ import (
|
||||
"net/http/httputil"
|
||||
"strings"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/rod"
|
||||
"github.com/projectdiscovery/rod/lib/proto"
|
||||
)
|
||||
|
||||
// routingRuleHandler handles proxy rule for actions related to request/response modification
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/rod/lib/launcher/flags"
|
||||
"github.com/go-rod/rod/lib/launcher/flags"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
|
||||
)
|
||||
|
||||
// bodySizeEstimate is the largest request body a template has built so far.
|
||||
type bodySizeEstimate struct {
|
||||
known atomic.Bool
|
||||
max atomic.Int64
|
||||
learning sync.Mutex
|
||||
}
|
||||
|
||||
type bodyReservation struct {
|
||||
budget *httpbody.Budget
|
||||
size int64
|
||||
release func()
|
||||
unlock func()
|
||||
}
|
||||
|
||||
func noop() {}
|
||||
|
||||
// reserveBody reserves the template's largest body size before the body is
|
||||
// built. Until a size is known, builds run one at a time.
|
||||
func (request *Request) reserveBody(ctx context.Context) (*bodyReservation, error) {
|
||||
r := &bodyReservation{budget: request.options.RequestBodyBudget, release: noop, unlock: noop}
|
||||
estimate := request.bodySize
|
||||
if r.budget == nil || estimate == nil {
|
||||
return r, nil
|
||||
}
|
||||
if !estimate.known.Load() {
|
||||
estimate.learning.Lock()
|
||||
if !estimate.known.Load() {
|
||||
r.unlock = estimate.learning.Unlock
|
||||
return r, nil
|
||||
}
|
||||
estimate.learning.Unlock()
|
||||
}
|
||||
r.size = estimate.max.Load()
|
||||
release, err := r.budget.Acquire(ctx, r.size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.release = release
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// makeReserved builds the next request inside a body reservation, waiting
|
||||
// for budget until waitCtx ends. On success the caller owns the reservation
|
||||
// and calls done once the send ends.
|
||||
func (request *Request) makeReserved(waitCtx context.Context, generator *requestGenerator, input *contextargs.Context, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, *bodyReservation, error) {
|
||||
reservation, err := request.reserveBody(waitCtx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
generated, err := generator.Make(input.Context(), input, data, payloads, dynamicValues)
|
||||
if err == nil {
|
||||
err = reservation.built(waitCtx, request, generated.bodyLen())
|
||||
}
|
||||
if err != nil {
|
||||
reservation.done()
|
||||
return nil, nil, err
|
||||
}
|
||||
generated.bodyReserved = true
|
||||
return generated, reservation, nil
|
||||
}
|
||||
|
||||
// built grows the reservation to the body's real size. It releases before
|
||||
// acquiring again: waiting while holding part of the budget can deadlock.
|
||||
func (r *bodyReservation) built(ctx context.Context, request *Request, size int64) error {
|
||||
if estimate := request.bodySize; estimate != nil {
|
||||
for {
|
||||
seen := estimate.max.Load()
|
||||
if size <= seen || estimate.max.CompareAndSwap(seen, size) {
|
||||
break
|
||||
}
|
||||
}
|
||||
estimate.known.Store(true)
|
||||
}
|
||||
r.unlock()
|
||||
r.unlock = noop
|
||||
if size <= r.size {
|
||||
return nil
|
||||
}
|
||||
r.release()
|
||||
r.release = noop
|
||||
release, err := r.budget.Acquire(ctx, size)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.release, r.size = release, size
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *bodyReservation) done() {
|
||||
r.unlock()
|
||||
r.release()
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
//go:build !race
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
|
||||
)
|
||||
|
||||
// Threads and pipeline build bodies ahead of the send. Built bodies must stay
|
||||
// within the budget while the server holds the first request.
|
||||
func TestRequestBodyBudgetBoundsBodiesBuiltAhead(t *testing.T) {
|
||||
const bodySize = 8 << 20
|
||||
const payloads = 12
|
||||
|
||||
for _, mode := range []string{"threads", "pipeline"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
if mode == "pipeline" {
|
||||
t.Skip("needs projectdiscovery/rawhttp#551: the pipeline client hangs after 10s idle")
|
||||
}
|
||||
arrived := make(chan struct{}, payloads)
|
||||
unblock := make(chan struct{})
|
||||
var received atomic.Int32
|
||||
// net/http drops pipelined connections with large bodies, so serve
|
||||
// Content-Length framed requests in order by hand.
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = listener.Close() }()
|
||||
go func() {
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer func() { _ = conn.Close() }()
|
||||
reader := bufio.NewReader(conn)
|
||||
for {
|
||||
req, err := http.ReadRequest(reader)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, req.Body)
|
||||
received.Add(1)
|
||||
arrived <- struct{}{}
|
||||
<-unblock
|
||||
if _, err := io.WriteString(conn, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}()
|
||||
target := "http://" + listener.Addr().String()
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
testutils.Init(options)
|
||||
|
||||
values := make([]string, payloads)
|
||||
for i := range values {
|
||||
values[i] = fmt.Sprint(i)
|
||||
}
|
||||
request := &Request{
|
||||
ID: "body-budget-" + mode,
|
||||
Payloads: map[string]interface{}{"v": values},
|
||||
}
|
||||
body := `{{repeat("A", 8388608)}}`
|
||||
if mode == "threads" {
|
||||
request.Threads = 8
|
||||
request.Path = []string{"{{BaseURL}}/?v={{v}}"}
|
||||
request.Method = HTTPMethodTypeHolder{MethodType: HTTPPost}
|
||||
request.Body = body
|
||||
} else {
|
||||
request.Pipeline = true
|
||||
request.PipelineConcurrentConnections = 1
|
||||
request.PipelineRequestsPerConnection = payloads
|
||||
request.Raw = []string{"POST /?v={{v}} HTTP/1.1\nHost: {{Hostname}}\nContent-Length: 8388608\n\n" + body}
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: request.ID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
executerOpts.RequestBodyBudget = httpbody.NewBudget(bodySize)
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
var before runtime.MemStats
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&before)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), target), make(output.InternalEvent), make(output.InternalEvent), func(*output.InternalWrappedEvent) {})
|
||||
}()
|
||||
<-arrived
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
var held runtime.MemStats
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&held)
|
||||
close(unblock)
|
||||
require.NoError(t, <-done)
|
||||
|
||||
grown := int64(held.HeapAlloc) - int64(before.HeapAlloc)
|
||||
t.Logf("heap held: %d MB", grown>>20)
|
||||
require.Less(t, grown, int64(4*bodySize), "bodies built ahead of the send must stay within the budget")
|
||||
require.EqualValues(t, payloads, received.Load())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,200 +0,0 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
|
||||
)
|
||||
|
||||
// The third request queues ~2.4s, so it only beats the 3s timeout if the
|
||||
// timeout starts after admission.
|
||||
func TestRequestBodyBudgetQueuesHeavyRequests(t *testing.T) {
|
||||
const bodySize = 1 << 20
|
||||
const requests = 3
|
||||
|
||||
var inFlight, maxInFlight atomic.Int32
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
n := inFlight.Add(1)
|
||||
defer inFlight.Add(-1)
|
||||
for {
|
||||
seen := maxInFlight.Load()
|
||||
if n <= seen || maxInFlight.CompareAndSwap(seen, n) {
|
||||
break
|
||||
}
|
||||
}
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
time.Sleep(1200 * time.Millisecond)
|
||||
if len(body) == bodySize {
|
||||
_, _ = io.WriteString(w, "full body")
|
||||
}
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.SetTimeouts(&types.Timeouts{DialTimeout: time.Second, HttpTimeout: 3 * time.Second})
|
||||
testutils.Init(options)
|
||||
|
||||
request := &Request{
|
||||
ID: "body-budget",
|
||||
Path: []string{"{{BaseURL}}"},
|
||||
Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
|
||||
Body: `{{repeat("A", 1048576)}}`,
|
||||
Operators: operators.Operators{
|
||||
Matchers: []*matchers.Matcher{{
|
||||
Part: "body",
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
|
||||
Words: []string{"full body"},
|
||||
}},
|
||||
},
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: request.ID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
executerOpts.RequestBodyBudget = httpbody.NewBudget(bodySize)
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
var matched atomic.Int32
|
||||
var wg sync.WaitGroup
|
||||
for range requests {
|
||||
wg.Go(func() {
|
||||
err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), ts.URL), make(output.InternalEvent), make(output.InternalEvent), func(event *output.InternalWrappedEvent) {
|
||||
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
|
||||
matched.Add(1)
|
||||
}
|
||||
})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
require.EqualValues(t, 1, maxInFlight.Load(), "budget must hold heavy requests to one in flight")
|
||||
require.EqualValues(t, requests, matched.Load(), "queued requests must not time out while waiting")
|
||||
}
|
||||
|
||||
func TestReserveBodyQueuesBuildsOnLearnedSize(t *testing.T) {
|
||||
const bodySize = 4 << 20
|
||||
request := &Request{
|
||||
options: &protocols.ExecutorOptions{RequestBodyBudget: httpbody.NewBudget(6 << 20)},
|
||||
bodySize: &bodySizeEstimate{},
|
||||
}
|
||||
|
||||
first, err := request.reserveBody(t.Context())
|
||||
require.NoError(t, err)
|
||||
blocked := make(chan struct{})
|
||||
go func() {
|
||||
second, err := request.reserveBody(t.Context())
|
||||
require.NoError(t, err)
|
||||
second.done()
|
||||
close(blocked)
|
||||
}()
|
||||
select {
|
||||
case <-blocked:
|
||||
t.Fatal("second build started before the first recorded its size")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
require.NoError(t, first.built(t.Context(), request, bodySize))
|
||||
first.done()
|
||||
<-blocked
|
||||
|
||||
held, err := request.reserveBody(t.Context())
|
||||
require.NoError(t, err)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
_, err = request.reserveBody(ctx)
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
|
||||
held.done()
|
||||
next, err := request.reserveBody(t.Context())
|
||||
require.NoError(t, err)
|
||||
next.done()
|
||||
}
|
||||
|
||||
func TestReserveBodyGrowsToLargerBody(t *testing.T) {
|
||||
request := &Request{
|
||||
options: &protocols.ExecutorOptions{RequestBodyBudget: httpbody.NewBudget(6 << 20)},
|
||||
bodySize: &bodySizeEstimate{},
|
||||
}
|
||||
learn, err := request.reserveBody(t.Context())
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, learn.built(t.Context(), request, 1<<20))
|
||||
learn.done()
|
||||
|
||||
grown, err := request.reserveBody(t.Context())
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, grown.built(t.Context(), request, 5<<20))
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
_, err = request.reserveBody(ctx)
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
grown.done()
|
||||
}
|
||||
|
||||
// Stop-at-first-match ends the workers with tasks still queued. Their
|
||||
// reservations must go back to the shared budget.
|
||||
func TestRequestBodyBudgetStopAtFirstMatchReleasesQueued(t *testing.T) {
|
||||
const budget = 16 << 20
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
_, _ = io.WriteString(w, "match")
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
testutils.Init(options)
|
||||
|
||||
values := make([]string, 64)
|
||||
for i := range values {
|
||||
values[i] = fmt.Sprint(i)
|
||||
}
|
||||
request := &Request{
|
||||
ID: "body-budget-stop-first",
|
||||
Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
|
||||
Path: []string{"{{BaseURL}}/?v={{v}}"},
|
||||
Body: `{{repeat("A", 1048576)}}`,
|
||||
Threads: 4,
|
||||
Payloads: map[string]interface{}{"v": values},
|
||||
StopAtFirstMatch: true,
|
||||
Operators: operators.Operators{Matchers: []*matchers.Matcher{{
|
||||
Part: "body",
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
|
||||
Words: []string{"match"},
|
||||
}}},
|
||||
}
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: request.ID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
executerOpts.RequestBodyBudget = httpbody.NewBudget(budget)
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), ts.URL), make(output.InternalEvent), make(output.InternalEvent), func(*output.InternalWrappedEvent) {})
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
|
||||
defer cancel()
|
||||
release, err := executerOpts.RequestBodyBudget.Acquire(ctx, budget)
|
||||
require.NoError(t, err, "every reservation must be released")
|
||||
release()
|
||||
}
|
||||
@@ -82,9 +82,6 @@ type generatedRequest struct {
|
||||
dynamicValues map[string]interface{}
|
||||
interactshURLs []string
|
||||
customCancelFunction context.CancelFunc
|
||||
// sendTimeout starts after the body budget admits the request.
|
||||
sendTimeout time.Duration
|
||||
bodyReserved bool
|
||||
// requestURLPattern tracks unmodified request url pattern without values ( it is used for constant vuln_hash)
|
||||
// ex: {{BaseURL}}/api/exp?param={{randstr}}
|
||||
requestURLPattern string
|
||||
|
||||
@@ -147,10 +147,6 @@ type Request struct {
|
||||
rawhttpClient *rawhttp.Client
|
||||
replayClient *http.Client // set when -replay-proxy is given
|
||||
dialer *fastdialer.Dialer
|
||||
bodySize *bodySizeEstimate
|
||||
// sharesResponse lets other templates reuse a response this request gets,
|
||||
// and this request reuse theirs (see httpclientpool.WithSharedResponse)
|
||||
sharesResponse bool
|
||||
|
||||
// description: |
|
||||
// SelfContained specifies if the request is self-contained.
|
||||
@@ -420,7 +416,6 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
|
||||
return errors.Wrap(err, "could not get dialer")
|
||||
}
|
||||
request.dialer = dialer
|
||||
request.bodySize = &bodySizeEstimate{}
|
||||
|
||||
request.options = options
|
||||
for _, option := range request.options.Options.CustomHeaders {
|
||||
@@ -590,24 +585,9 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
|
||||
if shouldDisableKeepAliveForHTTPProxy(request, options) {
|
||||
request.connConfiguration.Connection.DisableKeepAlive = true
|
||||
}
|
||||
request.sharesResponse = canShareResponse(request, options.Flow)
|
||||
return nil
|
||||
}
|
||||
|
||||
// canShareResponse reports whether a response another template got for the
|
||||
// same request is as good as sending it again. A shared response arrives at
|
||||
// once, so nothing that measures time may take one: race requests, timing
|
||||
// analyzers, and a request whose operators or flow read the duration. The
|
||||
// operators are searched whole, so a new field that can read it, as LLM
|
||||
// matcher inputs do, is covered without being listed.
|
||||
func canShareResponse(request *Request, flow string) bool {
|
||||
if request.Race || request.Analyzer != nil || strings.Contains(flow, "duration") {
|
||||
return false
|
||||
}
|
||||
operators, err := json.Marshal(request.Operators)
|
||||
return err == nil && !bytes.Contains(operators, []byte("duration"))
|
||||
}
|
||||
|
||||
// RebuildGenerator rebuilds the generator for the request
|
||||
func (request *Request) RebuildGenerator() error {
|
||||
generator, err := generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options)
|
||||
|
||||
@@ -514,11 +514,7 @@ func wrappedGet(options *types.Options, configuration *Configuration, host strin
|
||||
}
|
||||
}
|
||||
|
||||
tracking := &connTrackingTransport{base: transport}
|
||||
if shared := getSharedResponses(options); shared != nil {
|
||||
return &sharingTransport{base: tracking, shared: shared}, nil
|
||||
}
|
||||
return tracking, nil
|
||||
return &connTrackingTransport{base: transport}, nil
|
||||
}
|
||||
redirectFlow := configuration.RedirectFlow
|
||||
maxRedirects := configuration.MaxRedirects
|
||||
|
||||
@@ -175,9 +175,7 @@ func TestIdleConnPoolSizeIsWiredToTransport(t *testing.T) {
|
||||
client, err := Get(opts, &Configuration{}, "example.com")
|
||||
require.NoError(t, err)
|
||||
|
||||
sharing, ok := client.HTTPClient.Transport.(*sharingTransport)
|
||||
require.True(t, ok, "expected the shared-response transport wrapper")
|
||||
tracking, ok := sharing.base.(*connTrackingTransport)
|
||||
tracking, ok := client.HTTPClient.Transport.(*connTrackingTransport)
|
||||
require.True(t, ok, "expected the connection-tracking transport wrapper")
|
||||
transport, ok := tracking.base.(*http.Transport)
|
||||
require.True(t, ok, "expected an *http.Transport underneath")
|
||||
|
||||
@@ -1,281 +0,0 @@
|
||||
package httpclientpool
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"container/list"
|
||||
"context"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/useragent"
|
||||
)
|
||||
|
||||
// Templates send many requests another template already sent to the same
|
||||
// target, byte for byte: clustering merges only single-request templates. A
|
||||
// request marked with WithSharedResponse is sent once per scan and every
|
||||
// other template sending the same bytes gets a copy of the response. The key
|
||||
// is the request as it goes on the wire, cookies included, and the client
|
||||
// still processes each copy, so redirects and cookies behave as if it was
|
||||
// sent again.
|
||||
const (
|
||||
// sharedResponseMaxBody keeps large downloads out of the store; they are
|
||||
// sent each time
|
||||
sharedResponseMaxBody = 1 << 20
|
||||
sharedResponsesMaxBytes = 64 << 20
|
||||
)
|
||||
|
||||
type sharedResponseKey struct{}
|
||||
|
||||
// WithSharedResponse marks the request of templateID as one whose response
|
||||
// other templates may reuse. A template sending the same request again gets
|
||||
// it sent again: a repeat within one template is intended.
|
||||
func WithSharedResponse(ctx context.Context, templateID string) context.Context {
|
||||
return context.WithValue(ctx, sharedResponseKey{}, templateID)
|
||||
}
|
||||
|
||||
type sharedResponses struct {
|
||||
mu sync.Mutex
|
||||
entries map[string]*sharedEntry
|
||||
// order holds stored entries oldest first, for eviction
|
||||
order *list.List
|
||||
bytes int
|
||||
maxBytes int
|
||||
}
|
||||
|
||||
type sharedEntry struct {
|
||||
key string
|
||||
templateID string
|
||||
ready chan struct{}
|
||||
response *storedResponse
|
||||
}
|
||||
|
||||
type storedResponse struct {
|
||||
response *http.Response
|
||||
body []byte
|
||||
}
|
||||
|
||||
func getSharedResponses(options *types.Options) *sharedResponses {
|
||||
dialers := protocolstate.GetDialersWithId(options.ExecutionId)
|
||||
if dialers == nil {
|
||||
return nil
|
||||
}
|
||||
dialers.Lock()
|
||||
defer dialers.Unlock()
|
||||
if dialers.SharedResponses == nil {
|
||||
dialers.SharedResponses = &sharedResponses{entries: make(map[string]*sharedEntry), order: list.New(), maxBytes: sharedResponsesMaxBytes}
|
||||
}
|
||||
shared, _ := dialers.SharedResponses.(*sharedResponses)
|
||||
return shared
|
||||
}
|
||||
|
||||
// ResetSharedResponses forgets the responses of earlier scans, so a scan
|
||||
// that reuses the execution, as an SDK engine does, sends its requests again.
|
||||
func ResetSharedResponses(options *types.Options) {
|
||||
shared := getSharedResponses(options)
|
||||
if shared == nil {
|
||||
return
|
||||
}
|
||||
shared.mu.Lock()
|
||||
defer shared.mu.Unlock()
|
||||
clear(shared.entries)
|
||||
shared.order.Init()
|
||||
shared.bytes = 0
|
||||
}
|
||||
|
||||
type sharingTransport struct {
|
||||
base http.RoundTripper
|
||||
shared *sharedResponses
|
||||
}
|
||||
|
||||
func (t *sharingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
templateID, marked := req.Context().Value(sharedResponseKey{}).(string)
|
||||
if !marked || (req.Method != http.MethodGet && req.Method != http.MethodHead) || hasBody(req) {
|
||||
return t.base.RoundTrip(req)
|
||||
}
|
||||
key := sharedRequestKey(req)
|
||||
|
||||
t.shared.mu.Lock()
|
||||
entry, found := t.shared.entries[key]
|
||||
if !found {
|
||||
entry = &sharedEntry{key: key, templateID: templateID, ready: make(chan struct{})}
|
||||
t.shared.entries[key] = entry
|
||||
t.shared.mu.Unlock()
|
||||
return t.send(req, entry)
|
||||
}
|
||||
t.shared.mu.Unlock()
|
||||
if entry.templateID == templateID {
|
||||
return t.base.RoundTrip(req)
|
||||
}
|
||||
select {
|
||||
case <-entry.ready:
|
||||
case <-req.Context().Done():
|
||||
return nil, req.Context().Err()
|
||||
}
|
||||
if entry.response == nil {
|
||||
// the first send failed or was too large to keep
|
||||
return t.base.RoundTrip(req)
|
||||
}
|
||||
return entry.response.copyFor(req), nil
|
||||
}
|
||||
|
||||
func (t *sharingTransport) CloseIdleConnections() {
|
||||
type closeIdler interface{ CloseIdleConnections() }
|
||||
if ci, ok := t.base.(closeIdler); ok {
|
||||
ci.CloseIdleConnections()
|
||||
}
|
||||
}
|
||||
|
||||
// send sends the first request of key and stores its response for the rest.
|
||||
func (t *sharingTransport) send(req *http.Request, entry *sharedEntry) (*http.Response, error) {
|
||||
defer close(entry.ready)
|
||||
resp, err := t.base.RoundTrip(req)
|
||||
if err != nil {
|
||||
t.forget(entry)
|
||||
return resp, err
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, sharedResponseMaxBody+1))
|
||||
if err != nil || len(body) > sharedResponseMaxBody || !shareableStatus(resp.StatusCode) {
|
||||
t.forget(entry)
|
||||
// The bytes already taken off the body have to go back in front of it.
|
||||
// A short read is still this caller's response: a Content-Length that
|
||||
// is longer than the body is what some matchers look for. It is not
|
||||
// stored, and neither is a body too large to keep or a refusal.
|
||||
resp.Body = struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{io.MultiReader(bytes.NewReader(body), resp.Body), resp.Body}
|
||||
return resp, nil
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
entry.response = &storedResponse{response: resp, body: body}
|
||||
t.store(entry)
|
||||
return entry.response.copyFor(req), nil
|
||||
}
|
||||
|
||||
// shareableStatus reports whether code is an answer a later template can
|
||||
// reuse. 404 and 500 are answers templates match on. 429 and the gateway
|
||||
// errors mean the server did not answer, so the next template must send.
|
||||
func shareableStatus(code int) bool {
|
||||
switch code {
|
||||
case http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// hasBody reports whether req sends a body. retryablehttp gives every
|
||||
// request one, empty for a GET, so its content is what counts.
|
||||
func hasBody(req *http.Request) bool {
|
||||
if req.Body == nil || req.Body == http.NoBody {
|
||||
return false
|
||||
}
|
||||
if req.ContentLength > 0 || req.GetBody == nil {
|
||||
return true
|
||||
}
|
||||
body, err := req.GetBody()
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
defer func() { _ = body.Close() }()
|
||||
var first [1]byte
|
||||
n, _ := body.Read(first[:])
|
||||
return n > 0
|
||||
}
|
||||
|
||||
func (t *sharingTransport) forget(entry *sharedEntry) {
|
||||
t.shared.mu.Lock()
|
||||
defer t.shared.mu.Unlock()
|
||||
if t.shared.entries[entry.key] == entry {
|
||||
delete(t.shared.entries, entry.key)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *sharingTransport) store(entry *sharedEntry) {
|
||||
t.shared.mu.Lock()
|
||||
defer t.shared.mu.Unlock()
|
||||
// a reset while it was in flight dropped it
|
||||
if t.shared.entries[entry.key] != entry {
|
||||
return
|
||||
}
|
||||
t.shared.order.PushBack(entry)
|
||||
t.shared.bytes += entry.size()
|
||||
for t.shared.bytes > t.shared.maxBytes && t.shared.order.Len() > 1 {
|
||||
oldest := t.shared.order.Remove(t.shared.order.Front()).(*sharedEntry)
|
||||
t.shared.bytes -= oldest.size()
|
||||
if t.shared.entries[oldest.key] == oldest {
|
||||
delete(t.shared.entries, oldest.key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sharedEntryOverhead stands for the response struct and bookkeeping, so
|
||||
// bodyless responses, as every HEAD gets, still count toward the cap.
|
||||
const sharedEntryOverhead = 512
|
||||
|
||||
func (e *sharedEntry) size() int {
|
||||
size := sharedEntryOverhead + len(e.key) + len(e.response.body)
|
||||
for name, values := range e.response.response.Header {
|
||||
size += len(name)
|
||||
for _, value := range values {
|
||||
size += len(value)
|
||||
}
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
func (s *storedResponse) copyFor(req *http.Request) *http.Response {
|
||||
resp := *s.response
|
||||
resp.Header = s.response.Header.Clone()
|
||||
resp.Trailer = s.response.Trailer.Clone()
|
||||
resp.Body = io.NopCloser(bytes.NewReader(s.body))
|
||||
// A HEAD response has an empty body and a Content-Length for the body it
|
||||
// did not send. Matchers read that length, so a reused HEAD keeps it.
|
||||
if req.Method != http.MethodHead {
|
||||
resp.ContentLength = int64(len(s.body))
|
||||
}
|
||||
resp.Request = req
|
||||
return &resp
|
||||
}
|
||||
|
||||
// randomUserAgents are the values nuclei picks a User-Agent from for every
|
||||
// request a template does not set one for. Two templates sending the same
|
||||
// request almost never pick the same one, and either could have picked the
|
||||
// other's, so a picked User-Agent is left out of the key.
|
||||
var randomUserAgents = sync.OnceValue(func() map[string]struct{} {
|
||||
values := make(map[string]struct{}, len(useragent.UserAgents))
|
||||
for _, userAgent := range useragent.UserAgents {
|
||||
values[userAgent.Raw] = struct{}{}
|
||||
}
|
||||
return values
|
||||
})
|
||||
|
||||
// sharedRequestKey is the request as the server receives it, apart from the
|
||||
// order of headers and a picked User-Agent.
|
||||
func sharedRequestKey(req *http.Request) string {
|
||||
var key strings.Builder
|
||||
key.WriteString(req.Method)
|
||||
key.WriteByte(' ')
|
||||
key.WriteString(req.URL.String())
|
||||
key.WriteByte('\n')
|
||||
key.WriteString(req.Host)
|
||||
for _, name := range slices.Sorted(maps.Keys(req.Header)) {
|
||||
values := req.Header[name]
|
||||
if _, picked := randomUserAgents()[req.Header.Get("User-Agent")]; name == "User-Agent" && len(values) == 1 && picked {
|
||||
continue
|
||||
}
|
||||
for _, value := range values {
|
||||
key.WriteByte('\n')
|
||||
key.WriteString(name)
|
||||
key.WriteString(": ")
|
||||
key.WriteString(value)
|
||||
}
|
||||
}
|
||||
return key.String()
|
||||
}
|
||||
@@ -1,308 +0,0 @@
|
||||
package httpclientpool
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/projectdiscovery/useragent"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newSharingTransport(base http.RoundTripper) *sharingTransport {
|
||||
return &sharingTransport{base: base, shared: &sharedResponses{entries: make(map[string]*sharedEntry), order: list.New(), maxBytes: sharedResponsesMaxBytes}}
|
||||
}
|
||||
|
||||
func sharedGet(t *testing.T, transport http.RoundTripper, method, url, templateID string, header http.Header) string {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
if templateID != "" {
|
||||
ctx = WithSharedResponse(ctx, templateID)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, nil)
|
||||
require.NoError(t, err)
|
||||
for name, values := range header {
|
||||
req.Header[name] = values
|
||||
}
|
||||
resp, err := transport.RoundTrip(req)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
return string(body)
|
||||
}
|
||||
|
||||
func TestSharedResponsesSendEachRequestOncePerScan(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
n := hits.Add(1)
|
||||
_, _ = io.WriteString(w, r.URL.Path+strings.Repeat("!", int(n)))
|
||||
}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(http.DefaultTransport)
|
||||
|
||||
first := sharedGet(t, transport, http.MethodGet, server.URL+"/a", "one", nil)
|
||||
require.Equal(t, first, sharedGet(t, transport, http.MethodGet, server.URL+"/a", "two", nil), "another template gets the stored response")
|
||||
require.EqualValues(t, 1, hits.Load())
|
||||
|
||||
// nuclei builds a GET with an empty body
|
||||
empty, err := retryablehttp.NewRequestWithContext(WithSharedResponse(context.Background(), "four"), http.MethodGet, server.URL+"/a", strings.NewReader(""))
|
||||
require.NoError(t, err)
|
||||
resp, err := transport.RoundTrip(empty.Request)
|
||||
require.NoError(t, err)
|
||||
_ = resp.Body.Close()
|
||||
require.EqualValues(t, 1, hits.Load(), "an empty body is no body")
|
||||
|
||||
sharedGet(t, transport, http.MethodGet, server.URL+"/a", "one", nil)
|
||||
require.EqualValues(t, 2, hits.Load(), "a template repeating its own request sends it")
|
||||
|
||||
sharedGet(t, transport, http.MethodGet, server.URL+"/a", "", nil)
|
||||
require.EqualValues(t, 3, hits.Load(), "an unmarked request is always sent")
|
||||
|
||||
sharedGet(t, transport, http.MethodGet, server.URL+"/a", "three", http.Header{"Cookie": {"session=1"}})
|
||||
require.EqualValues(t, 4, hits.Load(), "other cookies make another request")
|
||||
|
||||
picked := func(i int) http.Header { return http.Header{"User-Agent": {useragent.UserAgents[i].Raw}} }
|
||||
sharedGet(t, transport, http.MethodGet, server.URL+"/b", "one", picked(0))
|
||||
sharedGet(t, transport, http.MethodGet, server.URL+"/b", "two", picked(1))
|
||||
require.EqualValues(t, 5, hits.Load(), "a User-Agent nuclei picked at random is not part of the request")
|
||||
sharedGet(t, transport, http.MethodGet, server.URL+"/b", "three", http.Header{"User-Agent": {"custom-scanner"}})
|
||||
require.EqualValues(t, 6, hits.Load(), "one a template sets is")
|
||||
|
||||
for range 2 {
|
||||
// A new request each time. Clone shares the body, so a second clone
|
||||
// of one request can send ContentLength with an already-read body.
|
||||
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), "one"), http.MethodPost, server.URL+"/a", strings.NewReader("x"))
|
||||
require.NoError(t, err)
|
||||
resp, err := transport.RoundTrip(req)
|
||||
require.NoError(t, err)
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
require.EqualValues(t, 8, hits.Load(), "a request with a body is always sent")
|
||||
}
|
||||
|
||||
func TestSharedResponsesWaitForTheRequestInFlight(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
release := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
<-release
|
||||
_, _ = io.WriteString(w, "slow")
|
||||
}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(http.DefaultTransport)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
bodies := make([]string, 5)
|
||||
for i := range bodies {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
bodies[i] = sharedGet(t, transport, http.MethodGet, server.URL, "template-"+string(rune('a'+i)), nil)
|
||||
}()
|
||||
}
|
||||
require.Eventually(t, func() bool { return hits.Load() > 0 }, 5*time.Second, time.Millisecond)
|
||||
close(release)
|
||||
wg.Wait()
|
||||
require.EqualValues(t, 1, hits.Load())
|
||||
require.Equal(t, []string{"slow", "slow", "slow", "slow", "slow"}, bodies)
|
||||
}
|
||||
|
||||
func TestSharedResponsesSendLargeBodiesEachTime(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
large := strings.Repeat("x", sharedResponseMaxBody+10)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
_, _ = io.WriteString(w, large)
|
||||
}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(http.DefaultTransport)
|
||||
|
||||
require.Equal(t, large, sharedGet(t, transport, http.MethodGet, server.URL, "one", nil), "the whole body still arrives")
|
||||
require.Equal(t, large, sharedGet(t, transport, http.MethodGet, server.URL, "two", nil))
|
||||
require.EqualValues(t, 2, hits.Load())
|
||||
}
|
||||
|
||||
type failOnce struct {
|
||||
failed atomic.Bool
|
||||
base http.RoundTripper
|
||||
}
|
||||
|
||||
func (f *failOnce) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
if f.failed.CompareAndSwap(false, true) {
|
||||
return nil, errors.New("connection reset")
|
||||
}
|
||||
return f.base.RoundTrip(req)
|
||||
}
|
||||
|
||||
func TestSharedResponsesDoNotKeepFailures(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.WriteString(w, "ok")
|
||||
}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(&failOnce{base: http.DefaultTransport})
|
||||
|
||||
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), "one"), http.MethodGet, server.URL, nil)
|
||||
require.NoError(t, err)
|
||||
_, err = transport.RoundTrip(req)
|
||||
require.Error(t, err)
|
||||
require.Equal(t, "ok", sharedGet(t, transport, http.MethodGet, server.URL, "two", nil), "the next template sends it again")
|
||||
}
|
||||
|
||||
func TestSharedResponsesStartEmptyEachScan(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
_, _ = io.WriteString(w, "ok")
|
||||
}))
|
||||
defer server.Close()
|
||||
opts := newTestOptions(t, "test-shared-responses-reset")
|
||||
transport := &sharingTransport{base: http.DefaultTransport, shared: getSharedResponses(opts)}
|
||||
|
||||
sharedGet(t, transport, http.MethodGet, server.URL, "one", nil)
|
||||
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
|
||||
require.EqualValues(t, 1, hits.Load())
|
||||
|
||||
// an SDK engine runs its next scan on the same execution
|
||||
ResetSharedResponses(opts)
|
||||
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
|
||||
require.EqualValues(t, 2, hits.Load(), "a new scan does not see the responses of the last one")
|
||||
}
|
||||
|
||||
func TestSharedResponsesCapCountsBodylessResponses(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(http.DefaultTransport)
|
||||
transport.shared.maxBytes = 20 * sharedEntryOverhead
|
||||
|
||||
for i := range 40 {
|
||||
sharedGet(t, transport, http.MethodHead, fmt.Sprintf("%s/%d", server.URL, i), "one", nil)
|
||||
}
|
||||
require.LessOrEqual(t, transport.shared.bytes, transport.shared.maxBytes)
|
||||
require.Less(t, len(transport.shared.entries), 20, "HEAD responses are evicted too")
|
||||
}
|
||||
|
||||
func TestSharedResponsesKeepHeadContentLength(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.Header().Set("Content-Length", "1234")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(http.DefaultTransport)
|
||||
|
||||
length := func(templateID string) int64 {
|
||||
t.Helper()
|
||||
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), templateID), http.MethodHead, server.URL, nil)
|
||||
require.NoError(t, err)
|
||||
resp, err := transport.RoundTrip(req)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = resp.Body.Close() })
|
||||
return resp.ContentLength
|
||||
}
|
||||
|
||||
require.Equal(t, int64(1234), length("one"))
|
||||
require.Equal(t, int64(1234), length("two"), "a reused HEAD keeps the length the server advertised")
|
||||
require.EqualValues(t, 1, hits.Load())
|
||||
}
|
||||
|
||||
func TestSharedResponsesDoNotKeepRefusals(t *testing.T) {
|
||||
for _, status := range []int{http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout} {
|
||||
t.Run(http.StatusText(status), func(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
http.Error(w, "no", status)
|
||||
}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(http.DefaultTransport)
|
||||
|
||||
sharedGet(t, transport, http.MethodGet, server.URL, "one", nil)
|
||||
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
|
||||
require.EqualValues(t, 2, hits.Load())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedResponsesKeepNotFound(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer server.Close()
|
||||
transport := newSharingTransport(http.DefaultTransport)
|
||||
|
||||
sharedGet(t, transport, http.MethodGet, server.URL, "one", nil)
|
||||
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
|
||||
require.EqualValues(t, 1, hits.Load())
|
||||
}
|
||||
|
||||
// shortBody yields a prefix and then ErrUnexpectedEOF, which is what a
|
||||
// response with a Content-Length longer than its body looks like.
|
||||
type shortBody struct {
|
||||
prefix []byte
|
||||
sent bool
|
||||
}
|
||||
|
||||
func (s *shortBody) Read(p []byte) (int, error) {
|
||||
if !s.sent {
|
||||
s.sent = true
|
||||
n := copy(p, s.prefix)
|
||||
return n, io.ErrUnexpectedEOF
|
||||
}
|
||||
return 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
func (s *shortBody) Close() error { return nil }
|
||||
|
||||
type countingTransport struct {
|
||||
hits atomic.Int32
|
||||
}
|
||||
|
||||
func (c *countingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
c.hits.Add(1)
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: http.Header{"Content-Length": {"50000"}},
|
||||
ContentLength: 50000,
|
||||
Body: &shortBody{prefix: []byte("fourteen bytes")},
|
||||
Request: req,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func TestSharedResponsesDoNotKeepShortReads(t *testing.T) {
|
||||
base := &countingTransport{}
|
||||
transport := newSharingTransport(base)
|
||||
|
||||
send := func(templateID string) *http.Response {
|
||||
t.Helper()
|
||||
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), templateID), http.MethodGet, "http://example.com", nil)
|
||||
require.NoError(t, err)
|
||||
resp, err := transport.RoundTrip(req)
|
||||
require.NoError(t, err)
|
||||
return resp
|
||||
}
|
||||
|
||||
first := send("one")
|
||||
body, err := io.ReadAll(first.Body)
|
||||
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
require.Equal(t, "fourteen bytes", string(body))
|
||||
require.Equal(t, int64(50000), first.ContentLength)
|
||||
_ = first.Body.Close()
|
||||
|
||||
second := send("two")
|
||||
_ = second.Body.Close()
|
||||
require.EqualValues(t, 2, base.hits.Load(), "a short read is not reused")
|
||||
}
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
convUtil "github.com/projectdiscovery/utils/conversion"
|
||||
)
|
||||
|
||||
// Match matches a generic data response again a given matcher
|
||||
@@ -226,7 +225,7 @@ func (request *Request) savedRequest(dump string) string {
|
||||
if request.options == nil || request.options.Options == nil {
|
||||
return dump
|
||||
}
|
||||
return string(capDumpBody(convUtil.Bytes(dump), request.options.Options.RequestSaveSize))
|
||||
return string(capDumpBody([]byte(dump), request.options.Options.RequestSaveSize))
|
||||
}
|
||||
|
||||
// eventResponse returns the full response of event. Events matched after the
|
||||
|
||||
+80
-121
@@ -331,7 +331,6 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
|
||||
// bounded worker-pool to avoid spawning one goroutine per payload
|
||||
type task struct {
|
||||
req *generatedRequest
|
||||
body *bodyReservation
|
||||
updatedInput *contextargs.Context
|
||||
hasInteractMarkers bool
|
||||
}
|
||||
@@ -339,87 +338,72 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
|
||||
var workersWg sync.WaitGroup
|
||||
currentWorkers := maxWorkers
|
||||
tasks := make(chan task, maxWorkers)
|
||||
// Workers exit on cancel without draining, and the budget is shared, so
|
||||
// release what they left queued.
|
||||
stopWorkers := func() {
|
||||
close(tasks)
|
||||
workersWg.Wait()
|
||||
for t := range tasks {
|
||||
t.body.done()
|
||||
}
|
||||
}
|
||||
spawnWorker := func(ctx context.Context) {
|
||||
workersWg.Add(1)
|
||||
go func() {
|
||||
defer workersWg.Done()
|
||||
for t := range tasks {
|
||||
if func() (exit bool) {
|
||||
defer t.body.done()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return true
|
||||
default:
|
||||
}
|
||||
if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
|
||||
return false
|
||||
}
|
||||
if err := spmHandler.Acquire(); err != nil {
|
||||
return false
|
||||
}
|
||||
if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
|
||||
spmHandler.Release()
|
||||
return false
|
||||
}
|
||||
// Full URL, so https and an explicit port stay on the same key the observation uses.
|
||||
// The handler context ends the wait when stop-at-first-match fires.
|
||||
waitCtx, cancelWait := stopWithHandler(ctx, spmHandler)
|
||||
err := request.rateLimitTake(waitCtx, hostBackoffKey(t.req, t.updatedInput))
|
||||
cancelWait()
|
||||
if err != nil {
|
||||
select {
|
||||
case <-spmHandler.Done():
|
||||
spmHandler.Release()
|
||||
return false
|
||||
case <-ctx.Done():
|
||||
spmHandler.Release()
|
||||
return false
|
||||
case spmHandler.ResultChan <- err:
|
||||
spmHandler.Release()
|
||||
return false
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil || spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
|
||||
spmHandler.Release()
|
||||
return false
|
||||
}
|
||||
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
|
||||
needsRequestEvent := hasInteractMatchers && request.NeedsRequestCondition()
|
||||
err = request.executeRequest(t.updatedInput, t.req, make(map[string]interface{}), hasInteractMatchers, func(event *output.InternalWrappedEvent) {
|
||||
if (t.hasInteractMarkers || needsRequestEvent) && request.options.Interactsh != nil {
|
||||
requestData := &interactsh.RequestData{
|
||||
MakeResultFunc: request.MakeResultEvent,
|
||||
Event: event,
|
||||
Operators: request.CompiledOperators,
|
||||
MatchFunc: request.Match,
|
||||
ExtractFunc: request.Extract,
|
||||
}
|
||||
allOASTUrls := httputils.GetInteractshURLSFromEvent(event.InternalEvent)
|
||||
allOASTUrls = append(allOASTUrls, t.req.interactshURLs...)
|
||||
request.options.RegisterInteractshRequest(sliceutil.Dedupe(allOASTUrls), requestData)
|
||||
}
|
||||
wrappedCallback(event)
|
||||
}, 0)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
|
||||
continue
|
||||
}
|
||||
if err := spmHandler.Acquire(); err != nil {
|
||||
continue
|
||||
}
|
||||
if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
|
||||
spmHandler.Release()
|
||||
continue
|
||||
}
|
||||
// Full URL, so https and an explicit port stay on the same key the observation uses.
|
||||
// The handler context ends the wait when stop-at-first-match fires.
|
||||
waitCtx, cancelWait := stopWithHandler(ctx, spmHandler)
|
||||
err := request.rateLimitTake(waitCtx, hostBackoffKey(t.req, t.updatedInput))
|
||||
cancelWait()
|
||||
if err != nil {
|
||||
select {
|
||||
case <-spmHandler.Done():
|
||||
spmHandler.Release()
|
||||
return false
|
||||
case spmHandler.ResultChan <- err:
|
||||
request.recordHostResultAndCancelIfUnresponsive(t.updatedInput, err, spmHandler.Cancel)
|
||||
continue
|
||||
case <-ctx.Done():
|
||||
spmHandler.Release()
|
||||
continue
|
||||
case spmHandler.ResultChan <- err:
|
||||
spmHandler.Release()
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}() {
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil || spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
|
||||
spmHandler.Release()
|
||||
continue
|
||||
}
|
||||
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
|
||||
needsRequestEvent := hasInteractMatchers && request.NeedsRequestCondition()
|
||||
err = request.executeRequest(t.updatedInput, t.req, make(map[string]interface{}), hasInteractMatchers, func(event *output.InternalWrappedEvent) {
|
||||
if (t.hasInteractMarkers || needsRequestEvent) && request.options.Interactsh != nil {
|
||||
requestData := &interactsh.RequestData{
|
||||
MakeResultFunc: request.MakeResultEvent,
|
||||
Event: event,
|
||||
Operators: request.CompiledOperators,
|
||||
MatchFunc: request.Match,
|
||||
ExtractFunc: request.Extract,
|
||||
}
|
||||
allOASTUrls := httputils.GetInteractshURLSFromEvent(event.InternalEvent)
|
||||
allOASTUrls = append(allOASTUrls, t.req.interactshURLs...)
|
||||
request.options.RegisterInteractshRequest(sliceutil.Dedupe(allOASTUrls), requestData)
|
||||
}
|
||||
wrappedCallback(event)
|
||||
}, 0)
|
||||
select {
|
||||
case <-spmHandler.Done():
|
||||
spmHandler.Release()
|
||||
continue
|
||||
case spmHandler.ResultChan <- err:
|
||||
request.recordHostResultAndCancelIfUnresponsive(t.updatedInput, err, spmHandler.Cancel)
|
||||
spmHandler.Release()
|
||||
}
|
||||
}
|
||||
}()
|
||||
@@ -438,7 +422,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
|
||||
|
||||
select {
|
||||
case <-input.Context().Done():
|
||||
stopWorkers()
|
||||
close(tasks)
|
||||
workersWg.Wait()
|
||||
return input.Context().Err()
|
||||
default:
|
||||
}
|
||||
@@ -446,7 +431,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
|
||||
// resize check point - nop if there are no changes
|
||||
if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency {
|
||||
if err := spmHandler.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil {
|
||||
stopWorkers()
|
||||
close(tasks)
|
||||
workersWg.Wait()
|
||||
return err
|
||||
}
|
||||
// if payload concurrency increased, add more workers
|
||||
@@ -463,22 +449,15 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
|
||||
break
|
||||
}
|
||||
|
||||
// Queued tasks hold budget and are only released once the workers
|
||||
// stop, so the wait has to end with them.
|
||||
generatedHttpRequest, reservation, err := request.makeReserved(workerCtx, generator, input, inputData, payloads, dynamicValues)
|
||||
ctx := input.Context()
|
||||
generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
|
||||
if err != nil {
|
||||
if err == types.ErrNoMoreRequests {
|
||||
break
|
||||
}
|
||||
if workerCtx.Err() != nil {
|
||||
stopWorkers()
|
||||
if spmHandler.FoundFirstMatch() {
|
||||
return nil
|
||||
}
|
||||
return context.Cause(workerCtx)
|
||||
}
|
||||
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
|
||||
stopWorkers()
|
||||
close(tasks)
|
||||
workersWg.Wait()
|
||||
return err
|
||||
}
|
||||
hasInteractMarkers := interactsh.HasMarkers(inputData) || len(generatedHttpRequest.interactshURLs) > 0
|
||||
@@ -488,32 +467,33 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
|
||||
updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
|
||||
if request.isUnresponsiveAddress(updatedInput) {
|
||||
// skip on unresponsive host no need to continue
|
||||
reservation.done()
|
||||
spmHandler.Cancel()
|
||||
stopWorkers()
|
||||
close(tasks)
|
||||
workersWg.Wait()
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-spmHandler.Done():
|
||||
reservation.done()
|
||||
stopWorkers()
|
||||
close(tasks)
|
||||
workersWg.Wait()
|
||||
spmHandler.Wait()
|
||||
if spmHandler.FoundFirstMatch() {
|
||||
return nil
|
||||
}
|
||||
return multierr.Combine(spmHandler.CombinedResults()...)
|
||||
case <-workerCtx.Done():
|
||||
reservation.done()
|
||||
stopWorkers()
|
||||
close(tasks)
|
||||
workersWg.Wait()
|
||||
if spmHandler.FoundFirstMatch() {
|
||||
return nil
|
||||
}
|
||||
return context.Cause(workerCtx)
|
||||
case tasks <- task{req: generatedHttpRequest, body: reservation, updatedInput: updatedInput, hasInteractMarkers: hasInteractMarkers}:
|
||||
case tasks <- task{req: generatedHttpRequest, updatedInput: updatedInput, hasInteractMarkers: hasInteractMarkers}:
|
||||
}
|
||||
request.options.Progress.IncrementRequests()
|
||||
}
|
||||
stopWorkers()
|
||||
close(tasks)
|
||||
workersWg.Wait()
|
||||
spmHandler.Wait()
|
||||
if spmHandler.FoundFirstMatch() {
|
||||
// ignore any context cancellation and in-transit execution errors
|
||||
@@ -602,7 +582,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
|
||||
}
|
||||
|
||||
ctx := input.Context()
|
||||
generatedHttpRequest, reservation, err := request.makeReserved(ctx, generator, input, inputData, payloads, dynamicValues)
|
||||
generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
|
||||
if err != nil {
|
||||
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
|
||||
return err
|
||||
@@ -613,18 +593,15 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
|
||||
updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
|
||||
if request.isUnresponsiveAddress(updatedInput) {
|
||||
// skip on unresponsive host no need to continue
|
||||
reservation.done()
|
||||
spmHandler.Cancel()
|
||||
return nil
|
||||
}
|
||||
generatedHttpRequest.pipelinedClient = pipeClient
|
||||
if err := spmHandler.Acquire(); err != nil {
|
||||
reservation.done()
|
||||
break
|
||||
}
|
||||
go func(httpRequest *generatedRequest, body *bodyReservation, requestInput *contextargs.Context) {
|
||||
go func(httpRequest *generatedRequest, requestInput *contextargs.Context) {
|
||||
defer spmHandler.Release()
|
||||
defer body.done()
|
||||
if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(requestInput) {
|
||||
// skip if first match is found
|
||||
return
|
||||
@@ -649,7 +626,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
|
||||
request.recordHostResultAndCancelIfUnresponsive(requestInput, err, spmHandler.Cancel)
|
||||
return
|
||||
}
|
||||
}(generatedHttpRequest, reservation, updatedInput)
|
||||
}(generatedHttpRequest, updatedInput)
|
||||
request.options.Progress.IncrementRequests()
|
||||
}
|
||||
spmHandler.Wait()
|
||||
@@ -699,17 +676,16 @@ executionLoop:
|
||||
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
|
||||
|
||||
ctx := input.Context()
|
||||
generatedHttpRequest, reservation, err := request.makeReserved(ctx, generator, input, data, payloads, dynamicValue)
|
||||
ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, request.options.Options.GetTimeouts().HttpTimeout, ErrHttpEngineRequestDeadline)
|
||||
defer cancel()
|
||||
|
||||
generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
|
||||
if err != nil {
|
||||
if err == types.ErrNoMoreRequests {
|
||||
return false, true, nil, nil
|
||||
}
|
||||
return false, true, nil, err
|
||||
}
|
||||
defer reservation.done()
|
||||
if generatedHttpRequest.customCancelFunction == nil {
|
||||
generatedHttpRequest.sendTimeout = request.options.Options.GetTimeouts().HttpTimeout
|
||||
}
|
||||
// ideally if http template used a custom port or hostname
|
||||
// we would want to update it in input but currently templateCtx logic
|
||||
// is closely tied to contextargs.Context so we are temporarily creating
|
||||
@@ -1084,20 +1060,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
||||
return fmt.Errorf("hostErrorsCache : host %s is unresponsive", input.MetaInput.Input)
|
||||
}
|
||||
|
||||
// Race requests must all be in flight together to reach the open gate.
|
||||
if !generatedRequest.original.Race && !generatedRequest.bodyReserved {
|
||||
releaseBody, err := request.options.RequestBodyBudget.Acquire(input.Context(), generatedRequest.bodyLen())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer releaseBody()
|
||||
}
|
||||
if generatedRequest.sendTimeout > 0 && generatedRequest.request != nil {
|
||||
ctx, cancel := context.WithTimeoutCause(generatedRequest.request.Context(), generatedRequest.sendTimeout, ErrHttpEngineRequestDeadline)
|
||||
defer cancel()
|
||||
generatedRequest.request = generatedRequest.request.WithContext(ctx)
|
||||
}
|
||||
|
||||
// wrap one more callback for validation and fixing event
|
||||
callback := func(event *output.InternalWrappedEvent) {
|
||||
// validateNFixEvent performs necessary validation on generated event
|
||||
@@ -1340,9 +1302,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
||||
}
|
||||
}
|
||||
|
||||
if request.sharesResponse {
|
||||
generatedRequest.request.WithContext(httpclientpool.WithSharedResponse(generatedRequest.request.Context(), request.options.TemplateID))
|
||||
}
|
||||
resp, err = executingClient.Do(generatedRequest.request)
|
||||
sent = true
|
||||
|
||||
|
||||
@@ -1400,7 +1400,7 @@ func TestExecuteParallelHTTP_GoroutineLeaks(t *testing.T) {
|
||||
defer goleak.VerifyNone(t,
|
||||
goleak.IgnoreAnyContainingPkg("go.opencensus.io/stats/view"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/rod"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/go-rod/rod"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/server"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/client"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/ratelimit"),
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCanShareResponseKeepsTimingChecksOnTheirOwnRequest(t *testing.T) {
|
||||
require.True(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Words: []string{"admin"}}}}}, ""))
|
||||
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{DSL: []string{"duration>=5"}}}}}, ""))
|
||||
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Part: "duration"}}}}, ""))
|
||||
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Extractors: []*extractors.Extractor{{DSL: []string{"duration_2"}}}}}, ""))
|
||||
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Type: matchers.MatcherTypeHolder{MatcherType: matchers.LLMMatcher}, Inputs: []string{"{{duration_1}}"}}}}}, ""), "llm inputs read the event too")
|
||||
require.False(t, canShareResponse(&Request{}, "http(1) && template.duration > 5"))
|
||||
require.False(t, canShareResponse(&Request{Race: true}, ""))
|
||||
}
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
@@ -83,9 +82,6 @@ func dumpRequest(req *retryablehttp.Request, maxBody int) (requestDump, error) {
|
||||
}
|
||||
|
||||
capture := &cappedWriter{headerEnd: -1, maxBody: maxBody}
|
||||
if maxBody <= 0 {
|
||||
capture.buf = make([]byte, 0, len(body)+dumpHeaderRoom)
|
||||
}
|
||||
if err := clone.Write(capture); err != nil {
|
||||
return requestDump{}, err
|
||||
}
|
||||
@@ -93,7 +89,7 @@ func dumpRequest(req *retryablehttp.Request, maxBody int) (requestDump, error) {
|
||||
// http.Transport asks for gzip unless the caller set an encoding.
|
||||
if clone.Header.Get("Accept-Encoding") == "" && clone.Header.Get("Range") == "" && clone.Method != http.MethodHead && capture.headerEnd >= 2 {
|
||||
end := capture.headerEnd - 2
|
||||
data = slices.Insert(data, end, acceptEncodingGzip...)
|
||||
data = append(data[:end:end], append([]byte("Accept-Encoding: gzip\r\n"), data[end:]...)...)
|
||||
}
|
||||
return requestDump{data: appendTruncationMarker(data, capture.dropped), truncated: capture.dropped}, nil
|
||||
}
|
||||
@@ -127,10 +123,6 @@ func capDumpBody(dump []byte, maxBody int) []byte {
|
||||
|
||||
var headerTerminator = []byte("\r\n\r\n")
|
||||
|
||||
var acceptEncodingGzip = []byte("Accept-Encoding: gzip\r\n")
|
||||
|
||||
const dumpHeaderRoom = 8 * 1024
|
||||
|
||||
// cappedWriter keeps the request line and headers in full and at most
|
||||
// maxBody bytes of the body that follows, counting the rest.
|
||||
type cappedWriter struct {
|
||||
@@ -185,19 +177,6 @@ func getHTTPProjectCacheScope(requestDump requestDump, body []byte, scheme, host
|
||||
return scoped.Bytes()
|
||||
}
|
||||
|
||||
func (g *generatedRequest) bodyLen() int64 {
|
||||
if g.rawRequest != nil {
|
||||
return int64(len(g.rawRequest.Data))
|
||||
}
|
||||
if g.request == nil || g.request.Body == nil {
|
||||
return 0
|
||||
}
|
||||
if body, ok := g.request.Body.(*httpbody.Body); ok {
|
||||
return int64(body.Len())
|
||||
}
|
||||
return g.request.ContentLength
|
||||
}
|
||||
|
||||
// body returns the request body without copying it.
|
||||
func (g *generatedRequest) body() []byte {
|
||||
if g.request != nil {
|
||||
|
||||
@@ -44,7 +44,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
|
||||
unitutils "github.com/projectdiscovery/utils/unit"
|
||||
)
|
||||
|
||||
@@ -107,8 +106,6 @@ type ExecutorOptions struct {
|
||||
Progress progress.Progress
|
||||
// RateLimiter is a rate-limiter for limiting sent number of requests.
|
||||
RateLimiter *ratelimit.Limiter
|
||||
// RequestBodyBudget bounds request-body bytes in flight; nil is no limit.
|
||||
RequestBodyBudget *httpbody.Budget
|
||||
// Catalog is a template catalog implementation for nuclei
|
||||
Catalog catalog.Catalog
|
||||
// ProjectFile is the project file for nuclei
|
||||
@@ -398,7 +395,6 @@ func (e *ExecutorOptions) Copy() *ExecutorOptions {
|
||||
IssuesClient: e.IssuesClient,
|
||||
Progress: e.Progress,
|
||||
RateLimiter: e.RateLimiter,
|
||||
RequestBodyBudget: e.RequestBodyBudget,
|
||||
Catalog: e.Catalog,
|
||||
ProjectFile: e.ProjectFile,
|
||||
Browser: e.Browser,
|
||||
@@ -710,7 +706,6 @@ func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) {
|
||||
e.IssuesClient = n.IssuesClient
|
||||
e.Progress = n.Progress
|
||||
e.RateLimiter = n.RateLimiter
|
||||
e.RequestBodyBudget = n.RequestBodyBudget
|
||||
e.Catalog = n.Catalog
|
||||
e.ProjectFile = n.ProjectFile
|
||||
e.Browser = n.Browser
|
||||
|
||||
@@ -23,7 +23,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
|
||||
@@ -35,7 +34,6 @@ import (
|
||||
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
|
||||
"github.com/projectdiscovery/tlsx/pkg/tlsx/openssl"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
iputil "github.com/projectdiscovery/utils/ip"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
)
|
||||
|
||||
@@ -204,21 +202,6 @@ func (request *Request) GetID() string {
|
||||
return request.ID
|
||||
}
|
||||
|
||||
// allowedAddress returns the first address of host that the network policy allows.
|
||||
func (request *Request) allowedAddress(host string) (string, error) {
|
||||
dnsData, err := request.dialer.GetDNSData(host)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
addresses := append(append([]string{}, dnsData.A...), dnsData.AAAA...)
|
||||
for _, address := range addresses {
|
||||
if protocolstate.IsHostAllowed(request.options.Options.ExecutionId, address) {
|
||||
return address, nil
|
||||
}
|
||||
}
|
||||
return "", protocolstate.ErrHostDenied.Msgf(host)
|
||||
}
|
||||
|
||||
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
|
||||
func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
hostPort := input.MetaInput.Input
|
||||
@@ -272,23 +255,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
||||
} else {
|
||||
hostIp = host
|
||||
}
|
||||
if !iputil.IsIP(hostIp) {
|
||||
// The openssl scan mode runs the openssl binary, which resolves a host
|
||||
// name again by itself, so it gets an address the policy allows.
|
||||
address, err := request.allowedAddress(hostIp)
|
||||
if err != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
return errkit.Wrap(err, "could not resolve host")
|
||||
}
|
||||
hostIp = address
|
||||
} else if !protocolstate.IsHostAllowed(request.options.Options.ExecutionId, hostIp) {
|
||||
// A custom dialer can allow an address that the execution policy denies.
|
||||
err := protocolstate.ErrHostDenied.Msgf(hostIp)
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
return err
|
||||
}
|
||||
|
||||
if err := request.options.HostBackoffWait(input.Context(), addressToDial); err != nil {
|
||||
return err
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
package ssl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"math/big"
|
||||
"net"
|
||||
"slices"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/fastdialer/fastdialer"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/tlsx/pkg/tlsx/openssl"
|
||||
)
|
||||
|
||||
func selfSignedCertificate(t *testing.T, name string) tls.Certificate {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
require.NoError(t, err)
|
||||
template := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: name},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
|
||||
require.NoError(t, err)
|
||||
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}
|
||||
}
|
||||
|
||||
// serveTLS counts the TLS handshakes that reach address.
|
||||
func serveTLS(t *testing.T, address, name string, handshakes *atomic.Int32) string {
|
||||
t.Helper()
|
||||
listener, err := tls.Listen("tcp", address, &tls.Config{Certificates: []tls.Certificate{selfSignedCertificate(t, name)}})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = listener.Close() })
|
||||
go func() {
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer func() { _ = conn.Close() }()
|
||||
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
if conn.(*tls.Conn).Handshake() == nil {
|
||||
handshakes.Add(1)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}()
|
||||
return listener.Addr().String()
|
||||
}
|
||||
|
||||
func TestSSLOpenSSLObeysNetworkPolicy(t *testing.T) {
|
||||
if !openssl.IsAvailable() {
|
||||
t.Skip("openssl binary is not installed")
|
||||
}
|
||||
addrs, err := net.LookupHost("localhost")
|
||||
if err != nil || len(addrs) < 2 {
|
||||
t.Skip("localhost does not resolve to both loopback addresses")
|
||||
}
|
||||
|
||||
// The policy allows 127.0.0.1 and denies ::1, so localhost passes the
|
||||
// dial check. The openssl binary must not connect to ::1.
|
||||
var allowedHandshakes, deniedHandshakes atomic.Int32
|
||||
allowed := serveTLS(t, "127.0.0.1:0", "allowed", &allowedHandshakes)
|
||||
_, port, err := net.SplitHostPort(allowed)
|
||||
require.NoError(t, err)
|
||||
serveTLS(t, net.JoinHostPort("::1", port), "denied", &deniedHandshakes)
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.SetExecutionID(t.Name())
|
||||
options.ExcludeTargets = []string{"::1"}
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() { testutils.Cleanup(options) })
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "ssl-openssl-policy"})
|
||||
executerOpts.IsMultiProtocol = true
|
||||
|
||||
request := &Request{ID: "ssl-openssl-policy", Address: "{{Host}}:{{Port}}", ScanMode: "openssl"}
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
// The scan resolves names with the fastdialer, not the system resolver. On
|
||||
// Windows the hosts file has no localhost entry, so the name does not resolve.
|
||||
if !resolvesToBothLoopbacks(request) {
|
||||
t.Skip("the scan dialer does not resolve localhost to both loopback addresses")
|
||||
}
|
||||
|
||||
var event output.InternalEvent
|
||||
input := contextargs.NewWithInput(context.Background(), net.JoinHostPort("localhost", port))
|
||||
err = request.ExecuteWithResults(input, nil, nil, func(e *output.InternalWrappedEvent) {
|
||||
event = e.InternalEvent
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "allowed", event["subject_cn"])
|
||||
require.Zero(t, deniedHandshakes.Load(), "openssl connected to a denied address")
|
||||
}
|
||||
|
||||
func resolvesToBothLoopbacks(request *Request) bool {
|
||||
data, err := request.dialer.GetDNSData("localhost")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return slices.Contains(data.A, "127.0.0.1") && slices.Contains(data.AAAA, "::1")
|
||||
}
|
||||
|
||||
func TestSSLOpenSSLLiteralIPObeysExecutionPolicy(t *testing.T) {
|
||||
if !openssl.IsAvailable() {
|
||||
t.Skip("openssl binary is not installed")
|
||||
}
|
||||
|
||||
// The input pins ::1 as its custom ip, and a custom dialer with no deny
|
||||
// list would connect to it. The execution policy denies it, so the openssl
|
||||
// binary must not connect to it.
|
||||
var deniedHandshakes atomic.Int32
|
||||
denied := serveTLS(t, "[::1]:0", "denied", &deniedHandshakes)
|
||||
_, port, err := net.SplitHostPort(denied)
|
||||
require.NoError(t, err)
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.SetExecutionID(t.Name())
|
||||
options.ExcludeTargets = []string{"::1"}
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() { testutils.Cleanup(options) })
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "ssl-openssl-literal-ip"})
|
||||
executerOpts.IsMultiProtocol = true
|
||||
customDialer, err := fastdialer.NewDialer(fastdialer.DefaultOptions)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(customDialer.Close)
|
||||
executerOpts.CustomFastdialer = customDialer
|
||||
|
||||
request := &Request{ID: "ssl-openssl-literal-ip", Address: "{{Host}}:{{Port}}", ScanMode: "openssl"}
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
input := contextargs.NewWithInput(context.Background(), net.JoinHostPort("localhost", port))
|
||||
input.MetaInput.CustomIP = "::1"
|
||||
err = request.ExecuteWithResults(input, nil, nil, func(*output.InternalWrappedEvent) {})
|
||||
require.Error(t, err)
|
||||
require.Zero(t, deniedHandshakes.Load(), "openssl connected to an address the execution policy denies")
|
||||
}
|
||||
|
||||
func TestSSLOpenSSLCustomIPObeysNetworkPolicy(t *testing.T) {
|
||||
if !openssl.IsAvailable() {
|
||||
t.Skip("openssl binary is not installed")
|
||||
}
|
||||
|
||||
// The input pins the denied address ::1 as its custom ip, so the request
|
||||
// skips the name lookup. The openssl binary must still not connect to it.
|
||||
var deniedHandshakes atomic.Int32
|
||||
denied := serveTLS(t, "[::1]:0", "denied", &deniedHandshakes)
|
||||
_, port, err := net.SplitHostPort(denied)
|
||||
require.NoError(t, err)
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.SetExecutionID(t.Name())
|
||||
options.ExcludeTargets = []string{"::1"}
|
||||
testutils.Init(options)
|
||||
t.Cleanup(func() { testutils.Cleanup(options) })
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "ssl-openssl-custom-ip"})
|
||||
executerOpts.IsMultiProtocol = true
|
||||
|
||||
request := &Request{ID: "ssl-openssl-custom-ip", Address: "{{Host}}:{{Port}}", ScanMode: "openssl"}
|
||||
require.NoError(t, request.Compile(executerOpts))
|
||||
|
||||
input := contextargs.NewWithInput(context.Background(), net.JoinHostPort("localhost", port))
|
||||
input.MetaInput.CustomIP = "::1"
|
||||
_ = request.ExecuteWithResults(input, nil, nil, func(*output.InternalWrappedEvent) {})
|
||||
require.Zero(t, deniedHandshakes.Load(), "openssl connected to a denied custom ip")
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
package replay
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
)
|
||||
|
||||
// Matchers type-assert some values, such as an int status code or a []string
|
||||
// of dns answers, and binary responses are not valid UTF-8, so plain JSON
|
||||
// would change what they see. Those values are wrapped in a one-key object
|
||||
// naming their Go type; everything else is stored as JSON.
|
||||
const (
|
||||
tagBytes = "$bytes"
|
||||
tagBinary = "$binary"
|
||||
tagInt = "$int"
|
||||
tagStrings = "$strings"
|
||||
)
|
||||
|
||||
func encodeEvent(event output.InternalEvent) map[string]any {
|
||||
encoded := make(map[string]any, len(event))
|
||||
for key, value := range event {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
if utf8.ValidString(v) {
|
||||
encoded[key] = v
|
||||
} else {
|
||||
encoded[key] = map[string]string{tagBinary: base64.StdEncoding.EncodeToString([]byte(v))}
|
||||
}
|
||||
case []byte:
|
||||
encoded[key] = map[string]string{tagBytes: base64.StdEncoding.EncodeToString(v)}
|
||||
case int:
|
||||
encoded[key] = map[string]int{tagInt: v}
|
||||
case []string:
|
||||
encoded[key] = map[string][]string{tagStrings: v}
|
||||
default:
|
||||
// values JSON cannot hold, such as funcs, are not something a
|
||||
// matcher reads
|
||||
if _, err := json.Marshal(v); err == nil {
|
||||
encoded[key] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
return encoded
|
||||
}
|
||||
|
||||
func decodeEvent(raw map[string]json.Message) (output.InternalEvent, error) {
|
||||
event := make(output.InternalEvent, len(raw))
|
||||
for key, message := range raw {
|
||||
value, err := decodeValue(message)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
event[key] = value
|
||||
}
|
||||
return event, nil
|
||||
}
|
||||
|
||||
func decodeValue(message json.Message) (any, error) {
|
||||
var tagged map[string]json.Message
|
||||
if json.Unmarshal(message, &tagged) == nil && len(tagged) == 1 {
|
||||
for tag, inner := range tagged {
|
||||
switch tag {
|
||||
case tagBytes, tagBinary:
|
||||
var text string
|
||||
if err := json.Unmarshal(inner, &text); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := base64.StdEncoding.DecodeString(text)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tag == tagBinary {
|
||||
return string(data), nil
|
||||
}
|
||||
return data, nil
|
||||
case tagInt:
|
||||
var number int
|
||||
err := json.Unmarshal(inner, &number)
|
||||
return number, err
|
||||
case tagStrings:
|
||||
var list []string
|
||||
err := json.Unmarshal(inner, &list)
|
||||
return list, err
|
||||
}
|
||||
}
|
||||
}
|
||||
var value any
|
||||
err := json.Unmarshal(message, &value)
|
||||
return value, err
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package replay
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestEventRoundTripKeepsWhatMatchersRead(t *testing.T) {
|
||||
event := output.InternalEvent{
|
||||
"body": "<title>admin</title>",
|
||||
"raw": "\x00\xffBIN\xfe",
|
||||
"data": []byte{0, 1, 2},
|
||||
"status_code": 200,
|
||||
"answer": []string{"a", "b"},
|
||||
"duration": 1.5,
|
||||
"matched": true,
|
||||
"info": map[string]any{"name": "x"},
|
||||
"callback": func() {},
|
||||
}
|
||||
data, err := json.Marshal(encodeEvent(event))
|
||||
require.NoError(t, err)
|
||||
var raw map[string]json.Message
|
||||
require.NoError(t, json.Unmarshal(data, &raw))
|
||||
decoded, err := decodeEvent(raw)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, "<title>admin</title>", decoded["body"])
|
||||
require.Equal(t, "\x00\xffBIN\xfe", decoded["raw"], "binary responses keep every byte")
|
||||
require.Equal(t, []byte{0, 1, 2}, decoded["data"])
|
||||
require.Equal(t, 200, decoded["status_code"], "the status matcher reads an int")
|
||||
require.Equal(t, []string{"a", "b"}, decoded["answer"])
|
||||
require.Equal(t, 1.5, decoded["duration"])
|
||||
require.Equal(t, true, decoded["matched"])
|
||||
require.Equal(t, map[string]any{"name": "x"}, decoded["info"])
|
||||
require.NotContains(t, decoded, "callback", "values JSON cannot hold are dropped")
|
||||
}
|
||||
Loaded 100 of 118 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user