Compare commits

..
Author SHA1 Message Date
Mzack9999 55bd0bbfb6 fix selection 2026-11-07 11:28:03 +01:00
Mzack9999 8f1c88e1f3 merge dev 2026-11-07 11:26:54 +01:00
Ice3manandClaude Opus 4.8 0305d5468c feat(automaticscan): tech-graph based template selection (automatic scan v2)
Automatic scan currently loads templates by overlapping detected
technology tags with template tags. This both over-selects (any template
sharing a generic tag fires) and under-selects (plugin/CVE templates whose
tags don't match the detected platform are missed).

This adds a tech-graph driven selection path. An offline generator walks a
nuclei-templates checkout and builds tech-graph.json: a deterministic
technology -> dependent-template graph derived from CPE, metadata
vendor/product, directory grouping, platform plugin ecosystems, and
conservative tag/id reconciliation. The artifact is embedded in the binary.

At runtime the selector resolves detected signals (wappalyzer names,
detection tags, CPE products) to canonical tech ids and emits a precise
fire-set: each tech's dependents plus a tier-gated, tech-agnostic baseline.
A new -automatic-scan-coverage/-asc flag selects the tier:

  lean      dependents only, no baseline
  balanced  dependents + balanced baseline (default)
  thorough  dependents + full baseline + tag-overlap fuzzy fallback

Behaviour never regresses: if the embedded graph is unavailable the service
silently falls back to legacy tag-overlap loading, and any detected signal
not present in the graph is recovered via tag-overlap (except at lean).

- techgraph: offline build + embedded artifact + deterministic selector
- techgraph-gen: regenerates tech-graph.json from a templates checkout
- automaticscan: graph selection with legacy fallback and tier gating

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
2026-06-18 22:18:39 +05:30
118 changed files with 78900 additions and 6034 deletions

No files matched your search

+1 -1
View File
@@ -17,4 +17,4 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: crate-ci/[email protected]2.0
- uses: crate-ci/[email protected]1.1
-6
View File
@@ -166,8 +166,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
@@ -207,7 +205,6 @@ OUTPUT:
-o, -output string output file to write found issues/vulnerabilities
-sresp, -store-resp store all request/response passed through nuclei to output directory
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
-silent display findings only
-nc, -no-color disable output content coloring (ANSI escape codes)
-j, -jsonl write output in JSONL(ines) format
@@ -222,7 +219,6 @@ OUTPUT:
-se, -sarif-export string file to export results in SARIF format
-je, -json-export string file to export results in JSON format
-jle, -jsonl-export string file to export results in JSONL(ine) format
-ce, -csv-export string file to export results in CSV format
-pe, -pdf-export string file to export results in PDF format
-rd, -redact string[] redact given list of keys from query parameter, request header and body
-nrd, -no-redact disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted
@@ -242,7 +238,6 @@ CONFIGURATIONS:
-sr, -system-resolvers use system DNS resolving as error fallback
-dc, -disable-clustering disable clustering of requests
-passive enable passive HTTP response processing mode
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
-fh2, -force-http2 force http2 connection on requests
-ev, -env-vars enable environment variables to be used in template
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
@@ -261,7 +256,6 @@ CONFIGURATIONS:
-rsr, -response-size-read int max response size to read in bytes
-rss, -response-size-save int max response size to read in bytes (default 1048576)
-rqss, -request-size-save int max request body size to save in bytes (0 to save the whole body) (default 32768)
-mrbi, -max-request-body-inflight int max request body bytes held by in-flight http requests; requests with bodies of 1MB or more queue past it (0 disables)
-reset reset removes all nuclei configuration and data files (including nuclei-templates)
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
-hae, -http-api-endpoint string experimental http api endpoint
-4
View File
@@ -164,8 +164,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
@@ -205,7 +203,6 @@ OUTPUT:
-o, -output string output file to write found issues/vulnerabilities
-sresp, -store-resp store all request/response passed through nuclei to output directory
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
-silent display findings only
-nc, -no-color disable output content coloring (ANSI escape codes)
-j, -jsonl write output in JSONL(ines) format
@@ -239,7 +236,6 @@ CONFIGURATIONS:
-sr, -system-resolvers use system DNS resolving as error fallback
-dc, -disable-clustering disable clustering of requests
-passive enable passive HTTP response processing mode
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
-fh2, -force-http2 force http2 connection on requests
-ev, -env-vars enable environment variables to be used in template
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
-4
View File
@@ -164,8 +164,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
@@ -205,7 +203,6 @@ OUTPUT:
-o, -output string output file to write found issues/vulnerabilities
-sresp, -store-resp store all request/response passed through nuclei to output directory
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
-silent display findings only
-nc, -no-color disable output content coloring (ANSI escape codes)
-j, -jsonl write output in JSONL(ines) format
@@ -239,7 +236,6 @@ CONFIGURATIONS:
-sr, -system-resolvers use system DNS resolving as error fallback
-dc, -disable-clustering disable clustering of requests
-passive enable passive HTTP response processing mode
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
-fh2, -force-http2 force http2 connection on requests
-ev, -env-vars enable environment variables to be used in template
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
-4
View File
@@ -181,8 +181,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
@@ -222,7 +220,6 @@ OUTPUT:
-o, -output string output file to write found issues/vulnerabilities
-sresp, -store-resp store all request/response passed through nuclei to output directory
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
-silent display findings only
-nc, -no-color disable output content coloring (ANSI escape codes)
-j, -jsonl write output in JSONL(ines) format
@@ -256,7 +253,6 @@ CONFIGURATIONS:
-sr, -system-resolvers use system DNS resolving as error fallback
-dc, -disable-clustering disable clustering of requests
-passive enable passive HTTP response processing mode
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
-fh2, -force-http2 force http2 connection on requests
-ev, -env-vars enable environment variables to be used in template
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
-4
View File
@@ -164,8 +164,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
@@ -205,7 +203,6 @@ OUTPUT:
-o, -output string output file to write found issues/vulnerabilities
-sresp, -store-resp store all request/response passed through nuclei to output directory
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
-silent display findings only
-nc, -no-color disable output content coloring (ANSI escape codes)
-j, -jsonl write output in JSONL(ines) format
@@ -239,7 +236,6 @@ CONFIGURATIONS:
-sr, -system-resolvers use system DNS resolving as error fallback
-dc, -disable-clustering disable clustering of requests
-passive enable passive HTTP response processing mode
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
-fh2, -force-http2 force http2 connection on requests
-ev, -env-vars enable environment variables to be used in template
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
-4
View File
@@ -164,8 +164,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
@@ -205,7 +203,6 @@ OUTPUT:
-o, -output string output file to write found issues/vulnerabilities
-sresp, -store-resp store all request/response passed through nuclei to output directory
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
-silent display findings only
-nc, -no-color disable output content coloring (ANSI escape codes)
-j, -jsonl write output in JSONL(ines) format
@@ -239,7 +236,6 @@ CONFIGURATIONS:
-sr, -system-resolvers use system DNS resolving as error fallback
-dc, -disable-clustering disable clustering of requests
-passive enable passive HTTP response processing mode
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
-fh2, -force-http2 force http2 connection on requests
-ev, -env-vars enable environment variables to be used in template
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
-4
View File
@@ -164,8 +164,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
@@ -205,7 +203,6 @@ OUTPUT:
-o, -output string output file to write found issues/vulnerabilities
-sresp, -store-resp store all request/response passed through nuclei to output directory
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
-silent display findings only
-nc, -no-color disable output content coloring (ANSI escape codes)
-j, -jsonl write output in JSONL(ines) format
@@ -239,7 +236,6 @@ CONFIGURATIONS:
-sr, -system-resolvers use system DNS resolving as error fallback
-dc, -disable-clustering disable clustering of requests
-passive enable passive HTTP response processing mode
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
-fh2, -force-http2 force http2 connection on requests
-ev, -env-vars enable environment variables to be used in template
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
-4
View File
@@ -164,8 +164,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
@@ -205,7 +203,6 @@ OUTPUT:
-o, -output string output file to write found issues/vulnerabilities
-sresp, -store-resp store all request/response passed through nuclei to output directory
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
-silent display findings only
-nc, -no-color disable output content coloring (ANSI escape codes)
-j, -jsonl write output in JSONL(ines) format
@@ -239,7 +236,6 @@ CONFIGURATIONS:
-sr, -system-resolvers use system DNS resolving as error fallback
-dc, -disable-clustering disable clustering of requests
-passive enable passive HTTP response processing mode
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
-fh2, -force-http2 force http2 connection on requests
-ev, -env-vars enable environment variables to be used in template
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
-4
View File
@@ -164,8 +164,6 @@ TEMPLATES:
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
-validate validate the passed templates to nuclei
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all templates matching current filters
@@ -205,7 +203,6 @@ OUTPUT:
-o, -output string output file to write found issues/vulnerabilities
-sresp, -store-resp store all request/response passed through nuclei to output directory
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
-store-events string store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)
-silent display findings only
-nc, -no-color disable output content coloring (ANSI escape codes)
-j, -jsonl write output in JSONL(ines) format
@@ -239,7 +236,6 @@ CONFIGURATIONS:
-sr, -system-resolvers use system DNS resolving as error fallback
-dc, -disable-clustering disable clustering of requests
-passive enable passive HTTP response processing mode
-replay string run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests
-fh2, -force-http2 force http2 connection on requests
-ev, -env-vars enable environment variables to be used in template
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
+13 -21
View File
@@ -1,13 +1,13 @@
<!-- Generated by securitycontext.dev. Source HEAD: 40d08ee493f6f84c9a6b3c7f15f22da83853130c. -->
<!-- Generated by securitycontext.dev. Source HEAD: df57ae6d2955864fd92a1bd2565fe9e4e9a7592b. -->
# Security Context: projectdiscovery/nuclei @ dev
> Reference context for editing projectdiscovery/nuclei. Before changing any component below, **preserve the listed guards and do not reintroduce these fixed issues**. Each fingerprint is a regression check, and each shared attack surface is a guard that must keep holding on every path.
_Analyzed [40d08ee493][c-40d08ee493] on `dev` · 5242 commits, 586 security-relevant examined · generated 2026-11-09_
_Analyzed [df57ae6d29][c-df57ae6d29] on `dev` · 5174 commits, 548 security-relevant examined · generated 2026-11-02_
## Recurring Security Patterns
**12.5% of classified fixes repeat a previously fixed pattern** (2 recurring, 14 first-seen). A recurring pattern means this repository fixed a similar issue before — not that the code is currently vulnerable.
**7.1% of classified fixes repeat a previously fixed pattern** (1 recurring, 13 first-seen). A recurring pattern means this repository fixed a similar issue before — not that the code is currently vulnerable.
## Top Risks
@@ -15,12 +15,12 @@ Components with the strongest concentration of past fixes for one class.
| # | Class | Component | Severity | Fixes | Evidence |
|---|---|---|---|---|---|
| 1 | Code Execution | `pkg/fuzz/execute.go` | high | 2 | [4fed40f75a][c-4fed40f75a], [ccbfb12bd0][c-ccbfb12bd0] |
| 2 | SSRF | `pkg/protocols/common/protocolstate/headless.go` | high | 2 | [d43c1f547f][c-d43c1f547f], [d9dabbb4a1][c-d9dabbb4a1] |
| 3 | Arbitrary File Access | `v2/pkg/protocols/common/generators/load.go` | high | 0 (1 component gone, not counted) | [e5154d362a][c-e5154d362a] |
| 4 | Arbitrary File Access | `pkg/js/libs/mysql/mysql.go` | high | 1 | [634cecbb28][c-634cecbb28] |
| 5 | Certificate Validation Bypass | `pkg/reporting/exporters/es/elasticsearch.go` | high | 1 | [82d5a8acc2][c-82d5a8acc2] |
| 6 | Code Execution | `pkg/protocols/common/expressions/expressions.go` | high | 1 | [d221732016][c-d221732016] |
| 1 | SSRF | `pkg/protocols/common/protocolstate/headless.go` | high | 2 | [d43c1f547f][c-d43c1f547f], [d9dabbb4a1][c-d9dabbb4a1] |
| 2 | Arbitrary File Access | `v2/pkg/protocols/common/generators/load.go` | high | 0 (1 component gone, not counted) | [e5154d362a][c-e5154d362a] |
| 3 | Arbitrary File Access | `pkg/js/libs/mysql/mysql.go` | high | 1 | [634cecbb28][c-634cecbb28] |
| 4 | Code Execution | `pkg/fuzz/execute.go` | high | 1 | [ccbfb12bd0][c-ccbfb12bd0] |
| 5 | Code Execution | `pkg/protocols/common/expressions/expressions.go` | high | 1 | [d221732016][c-d221732016] |
| 6 | Information Disclosure | `internal/httpapi/apiendpoint.go` | high | 1 | [a20aa26024][c-a20aa26024] |
## Fixed-Issue Fingerprints
@@ -28,13 +28,8 @@ Real security fixes mined from git history; use them as regression checks. `veri
**By component** (directories with the most fixes):
- `pkg/fuzz` — 2 fixes: Code Execution (2)
- `pkg/protocols/common/protocolstate` — 2 fixes: SSRF (2)
- [4fed40f75a][c-4fed40f75a] **fix(fuzz): stop re-rendering evaluated template variables (#8057)** — Code Execution, high [CWE-94] · inferred · 2026-11-05
`pkg/fuzz/execute.go` · sink `rule.evaluateVarsWithInteractsh` · fix: safe parser configuration
- [82d5a8acc2][c-82d5a8acc2] **exporters: verify TLS certificates (#8042)** — Certificate Validation Bypass, high [CWE-295] · inferred · 2026-11-04
`pkg/reporting/exporters/es/elasticsearch.go` · sink `DialTLS` · fix: cryptographic verification
- [1c0f77d28e][c-1c0f77d28e] **fix(js/net): enforce network policy on proxied target in JS net dial** — SSRF, high [CWE-918] · inferred · 2026-10-18
`pkg/js/libs/net/net.go` · sink `dialHTTPProxy` · fix: input validation
- [d43c1f547f][c-d43c1f547f] **fix(headless): canonicalize IPv4-mapped IPv6 literals and re-validate dial targets** — SSRF, high [CWE-918] · inferred · 2026-10-13
@@ -68,12 +63,11 @@ Real security fixes mined from git history; use them as regression checks. `veri
| Class | Fixes | Verified | Most recent fix |
|---|---|---|---|
| Code Execution | 3 | 2 | [2026-11-05][c-4fed40f75a] 4fed40f75a |
| SSRF | 3 | 0 | [2026-10-18][c-1c0f77d28e] 1c0f77d28e |
| Arbitrary File Access | 1 (1 component gone, not counted) | 2 | [2026-06-19][c-634cecbb28] 634cecbb28 |
| Code Execution | 2 | 2 | [2026-06-30][c-ccbfb12bd0] ccbfb12bd0 |
| Sandbox Escape | 2 | 0 | [2026-06-24][c-3a8b65cb45] 3a8b65cb45 |
| Signature Verification Bypass | 2 | 2 | [2026-07-03][c-fac0b7196d] fac0b7196d |
| Certificate Validation Bypass | 1 | 0 | [2026-11-04][c-82d5a8acc2] 82d5a8acc2 |
| Information Disclosure | 1 | 0 | [2026-08-03][c-a20aa26024] a20aa26024 |
| Path Traversal | 1 | 0 | [2026-06-04][c-20cbf0bd73] 20cbf0bd73 |
| SQL Injection | 1 | 0 | [2026-06-23][c-68daef44a7] 68daef44a7 |
@@ -106,17 +100,15 @@ Published elsewhere, fixed before the history window, or not yet linked; full de
| CVE-2024-40641 / GHSA-c3q9-c27p-cw9h | high 7.4 | 2024-07-17 | CWE-78 | Unsigned code template execution through workflows in projectdiscovery/nuclei. ### Summary... |
| CVE-2024-27920 / GHSA-w5wx-6g2r-r78q | high 7.4 | 2024-03-15 | CWE-78 | Unsigned code template execution through workflows in projectdiscovery/nuclei. ### Overvie... |
[c-40d08ee493]: https://github.com/projectdiscovery/nuclei/commit/40d08ee493f6f84c9a6b3c7f15f22da83853130c
[c-4fed40f75a]: https://github.com/projectdiscovery/nuclei/commit/4fed40f75a57c7c414aac62d213d772cc3f430d3
[c-ccbfb12bd0]: https://github.com/projectdiscovery/nuclei/commit/ccbfb12bd01447ba25f6e755dfb2bee677736da6
[c-df57ae6d29]: https://github.com/projectdiscovery/nuclei/commit/df57ae6d2955864fd92a1bd2565fe9e4e9a7592b
[c-d43c1f547f]: https://github.com/projectdiscovery/nuclei/commit/d43c1f547f98ac56654f4335ad20af0cca4c229c
[c-d9dabbb4a1]: https://github.com/projectdiscovery/nuclei/commit/d9dabbb4a19e897a6bc12334a030a43035b27b73
[c-e5154d362a]: https://github.com/projectdiscovery/nuclei/commit/e5154d362af900e83b30fb79028bfab1a7f0c648
[c-634cecbb28]: https://github.com/projectdiscovery/nuclei/commit/634cecbb289db494247c04a9c4ec6a113aef3cc0
[c-82d5a8acc2]: https://github.com/projectdiscovery/nuclei/commit/82d5a8acc2c756394c7fa92eae08b41330e89d69
[c-ccbfb12bd0]: https://github.com/projectdiscovery/nuclei/commit/ccbfb12bd01447ba25f6e755dfb2bee677736da6
[c-d221732016]: https://github.com/projectdiscovery/nuclei/commit/d2217320162d5782ca7cb95bef9dda17063818f3
[c-1c0f77d28e]: https://github.com/projectdiscovery/nuclei/commit/1c0f77d28ed639a34c2036fa23c7503ffac0bf27
[c-a20aa26024]: https://github.com/projectdiscovery/nuclei/commit/a20aa260245e5db889c5a421d4a65aeb2a64bf46
[c-1c0f77d28e]: https://github.com/projectdiscovery/nuclei/commit/1c0f77d28ed639a34c2036fa23c7503ffac0bf27
[c-fac0b7196d]: https://github.com/projectdiscovery/nuclei/commit/fac0b7196d2fbca56c8af8ff8e1899c111041a37
[c-3a8b65cb45]: https://github.com/projectdiscovery/nuclei/commit/3a8b65cb45f121fcdf418f60e5177ed9f250c122
[c-68daef44a7]: https://github.com/projectdiscovery/nuclei/commit/68daef44a7eafd41c39690924620e418762909ef
+2
View File
@@ -22,6 +22,8 @@ extend-exclude = [
"go.sum",
# Generated history. Commit SHAs are not words.
"SECURITY_CONTEXT.md",
# Generated technology graph. Entries are third-party product identifiers.
"pkg/protocols/common/automaticscan/techgraph/tech-graph.json",
]
[default.extend-identifiers]
+1 -6
View File
@@ -307,14 +307,13 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"),
flagSet.StringSliceVarP(&options.NewTemplatesWithVersion, "new-templates-version", "ntv", nil, "run new templates added in specific version", goflags.CommaSeparatedStringSliceOptions),
flagSet.BoolVarP(&options.AutomaticScan, "automatic-scan", "as", false, "automatic web scan using wappalyzer technology detection to tags mapping"),
flagSet.StringVarP(&options.AutomaticScanCoverage, "automatic-scan-coverage", "asc", "balanced", "automatic scan coverage tier: lean, balanced, or thorough"),
flagSet.StringSliceVarP(&options.Templates, "templates", "t", nil, "list of template or template directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "turl", nil, "template url or list containing template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
flagSet.StringVarP(&options.AITemplatePrompt, "prompt", "ai", "", "generate and run template using ai prompt"),
flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", nil, "list of workflow or workflow directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
flagSet.StringVar(&options.PlanFile, "plan", "", "write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target"),
flagSet.StringVar(&options.RunPlanFile, "run-plan", "", "run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes"),
flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all templates matching current filters"),
@@ -357,7 +356,6 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write found issues/vulnerabilities"),
flagSet.BoolVarP(&options.StoreResponse, "store-resp", "sresp", false, "store all request/response passed through nuclei to output directory"),
flagSet.StringVarP(&options.StoreResponseDir, "store-resp-dir", "srd", runner.DefaultDumpTrafficOutputFolder, "store all request/response passed through nuclei to custom directory"),
flagSet.StringVar(&options.StoreEvents, "store-events", "", "store what the matchers of every request see, for any protocol, to a jsonl file for -replay (holds raw traffic, credentials included)"),
flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"),
flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"),
flagSet.BoolVarP(&options.JSONL, "jsonl", "j", false, "write output in JSONL(ines) format"),
@@ -372,7 +370,6 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"),
flagSet.StringVarP(&options.JSONExport, "json-export", "je", "", "file to export results in JSON format"),
flagSet.StringVarP(&options.JSONLExport, "jsonl-export", "jle", "", "file to export results in JSONL(ine) format"),
flagSet.StringVarP(&options.CSVExport, "csv-export", "ce", "", "file to export results in CSV format"),
flagSet.StringVarP(&options.PDFExport, "pdf-export", "pe", "", "file to export results in PDF format"),
flagSet.StringSliceVarP(&options.Redact, "redact", "rd", nil, "redact given list of keys from query parameter, request header and body", goflags.CommaSeparatedStringSliceOptions),
flagSet.BoolVarP(&options.NoRedact, "no-redact", "nrd", false, "disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted"),
@@ -393,7 +390,6 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.BoolVarP(&options.SystemResolvers, "system-resolvers", "sr", false, "use system DNS resolving as error fallback"),
flagSet.BoolVarP(&options.DisableClustering, "disable-clustering", "dc", false, "disable clustering of requests"),
flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"),
flagSet.StringVar(&options.ReplayFile, "replay", "", "run the templates' matchers and extractors over a -store-events file and print the findings sorted, without sending requests"),
flagSet.BoolVarP(&options.ForceAttemptHTTP2, "force-http2", "fh2", false, "force http2 connection on requests"),
flagSet.BoolVarP(&options.EnvironmentVariables, "env-vars", "ev", false, "enable environment variables to be used in template"),
flagSet.StringVarP(&options.ClientCertFile, "client-cert", "cc", "", "client certificate file (PEM-encoded) used for authenticating against scanned hosts"),
@@ -412,7 +408,6 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 0, "max response size to read in bytes"),
flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", unitutils.Mega, "max response size to read in bytes"),
flagSet.IntVarP(&options.RequestSaveSize, "request-size-save", "rqss", 32*unitutils.Kilo, "max request body size to save in bytes (0 to save the whole body)"),
flagSet.IntVarP(&options.MaxRequestBodyInFlight, "max-request-body-inflight", "mrbi", 0, "max request body bytes held by in-flight http requests; requests with bodies of 1MB or more queue past it (0 disables)"),
flagSet.CallbackVar(func() { resetRequested = true }, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"),
+15 -11
View File
@@ -1,6 +1,6 @@
module github.com/projectdiscovery/nuclei/v3
go 1.27.4
go 1.27.3
require (
carvel.dev/ytt v0.52.0
@@ -18,7 +18,7 @@ require (
github.com/alecthomas/chroma/v2 v2.27.0
github.com/alexsnet/go-vnc v0.1.0
github.com/alitto/pond v1.9.2
github.com/andybalholm/brotli v1.2.7
github.com/andybalholm/brotli v1.2.0
github.com/andygrunwald/go-jira v1.16.1
github.com/antchfx/htmlquery v1.3.7
github.com/antchfx/xmlquery v1.4.4
@@ -46,6 +46,7 @@ require (
github.com/go-pdf/fpdf v0.9.0
github.com/go-pg/pg/v10 v10.15.0
github.com/go-playground/validator/v10 v10.26.0
github.com/go-rod/rod v0.116.2
github.com/go-sql-driver/mysql v1.10.0
github.com/gobwas/ws v1.4.0
github.com/goccy/go-json v0.10.5
@@ -62,7 +63,7 @@ require (
github.com/jhump/protoreflect v1.17.0
github.com/julienschmidt/httprouter v1.3.0
github.com/kitabisa/go-ci v1.0.3
github.com/klauspost/compress v1.20.2
github.com/klauspost/compress v1.18.7
github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa
github.com/lib/pq v1.12.3
github.com/logrusorgru/aurora/v4 v4.0.0
@@ -101,7 +102,6 @@ require (
github.com/projectdiscovery/rdap v0.9.0
github.com/projectdiscovery/retryabledns v1.0.116
github.com/projectdiscovery/retryablehttp-go v1.3.31
github.com/projectdiscovery/rod v0.116.3-0.20261113150229-d775599ff53d
github.com/projectdiscovery/sarif v0.1.0
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75
github.com/projectdiscovery/uncover v1.2.1
@@ -129,14 +129,14 @@ require (
go.mongodb.org/mongo-driver v1.17.9
go.uber.org/multierr v1.11.0
golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3
golang.org/x/net v0.62.0
golang.org/x/net v0.59.0
golang.org/x/oauth2 v0.36.0
golang.org/x/sync v0.24.0
golang.org/x/sys v0.49.0
golang.org/x/term v0.47.0
golang.org/x/text v0.43.0
golang.org/x/sync v0.23.0
golang.org/x/sys v0.48.0
golang.org/x/term v0.46.0
golang.org/x/text v0.42.0
golang.org/x/time v0.15.0
golang.org/x/tools v0.51.0
golang.org/x/tools v0.49.0
google.golang.org/grpc v1.83.2
google.golang.org/protobuf v1.36.11
gopkg.in/yaml.v3 v3.0.1
@@ -427,7 +427,11 @@ require (
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
github.com/ysmood/fetchup v0.2.3 // indirect
github.com/ysmood/goob v0.4.0 // indirect
github.com/ysmood/got v0.40.0 // indirect
github.com/ysmood/gson v0.7.3 // indirect
github.com/ysmood/leakless v0.9.0 // indirect
github.com/yuin/goldmark v1.7.17 // indirect
github.com/yuin/goldmark-emoji v1.0.6 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
@@ -451,7 +455,7 @@ require (
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
goftp.io/server/v2 v2.0.1 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.58.0 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/mod v0.41.0 // indirect
google.golang.org/api v0.264.0 // indirect
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
+30 -24
View File
@@ -170,8 +170,8 @@ github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e
github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
github.com/alitto/pond v1.9.2 h1:9Qb75z/scEZVCoSU+osVmQ0I0JOeLfdTDafrbcJ8CLs=
github.com/alitto/pond v1.9.2/go.mod h1:xQn3P/sHTYcU/1BR3i86IGIrilcrGC2LiS+E2+CJWsI=
github.com/andybalholm/brotli v1.2.7 h1:UgafRPGMam5ySHfRkhthMqOFA6IFoeF3D7mWIDLjG5A=
github.com/andybalholm/brotli v1.2.7/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q38Qufg=
github.com/andybalholm/cascadia v1.3.4/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41GBe/5O+G93kM=
github.com/andygrunwald/go-jira v1.16.1 h1:WoQEar5XoDRAibOgKzTFELlPNlKAtnfWr296R9zdFLA=
@@ -493,6 +493,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k=
github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA=
github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg=
github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
@@ -747,8 +749,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/kitabisa/go-ci v1.0.3 h1:JmIUIvcercRQc/9x/v02ydCCqU4MadSHaNaOF8T2pGA=
github.com/kitabisa/go-ci v1.0.3/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo=
github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
@@ -983,8 +985,6 @@ github.com/projectdiscovery/retryabledns v1.0.116 h1:tK28PJ4tLO7bgM89RpIFjkfTPMS
github.com/projectdiscovery/retryabledns v1.0.116/go.mod h1:r3vH0AKYtJ9fculcqRFrdVYe2ABcHwZeR4C6j3rqCbY=
github.com/projectdiscovery/retryablehttp-go v1.3.31 h1:0hSEZ5+jj6Oo1IL8qrytD36TK/qCxfe543jEaRqSTjs=
github.com/projectdiscovery/retryablehttp-go v1.3.31/go.mod h1:/WEa7YejxTCXEMLCGfGYaC3JsgXgzuqw8rv8cb4buEc=
github.com/projectdiscovery/rod v0.116.3-0.20261113150229-d775599ff53d h1:6mwNBdlYWe0hu1/QLLhIrZiz+G1JYsU7OIVmCOaF+q0=
github.com/projectdiscovery/rod v0.116.3-0.20261113150229-d775599ff53d/go.mod h1:FLVLMgXw6OFcOU5nd4Il8cdSWgDfAlDeRZMol3drmGs=
github.com/projectdiscovery/sarif v0.1.0 h1:O541T+a448nSJsmIMnXXSOeDQEzpnCAYvRfe0eG5h74=
github.com/projectdiscovery/sarif v0.1.0/go.mod h1:LBC+reM3bkI3qIIhE0rZaINaYX6VG+En6u2hHa5mA7E=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -1212,14 +1212,20 @@ github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+Seva
github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
github.com/ysmood/gop v0.4.2 h1:KtW1/RugapWs1iW9ynQdk52BK6TWdP9zC0qDOhW8EDE=
github.com/ysmood/gop v0.4.2/go.mod h1:K4LAB/BdLArknXrVSy+DyXcfb8P85JwAtwYM8zlb0mU=
github.com/ysmood/got v0.43.0 h1:bcHlKg+qnhFz5JxNZ+j4fIfxxB8WWjTh6qdqeS8FfZw=
github.com/ysmood/got v0.43.0/go.mod h1:rsfH2KJ5rcS/5n9pmybIGodoHlT/NTYxjgh3ob7UdSU=
github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ=
github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
github.com/ysmood/gop v0.2.0 h1:+tFrG0TWPxT6p9ZaZs+VY+opCvHU8/3Fk6BaNv6kqKg=
github.com/ysmood/gop v0.2.0/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
github.com/ysmood/got v0.40.0 h1:ZQk1B55zIvS7zflRrkGfPDrPG3d7+JOza1ZkNxcc74Q=
github.com/ysmood/got v0.40.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg=
github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU=
github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -1329,8 +1335,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1418,8 +1424,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY=
golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1446,8 +1452,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1512,8 +1518,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1526,8 +1532,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY=
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1544,8 +1550,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1597,8 +1603,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI=
golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA=
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+8 -58
View File
@@ -25,7 +25,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/csv"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
@@ -64,8 +63,7 @@ func ConfigureOptions() error {
// would hold the scan until it does.
func readsTargetsFromStdin(options *types.Options, stdinIsPipe bool) bool {
hasOtherTargets := len(options.Targets) > 0 || options.TargetsFilePath != "" || options.Uncover || len(options.UncoverQuery) > 0
// a replay runs over stored events and has no targets to wait for
return stdinIsPipe && !options.DisableStdin && !hasOtherTargets && options.ReplayFile == ""
return stdinIsPipe && !options.DisableStdin && !hasOtherTargets
}
// ParseOptions parses the command line flags provided by a user
@@ -121,14 +119,6 @@ func ParseOptions(options *types.Options) {
options.Logger.Debug().Msgf("Store response directory specified, enabling \"store-resp\" flag automatically\n")
options.StoreResponse = true
}
// Derived before validation: -uncover-query turns uncover on, and -plan
// has to reject that before any target lookup starts.
if options.UncoverQuery != nil {
options.Uncover = true
if len(options.UncoverEngine) == 0 {
options.UncoverEngine = append(options.UncoverEngine, "shodan")
}
}
// Validate the options passed by the user and if any
// invalid options have been used, exit.
if err := ValidateOptions(options); err != nil {
@@ -148,6 +138,13 @@ func ParseOptions(options *types.Options) {
_ = os.Setenv("GITHUB_TOKEN", options.GitHubToken)
}
if options.UncoverQuery != nil {
options.Uncover = true
if len(options.UncoverEngine) == 0 {
options.UncoverEngine = append(options.UncoverEngine, "shodan")
}
}
if options.OfflineHTTP {
options.DisableHTTPProbe = true
}
@@ -165,9 +162,6 @@ func ValidateOptions(options *types.Options) error {
}
return errors.Wrap(errors.New(strings.Join(errs, ", ")), "validation failed for these fields")
}
if err := validatePlanOptions(options); err != nil {
return err
}
if options.Verbose && options.Silent {
return errors.New("both verbose and silent mode specified")
}
@@ -373,18 +367,6 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error)
}
}
}
if options.CSVExport != "" {
// Combine the CLI options with the config file options with the CLI options taking precedence
if reportingOptions.CSVExporter != nil {
reportingOptions.CSVExporter.File = options.CSVExport
reportingOptions.CSVExporter.OmitRaw = options.OmitRawRequests
} else {
reportingOptions.CSVExporter = &csv.Options{
File: options.CSVExport,
OmitRaw: options.OmitRawRequests,
}
}
}
if options.PDFExport != "" {
if reportingOptions.PDFExporter != nil {
reportingOptions.PDFExporter.File = options.PDFExport
@@ -554,35 +536,3 @@ func getBoolEnvValue(key string) bool {
value := os.Getenv(key)
return strings.EqualFold(value, "true")
}
// validatePlanOptions rejects options that change the templates or targets
// only once scanning starts, which a plan written beforehand cannot show.
func validatePlanOptions(options *types.Options) error {
if options.RunPlanFile != "" && options.PlanFile != "" {
return errors.New("-plan cannot be used with -run-plan")
}
if options.RunPlanFile != "" && options.Resume != "" {
return errors.New("-run-plan cannot be used with -resume, the plan's done log resumes it")
}
if options.PlanFile == "" && options.RunPlanFile == "" {
return nil
}
// -automatic-scan builds its own engine, so a plan set on the main engine
// would never be consulted. The other flags change the targets only after
// a plan has been written.
flag := "-plan"
if options.PlanFile == "" {
flag = "-run-plan"
}
switch {
case options.AutomaticScan:
return fmt.Errorf("%s cannot be used with -automatic-scan, which picks templates while scanning", flag)
case options.Uncover || len(options.UncoverQuery) > 0:
return fmt.Errorf("%s cannot be used with -uncover, which finds targets over the network", flag)
case options.PreflightPortScan:
return fmt.Errorf("%s cannot be used with -preflight-portscan, which drops targets after scanning their ports", flag)
case options.DASTServer || options.DASTProxy:
return fmt.Errorf("%s cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan", flag)
}
return nil
}
-1
View File
@@ -84,7 +84,6 @@ func TestReadsTargetsFromStdin(t *testing.T) {
{"targets given with -l", &types.Options{TargetsFilePath: "targets.txt"}, true, false},
{"uncover query", &types.Options{UncoverQuery: goflags.StringSlice{"example"}}, true, false},
{"uncover enabled", &types.Options{Uncover: true}, true, false},
{"replay", &types.Options{ReplayFile: "events.jsonl"}, true, false},
} {
t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.want, readsTargetsFromStdin(tc.options, tc.stdinIsPipe))
-98
View File
@@ -1,98 +0,0 @@
package runner
import (
"bufio"
"os"
"slices"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
"github.com/projectdiscovery/nuclei/v3/pkg/core"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
)
// httpRequestSeconds is the time of one http request the plan costs are
// weighted against (see core.PlanEntry.Cost).
const httpRequestSeconds = 0.015
// planSummary is the last line of a plan file, written once every entry is
// known.
type planSummary struct {
Entries int `json:"entries"`
// Alternatives are entries that run only when another of their origin
// fails, so they are left out of every other total.
Alternatives int `json:"alternatives,omitempty"`
Requests int `json:"requests"`
Cost int `json:"cost"`
// EstimatedSeconds is the largest of the time the rate limit allows, the
// cost spread over every concurrent slot, and the longest entry, whose
// requests run one after another. Hosts slower than the measured request
// time make the scan take longer.
EstimatedSeconds int `json:"estimated-seconds"`
}
// writePlan writes one JSON line per unit of work the scan would run,
// followed by the summary. Targets are listed as given: inputs that the scan
// would probe to an http url first are not probed here, since probing sends
// requests.
func (r *Runner) writePlan(store *loader.Store, engine *core.Engine) error {
if !r.options.DisableHTTPProbe && loader.IsHTTPBasedProtocolUsed(store) && r.isInputNonHTTP() {
r.Logger.Info().Msgf("Targets without a scheme are planned as given; the scan probes them for http(s) first")
}
file, err := os.Create(r.options.PlanFile)
if err != nil {
return err
}
defer func() { _ = file.Close() }()
writer := bufio.NewWriter(file)
encoder := json.NewEncoder(writer)
var summary planSummary
var headlessCost, longest int
var encodeErr error
engine.Plan(slices.Concat(store.Templates(), store.Workflows()), r.inputProvider, r.options.DisableClustering, func(entry core.PlanEntry) {
if encodeErr != nil {
return
}
encodeErr = encoder.Encode(entry)
if entry.AlternativeTo != "" {
summary.Alternatives++
return
}
summary.Entries++
summary.Requests += entry.Requests
summary.Cost += entry.Cost
longest = max(longest, entry.Cost)
if entry.Protocol == types.HeadlessProtocol.String() {
headlessCost += entry.Cost
}
})
if encodeErr != nil {
return encodeErr
}
summary.EstimatedSeconds = r.estimateSeconds(summary.Requests, summary.Cost-headlessCost, headlessCost, longest)
if err := encoder.Encode(map[string]planSummary{"summary": summary}); err != nil {
return err
}
if err := writer.Flush(); err != nil {
return err
}
r.Logger.Info().Msgf("Plan written to %s: %d template runs, up to %d requests, about %ds", r.options.PlanFile, summary.Entries, summary.Requests, summary.EstimatedSeconds)
return nil
}
// estimateSeconds estimates the scan duration. Headless templates run in
// their own, smaller pool, so their cost is spread over it separately.
func (r *Runner) estimateSeconds(requests, cost, headlessCost, longest int) int {
seconds := float64(longest) * httpRequestSeconds
if r.options.RateLimit > 0 && r.options.RateLimitDuration > 0 {
seconds = max(seconds, float64(requests)/(float64(r.options.RateLimit)/r.options.RateLimitDuration.Seconds()))
}
if slots := r.options.TemplateThreads * r.options.BulkSize; slots > 0 {
seconds = max(seconds, float64(cost)*httpRequestSeconds/float64(slots))
}
if slots := r.options.HeadlessTemplateThreads * r.options.HeadlessBulkSize; slots > 0 {
seconds = max(seconds, float64(headlessCost)*httpRequestSeconds/float64(slots))
}
return int(seconds + 0.5)
}
-41
View File
@@ -1,41 +0,0 @@
package runner
import (
"testing"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestPlanRejectsOptionsThatChangeTargetsLater(t *testing.T) {
for name, options := range map[string]*types.Options{
"automatic scan": {PlanFile: "p", AutomaticScan: true},
"uncover": {PlanFile: "p", Uncover: true},
"uncover query": {PlanFile: "p", UncoverQuery: goflags.StringSlice{"example"}},
"preflight": {PlanFile: "p", PreflightPortScan: true},
"dast server": {PlanFile: "p", DASTServer: true},
"dast proxy": {PlanFile: "p", DASTProxy: true},
} {
require.Error(t, validatePlanOptions(options), name)
}
require.NoError(t, validatePlanOptions(&types.Options{PlanFile: "p"}))
require.NoError(t, validatePlanOptions(&types.Options{Uncover: true}), "only -plan restricts them")
require.NoError(t, validatePlanOptions(&types.Options{UncoverQuery: goflags.StringSlice{"example"}}))
require.NoError(t, validatePlanOptions(&types.Options{DASTServer: true}))
require.Error(t, validatePlanOptions(&types.Options{RunPlanFile: "p", PlanFile: "q"}))
require.Error(t, validatePlanOptions(&types.Options{RunPlanFile: "p", Resume: "r"}))
require.NoError(t, validatePlanOptions(&types.Options{RunPlanFile: "p"}))
for name, options := range map[string]*types.Options{
"run automatic scan": {RunPlanFile: "p", AutomaticScan: true},
"run uncover": {RunPlanFile: "p", Uncover: true},
"run uncover query": {RunPlanFile: "p", UncoverQuery: goflags.StringSlice{"example"}},
"run preflight": {RunPlanFile: "p", PreflightPortScan: true},
"run dast server": {RunPlanFile: "p", DASTServer: true},
"run dast proxy": {RunPlanFile: "p", DASTProxy: true},
} {
require.Error(t, validatePlanOptions(options), name)
}
}
-24
View File
@@ -1,24 +0,0 @@
package runner
import (
"testing"
"time"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestEstimateSecondsTakesTheSlowestBound(t *testing.T) {
r := &Runner{options: &types.Options{
RateLimit: 150, RateLimitDuration: time.Second,
TemplateThreads: 25, BulkSize: 25,
HeadlessTemplateThreads: 10, HeadlessBulkSize: 10,
}}
require.Equal(t, 20, r.estimateSeconds(3000, 3000, 0, 12), "rate limit bound")
require.Equal(t, 10, r.estimateSeconds(52, 41, 660, 660), "one long entry runs its requests in sequence")
require.Equal(t, 9, r.estimateSeconds(100, 0, 60000, 60), "headless cost spreads over the headless pool")
r.options.RateLimit = 0
require.Equal(t, 0, r.estimateSeconds(3000, 3000, 0, 1), "no rate limit, cost spread over 625 slots")
}
-24
View File
@@ -1,24 +0,0 @@
package runner
import (
"slices"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
"github.com/projectdiscovery/nuclei/v3/pkg/replay"
)
// replay writes the findings of the loaded templates over a stored corpus
// through the usual output, so -o, -j and the other output options apply.
func (r *Runner) replay(store *loader.Store) error {
results, err := replay.Replay(r.options.ReplayFile, slices.Concat(store.Templates(), store.Workflows()))
if err != nil {
return err
}
for _, result := range results {
if err := r.output.Write(result); err != nil {
return err
}
}
r.Logger.Info().Msgf("Replayed %s: %d findings", r.options.ReplayFile, len(results))
return nil
}
-215
View File
@@ -1,215 +0,0 @@
package runner
import (
"bufio"
"bytes"
"fmt"
"io"
"os"
"strconv"
"sync"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
)
// planRun limits a scan to the entries of a plan file and appends the id of
// each finished entry to the plan's done log, so running the plan again skips
// them. Ids are derived from what an entry runs, not from its position, so the
// log still applies after the plan is written again for changed targets or
// templates.
type planRun struct {
mu sync.Mutex
entries map[uint64]plannedEntry
// done holds finished origin groups by the id of their first entry; an
// entry outside a group is its own group
done map[uint64]struct{}
log *os.File
logErr error
// resumed counts the entries finished before this run
resumed int
}
type plannedEntry struct {
group uint64
seen bool
}
func planDoneLog(planFile string) string {
return planFile + ".done"
}
func openPlanRun(planFile string) (*planRun, error) {
run := &planRun{entries: make(map[uint64]plannedEntry), done: make(map[uint64]struct{})}
if err := run.readPlan(planFile); err != nil {
return nil, err
}
if err := run.readDoneLog(planDoneLog(planFile)); err != nil {
return nil, err
}
log, err := os.OpenFile(planDoneLog(planFile), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o644)
if err != nil {
return nil, err
}
// an interrupted write leaves a partial line, which the next id must not
// extend
if info, err := log.Stat(); err == nil && info.Size() > 0 {
last := make([]byte, 1)
if _, err := log.ReadAt(last, info.Size()-1); err == nil && last[0] != '\n' {
_, _ = log.WriteString("\n")
}
}
run.log = log
return run, nil
}
func (run *planRun) readPlan(planFile string) error {
file, err := os.Open(planFile)
if err != nil {
return err
}
defer func() { _ = file.Close() }()
reader := bufio.NewReader(file)
for number := 1; ; number++ {
line, readErr := reader.ReadBytes('\n')
if line = bytes.TrimSpace(line); len(line) > 0 {
var entry struct {
ID string `json:"id"`
AlternativeTo string `json:"alternative-to"`
}
if err := json.Unmarshal(line, &entry); err != nil {
return fmt.Errorf("%s:%d: %w", planFile, number, err)
}
// the summary line has no id
if entry.ID != "" {
id, ok := parsePlanID(entry.ID)
if !ok {
return fmt.Errorf("%s:%d: invalid id %q", planFile, number, entry.ID)
}
group := id
if entry.AlternativeTo != "" {
if group, ok = parsePlanID(entry.AlternativeTo); !ok {
return fmt.Errorf("%s:%d: invalid alternative-to %q", planFile, number, entry.AlternativeTo)
}
}
run.entries[id] = plannedEntry{group: group}
}
}
if readErr == io.EOF {
return nil
}
if readErr != nil {
return readErr
}
}
}
func (run *planRun) readDoneLog(path string) error {
file, err := os.Open(path)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
defer func() { _ = file.Close() }()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
// a partial line from an interrupted write does not parse and is
// run again
id, ok := parsePlanID(scanner.Text())
if !ok {
continue
}
if entry, planned := run.entries[id]; planned {
run.done[entry.group] = struct{}{}
}
}
if err := scanner.Err(); err != nil {
return err
}
for _, entry := range run.entries {
if _, done := run.done[entry.group]; done {
run.resumed++
}
}
return nil
}
// parsePlanID keeps ids as integers, which a plan of millions of entries
// holds in a fraction of the memory of strings.
func parsePlanID(id string) (uint64, bool) {
if len(id) != 16 {
return 0, false
}
value, err := strconv.ParseUint(id, 16, 64)
return value, err == nil
}
func (run *planRun) Skip(id string) bool {
key, ok := parsePlanID(id)
if !ok {
return true
}
run.mu.Lock()
defer run.mu.Unlock()
entry, planned := run.entries[key]
if !planned {
return true
}
if !entry.seen {
entry.seen = true
run.entries[key] = entry
}
_, done := run.done[entry.group]
return done
}
func (run *planRun) Finished(id string) {
key, ok := parsePlanID(id)
if !ok {
return
}
run.mu.Lock()
defer run.mu.Unlock()
entry, planned := run.entries[key]
if !planned {
return
}
run.done[entry.group] = struct{}{}
// written straight to the file: ctrl+c exits without running defers
if _, err := run.log.WriteString(id + "\n"); err != nil && run.logErr == nil {
run.logErr = err
}
}
// unreached counts unfinished entries the scan did not get to: those of a
// host it found unresponsive, and those it does not have because it runs
// with other templates, targets or options than the plan.
func (run *planRun) unreached() int {
run.mu.Lock()
defer run.mu.Unlock()
count := 0
for _, entry := range run.entries {
if _, done := run.done[entry.group]; !done && !entry.seen {
count++
}
}
return count
}
func (run *planRun) close() error {
if err := run.log.Close(); err != nil && run.logErr == nil {
run.logErr = err
}
return run.logErr
}
// finishPlanRun closes the done log and reports the entries left to run.
func (r *Runner) finishPlanRun(run *planRun) {
if err := run.close(); err != nil {
r.Logger.Error().Msgf("Could not write %s: %s", planDoneLog(r.options.RunPlanFile), err)
}
if unreached := run.unreached(); unreached > 0 {
r.Logger.Info().Msgf("%d plan entries did not run; running the plan again retries them. Entries run only when the scan has the templates, targets and options the plan was written with", unreached)
}
}
-52
View File
@@ -1,52 +0,0 @@
package runner
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
)
func TestPlanRunResumesFromItsDoneLog(t *testing.T) {
planFile := filepath.Join(t.TempDir(), "plan.jsonl")
require.NoError(t, os.WriteFile(planFile, []byte(`{"id":"00000000000000a1","target":"https://a.example/x","templates":["root"]}
{"id":"00000000000000a2","target":"https://a.example/y","templates":["root"],"alternative-to":"00000000000000a1"}
{"id":"00000000000000b1","target":"https://b.example","templates":["generic"]}
{"id":"00000000000000c1","target":"https://c.example","templates":["generic"]}
{"summary":{"entries":3,"alternatives":1}}
`), 0o644))
// the last line was cut off by an interrupted write
require.NoError(t, os.WriteFile(planDoneLog(planFile), []byte("00000000000000b1\n00000000"), 0o644))
run, err := openPlanRun(planFile)
require.NoError(t, err)
require.Len(t, run.entries, 4)
require.Equal(t, 1, run.resumed)
require.True(t, run.Skip("00000000000000b1"), "finished in the previous run")
require.True(t, run.Skip("00000000000000ff"), "not in the plan")
require.False(t, run.Skip("00000000000000a1"))
run.Finished("00000000000000a1")
require.True(t, run.Skip("00000000000000a2"), "its origin is covered")
require.Equal(t, 1, run.unreached(), "c1 never ran")
require.NoError(t, run.close())
log, err := os.ReadFile(planDoneLog(planFile))
require.NoError(t, err)
require.Equal(t, "00000000000000b1\n00000000\n00000000000000a1\n", string(log), "the partial line is not extended")
again, err := openPlanRun(planFile)
require.NoError(t, err)
require.Equal(t, 3, again.resumed, "b1, and a1 with its alternative")
require.True(t, again.Skip("00000000000000a2"), "a finished origin stays covered")
require.False(t, again.Skip("00000000000000c1"))
require.NoError(t, again.close())
}
func TestPlanRunRejectsAMalformedPlan(t *testing.T) {
planFile := filepath.Join(t.TempDir(), "plan.jsonl")
require.NoError(t, os.WriteFile(planFile, []byte("{\"id\":\"00000000000000a1\"}\nnot json\n"), 0o644))
_, err := openPlanRun(planFile)
require.ErrorContains(t, err, "plan.jsonl:2")
}
+9 -58
View File
@@ -57,7 +57,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/honeypotdetector"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
@@ -70,12 +69,10 @@ import (
httpProtocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/catchall"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/replay"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
"github.com/projectdiscovery/retryablehttp-go"
@@ -149,18 +146,10 @@ func New(options *types.Options) (*Runner, error) {
// Version check by default
if config.DefaultConfig.CanCheckForUpdates() {
// -plan sends no request it can avoid, so it skips the version and
// update checks; it still installs templates on a fresh setup, which
// would otherwise plan nothing
checkUpdates := options.PlanFile == ""
var latestIgnoreHash string
if checkUpdates {
var versionCheckErr error
latestIgnoreHash, versionCheckErr = installer.NucleiVersionCheck()
if versionCheckErr != nil {
if options.Verbose || options.Debug {
runner.Logger.Error().Msgf("nuclei version check failed got: %s\n", versionCheckErr)
}
latestIgnoreHash, versionCheckErr := installer.NucleiVersionCheck()
if versionCheckErr != nil {
if options.Verbose || options.Debug {
runner.Logger.Error().Msgf("nuclei version check failed got: %s\n", versionCheckErr)
}
}
@@ -181,19 +170,17 @@ func New(options *types.Options) (*Runner, error) {
if err := tm.FreshInstallIfNotExists(); err != nil {
runner.Logger.Warning().Msgf("Failed to install nuclei templates: %s\n", err)
}
if checkUpdates {
if err := tm.UpdateIfOutdated(); err != nil {
runner.Logger.Warning().Msgf("Failed to update nuclei templates: %s\n", err)
}
if err := tm.UpdateIfOutdated(); err != nil {
runner.Logger.Warning().Msgf("Failed to update nuclei templates: %s\n", err)
}
if checkUpdates && config.DefaultConfig.IgnoreFileNeedsUpdate(latestIgnoreHash) {
if config.DefaultConfig.IgnoreFileNeedsUpdate(latestIgnoreHash) {
if err := installer.UpdateIgnoreFile(); err != nil {
runner.Logger.Warning().Msgf("Failed to update .nuclei-ignore file: %s\n", err)
}
}
if checkUpdates && options.UpdateTemplates {
if options.UpdateTemplates {
// we automatically check for updates unless explicitly disabled
// this print statement is only to inform the user that there are no updates
noUpdates := !config.DefaultConfig.NeedsTemplateUpdate()
@@ -604,13 +591,8 @@ func (r *Runner) RunEnumeration() error {
httpclientpool.ResetConnectionStats()
// If the user has asked for DAST server mode, run the live
// DAST fuzzing server. -plan never reaches this with those flags set:
// validation rejects the combination, and this check covers a caller
// that skipped it.
// DAST fuzzing server.
if r.options.DASTServer || r.options.DASTProxy {
if err := validatePlanOptions(r.options); err != nil {
return err
}
execurOpts := &server.NucleiExecutorOptions{
Options: r.options,
Output: r.output,
@@ -694,7 +676,6 @@ func (r *Runner) RunEnumeration() error {
Catalog: r.catalog,
IssuesClient: r.issuesClient,
RateLimiter: r.rateLimiter,
RequestBodyBudget: httpbody.NewBudget(int64(r.options.MaxRequestBodyInFlight)),
Interactsh: r.interactsh,
ProjectFile: r.projectFile,
Browser: r.browser,
@@ -834,36 +815,6 @@ func (r *Runner) RunEnumeration() error {
profiles.Prepare(slices.Concat(store.Templates(), store.Workflows()))
executorOpts.TargetScope = profiles
}
// before uncover, preflight, auth prefetch and probing below, all of which
// reach the network
if r.options.PlanFile != "" {
return r.writePlan(store, executorEngine)
}
if r.options.ReplayFile != "" {
return r.replay(store)
}
if r.options.StoreEvents != "" {
recorder, err := replay.NewRecorder(r.options.StoreEvents, slices.Concat(store.Templates(), store.Workflows()))
if err != nil {
return errors.Wrap(err, "could not create events file")
}
previous := eventcreator.SetRecord(recorder.Record)
defer func() {
eventcreator.SetRecord(previous)
if err := recorder.Close(); err != nil {
r.Logger.Error().Msgf("Could not write %s: %s", r.options.StoreEvents, err)
}
}()
}
if r.options.RunPlanFile != "" {
run, err := openPlanRun(r.options.RunPlanFile)
if err != nil {
return errors.Wrap(err, "could not read plan")
}
r.Logger.Info().Msgf("Running %d plan entries from %s, %d already finished", len(run.entries), r.options.RunPlanFile, run.resumed)
executorEngine.SetPlanRun(run)
defer r.finishPlanRun(run)
}
// TODO: remove below functions after v3 or update warning messages
templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
-29
View File
@@ -38,35 +38,6 @@ func TestCreateReportingOptions(t *testing.T) {
require.Equal(t, resultOptions2.DenyList.Severities, resultOptions.DenyList.Severities)
}
func TestCreateReportingOptionsCSVExport(t *testing.T) {
t.Run("flag only", func(t *testing.T) {
resultOptions, err := createReportingOptions(&types.Options{CSVExport: "out.csv", OmitRawRequests: true})
require.NoError(t, err)
require.NotNil(t, resultOptions.CSVExporter)
require.Equal(t, "out.csv", resultOptions.CSVExporter.File)
require.True(t, resultOptions.CSVExporter.OmitRaw)
})
t.Run("flag is not set", func(t *testing.T) {
resultOptions, err := createReportingOptions(&types.Options{})
require.NoError(t, err)
require.Nil(t, resultOptions.CSVExporter)
})
t.Run("flag takes precedence over config file", func(t *testing.T) {
config := filepath.Join(t.TempDir(), "report.yaml")
require.NoError(t, os.WriteFile(config, []byte("csv:\n file: from-config.csv\n"), 0o600))
resultOptions, err := createReportingOptions(&types.Options{ReportingConfig: config})
require.NoError(t, err)
require.Equal(t, "from-config.csv", resultOptions.CSVExporter.File)
resultOptions, err = createReportingOptions(&types.Options{ReportingConfig: config, CSVExport: "from-flag.csv"})
require.NoError(t, err)
require.Equal(t, "from-flag.csv", resultOptions.CSVExporter.File)
})
}
type TestStruct1 struct {
A string `yaml:"a"`
Struct *TestStruct2 `yaml:"b"`
@@ -46,35 +46,6 @@ func TestNetwork(t *testing.T) {
}
})
t.Run("ReplayStoredEvents", func(t *testing.T) {
server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
defer func() { _ = conn.Close() }()
data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
if err == nil && string(data) == "PING" {
_, _ = conn.Write([]byte("PONG"))
}
})
events := filepath.Join(t.TempDir(), "events.jsonl")
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/basic.yaml", server.URL, suite.debug, "-store-events", events)
server.Close()
if err != nil {
t.Fatalf("recording network events failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
// the server is gone, so the finding can only come from the stored event
results, err = testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil, "-t", "protocols/network/basic.yaml", "-replay", events)
if err != nil {
t.Fatalf("replaying network events failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("PinnedTargetLine", func(t *testing.T) {
server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
defer func() { _ = conn.Close() }()
+6 -8
View File
@@ -39,14 +39,12 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
Catalog: base.catalog,
IssuesClient: base.rc,
RateLimiter: base.rateLimiter,
RequestBodyBudget: base.executerOpts.RequestBodyBudget,
Interactsh: base.interactshClient,
Colorizer: aurora.New(aurora.WithColors(true)),
ResumeCfg: types.NewResumeCfg(),
Parser: base.parser,
Browser: base.browserInstance,
DoNotCache: opts.DoNotCacheTemplates,
Interactsh: base.interactshClient,
Colorizer: aurora.New(aurora.WithColors(true)),
ResumeCfg: types.NewResumeCfg(),
Parser: base.parser,
Browser: base.browserInstance,
DoNotCache: opts.DoNotCacheTemplates,
}
if base.interactshClient != nil {
u.executerOpts.InteractshScope = base.interactshClient.NewRequestScope()
-2
View File
@@ -36,7 +36,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
nucleiUtils "github.com/projectdiscovery/nuclei/v3/pkg/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
"github.com/projectdiscovery/ratelimit"
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
)
@@ -280,7 +279,6 @@ func (e *NucleiEngine) init(ctx context.Context) error {
Catalog: e.catalog,
IssuesClient: e.rc,
RateLimiter: e.rateLimiter,
RequestBodyBudget: httpbody.NewBudget(int64(e.opts.MaxRequestBodyInFlight)),
Interactsh: e.interactshClient,
Colorizer: aurora.New(aurora.WithColors(true)),
ResumeCfg: types.NewResumeCfg(),
-128
View File
@@ -2,7 +2,6 @@ package loader
import (
"bytes"
"crypto/sha256"
"os"
"path/filepath"
"reflect"
@@ -20,7 +19,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
templatesigner "github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
"github.com/stretchr/testify/require"
@@ -687,135 +685,9 @@ http:
loaded := loadSingleTemplateForTest(t, templatePath, "loader-unsigned-flow-template")
require.Len(t, loaded, 1)
require.Equal(t, "unsigned-flow-template", loaded[0].ID)
require.False(t, loaded[0].Verified)
require.Equal(t, initialUnverifiedJavascript, stats.GetValue(templates.SkippedUnverifiedJavascriptTemplateStats))
}
func contentDigestForLoaderTest(data []byte) [32]byte {
dataDigest := sha256.Sum256(data)
return sha256.Sum256(dataDigest[:])
}
func loadTemplateWithPoisonedMetadata(t *testing.T, templatePath, executionID string, source []byte, disableUnsigned bool) []*templates.Template {
t.Helper()
fileInfo, err := os.Stat(templatePath)
require.NoError(t, err)
metadataIndex, err := metadataindex.NewIndex(t.TempDir())
require.NoError(t, err)
metadataIndex.Set(templatePath, &metadataindex.Metadata{
ID: "poisoned",
FilePath: templatePath,
ModTime: fileInfo.ModTime(),
Name: "Poisoned",
Authors: []string{"pdteam"},
Severity: "info",
Verified: true,
TemplateVerifier: "projectdiscovery/nuclei-templates",
VerifierFingerprint: templatesigner.DefaultTemplateVerifiers[0].Fingerprint(),
ContentDigest: contentDigestForLoaderTest(source),
Validation: metadataindex.ValidationStrict,
})
options := testutils.DefaultOptions.Copy()
options.Logger = &gologger.Logger{}
options.ExecutionId = executionID
options.DisableUnsignedTemplates = disableUnsigned
options.TemplateLoadingConcurrency = 1
testutils.Init(options)
t.Cleanup(func() {
testutils.Cleanup(options)
})
catalog := disk.NewCatalog("")
executerOpts := testutils.NewMockExecuterOptions(options, nil)
executerOpts.Catalog = catalog
executerOpts.Parser = templates.NewParser()
executerOpts.Logger = options.Logger
workflowLoader, err := workflow.NewLoader(executerOpts)
require.NoError(t, err)
executerOpts.WorkflowLoader = workflowLoader
loaderConfig := NewConfig(options, catalog, executerOpts)
loaderConfig.MetadataIndex = metadataIndex
store, err := New(loaderConfig)
require.NoError(t, err)
loaded, err := store.LoadTemplates([]string{templatePath})
require.NoError(t, err)
return loaded
}
func TestLoadTemplatesIgnoresSelfConsistentCacheForJavascript(t *testing.T) {
templatePath := filepath.Join(t.TempDir(), "poisoned-javascript.yaml")
source := []byte(`id: poisoned-cached-javascript
info:
name: Poisoned Cached Javascript
author: pdteam
severity: info
javascript:
- init: |
set("init-status", "executed")
code: |
Export("poisoned-cached-javascript")
`)
require.NoError(t, os.WriteFile(templatePath, source, 0o600))
loaded := loadTemplateWithPoisonedMetadata(t, templatePath, "loader-poisoned-javascript", source, false)
require.Empty(t, loaded)
}
func TestLoadTemplatesAllowsUnsignedFlowDespitePoisonedCache(t *testing.T) {
templatePath := filepath.Join(t.TempDir(), "poisoned-flow.yaml")
source := []byte(`id: poisoned-cached-flow
info:
name: Poisoned Cached Flow
author: pdteam
severity: info
flow: http(1)
http:
- method: GET
path:
- "{{BaseURL}}"
`)
require.NoError(t, os.WriteFile(templatePath, source, 0o600))
loaded := loadTemplateWithPoisonedMetadata(t, templatePath, "loader-poisoned-flow", source, false)
require.Len(t, loaded, 1)
require.Equal(t, "poisoned-cached-flow", loaded[0].ID)
require.False(t, loaded[0].Verified)
require.True(t, loaded[0].IsFlowTemplate())
}
func TestLoadTemplatesSkipsPoisonedHttpWhenUnsignedDisabled(t *testing.T) {
templatePath := filepath.Join(t.TempDir(), "poisoned-http.yaml")
source := []byte(`id: poisoned-cached-http
info:
name: Poisoned Cached HTTP
author: pdteam
severity: info
http:
- method: GET
path:
- "{{BaseURL}}"
`)
require.NoError(t, os.WriteFile(templatePath, source, 0o600))
initialUnverified := stats.GetValue(templates.SkippedUnverifiedTemplateStats)
loaded := loadTemplateWithPoisonedMetadata(t, templatePath, "loader-poisoned-http-dut", source, true)
require.Empty(t, loaded)
require.Equal(t, initialUnverified+1, stats.GetValue(templates.SkippedUnverifiedTemplateStats))
}
func TestLoadTemplatesDoesNotRequireGlobalMatchersFlagToLoadTemplate(t *testing.T) {
templatePath := filepath.Join(t.TempDir(), "global-matchers.yaml")
err := os.WriteFile(templatePath, []byte(`id: global-matchers-template
-1
View File
@@ -51,7 +51,6 @@ type Engine struct {
executerOpts *protocols.ExecutorOptions
Callback func(*output.ResultEvent) // Executed on results
templateExecutionCallback TemplateExecutionCallback
planRun PlanRun
Logger *gologger.Logger
}
+13 -17
View File
@@ -9,7 +9,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
@@ -36,11 +35,22 @@ func (e *Engine) ExecuteWithResults(ctx context.Context, templatesList []*templa
func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider, noCluster bool) *atomic.Bool {
results := &atomic.Bool{}
selfcontainedWg := &sync.WaitGroup{}
httpclientpool.ResetSharedResponses(e.options)
totalReqBeforeCluster := getRequestCount(templatesList) * int(target.Count())
finalTemplates, clusterCount := e.clusterTemplates(templatesList, noCluster)
// attempt to cluster templates if noCluster is false
var finalTemplates []*templates.Template
clusterCount := 0
if !noCluster {
var clusterMappings map[string][]string
finalTemplates, clusterCount, clusterMappings = templates.ClusterTemplates(templatesList, e.executerOpts)
// Store cluster mappings in executerOpts for SDK access (thread-safe)
if clusterMappings != nil {
e.executerOpts.ClusterMappings = types.NewClusterMappingsMap(clusterMappings)
}
} else {
finalTemplates = templatesList
}
totalReqAfterClustering := getRequestCount(finalTemplates) * int(target.Count())
@@ -176,20 +186,6 @@ func (e *Engine) executeHostSpray(ctx context.Context, templatesList []*template
return results
}
// clusterTemplates merges templates that send identical requests unless
// noCluster is set, and returns the templates to run with the number merged.
func (e *Engine) clusterTemplates(templatesList []*templates.Template, noCluster bool) ([]*templates.Template, int) {
if noCluster {
return templatesList, 0
}
finalTemplates, clusterCount, clusterMappings := templates.ClusterTemplates(templatesList, e.executerOpts)
// Store cluster mappings in executerOpts for SDK access (thread-safe)
if clusterMappings != nil {
e.executerOpts.ClusterMappings = types.NewClusterMappingsMap(clusterMappings)
}
return finalTemplates, clusterCount
}
// expectedWork returns the requests and the tasks the target-bound templates
// will run. It counts only the template and target pairs that per-target
// profiles select, so progress reflects the work that will be done. A task is
+7 -32
View File
@@ -36,13 +36,9 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
if sharedBudget {
defer e.options.ReleaseTemplateThread()
}
defer e.tasksDone(1)
planned, skip := e.claimPlanned(template, &contextargs.MetaInput{})
if skip {
return
}
finished := e.templateExecutionStarted(template, "")
defer func() { finished(ctx.Err()) }()
defer e.tasksDone(1)
var err error
var match bool
ctx := scan.NewScanContext(ctx, contextargs.New(ctx))
@@ -59,8 +55,6 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
}
if err != nil {
e.options.Logger.Warning().Msgf("[%s] Could not execute step (self-contained): %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
} else if ctx.Context().Err() == nil {
planned()
}
results.CompareAndSwap(false, match)
}(v, usesSharedTemplateBudget)
@@ -336,16 +330,6 @@ func (e *Engine) tasksDone(count int64) {
func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates.Template, value *contextargs.MetaInput) (matched bool, err error) {
// a pair skipped as a repeat of its origin is done as well
defer e.tasksDone(1)
planned, skip := e.claimPlanned(template, value)
if skip {
return false, nil
}
// an origin a sibling already covered is settled too, so it is recorded
defer func() {
if err == nil && ctx.Err() == nil {
planned()
}
}()
// A template fixed to the target's origin makes the same requests for every
// target that shares one, which a crawled list produces by the hundred.
// Running it once per origin sends exactly the same traffic, minus the
@@ -398,19 +382,6 @@ func newOriginAttempt() *originAttempt {
return attempt
}
// originScopeKey returns the key an origin-scoped template runs once per, and
// false when the template runs on every target.
func originScopeKey(template *templates.Template, value *contextargs.MetaInput) (string, bool) {
if template == nil || value == nil || !template.IsOriginScoped() {
return "", false
}
origin := normalize.Origin(value.Input)
if origin == "" {
return "", false
}
return template.ID + "\x00" + origin + "\x00" + value.CustomIP, true
}
// claimOriginScoped reports whether this caller should execute an origin-scoped
// template. finish(true) records a successful run. finish(false) releases the
// claim so a later target, including one that waited, can try again. Templates
@@ -418,10 +389,14 @@ func originScopeKey(template *templates.Template, value *contextargs.MetaInput)
// key so each address selected for one URL is tested on its own.
func (e *Engine) claimOriginScoped(template *templates.Template, value *contextargs.MetaInput) (finish func(bool), run bool) {
noop := func(bool) {}
key, scoped := originScopeKey(template, value)
if !scoped {
if template == nil || value == nil || !template.IsOriginScoped() {
return noop, true
}
origin := normalize.Origin(value.Input)
if origin == "" {
return noop, true
}
key := template.ID + "\x00" + origin + "\x00" + value.CustomIP
actual, _ := e.originScoped.LoadOrStore(key, newOriginAttempt())
slot := actual.(*originAttempt)
-197
View File
@@ -1,197 +0,0 @@
package core
import (
"crypto/sha256"
"encoding/hex"
"slices"
"strings"
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
)
// PlanEntry is one unit of work a scan runs: a template, a cluster of
// templates sharing one request, or a workflow, on one target.
type PlanEntry struct {
// ID identifies the entry across runs of the same plan.
ID string `json:"id"`
// Target is empty for self-contained templates, which run once.
Target string `json:"target,omitempty"`
// IP is the address the target is pinned to, when it is.
IP string `json:"ip,omitempty"`
Templates []string `json:"templates"`
Protocol string `json:"protocol"`
// Requests is what the templates declare, the count -stats starts from.
// The scan sends fewer when a request needs a value the target lacks,
// such as {{DN}} on an ip.
Requests int `json:"requests"`
// Cost is Requests weighted by how long one request of the protocol
// takes, in http requests.
Cost int `json:"cost"`
// Dynamic marks entries whose requests depend on responses: workflows
// (only the root is counted), flows and fuzzing.
Dynamic bool `json:"dynamic,omitempty"`
// AlternativeTo is the id of an entry running the same origin-scoped
// template on the same origin. Workers race for an origin, so the scan
// runs whichever of the group claims it first, and another only when
// that run fails. Totals count the group once.
AlternativeTo string `json:"alternative-to,omitempty"`
}
// Plan reports the work ExecuteScanWithOpts would do with the same arguments,
// without sending anything. It applies the same clustering, per-target
// profiles and once-per-origin rule, so a pair is listed only if the scan
// would run it. Later targets of an origin-scoped template are listed as
// alternatives to the first, since the scan retries them when a run fails.
func (e *Engine) Plan(templatesList []*templates.Template, target provider.InputProvider, noCluster bool, emit func(PlanEntry)) {
finalTemplates, _ := e.clusterTemplates(templatesList, noCluster)
// clustering does not keep an order; a plan is diffed and edited, so it does
finalTemplates = slices.SortedFunc(slices.Values(finalTemplates), func(a, b *templates.Template) int {
return strings.Compare(a.ID, b.ID)
})
var targetBound []*templates.Template
for _, template := range finalTemplates {
if template.SelfContained {
emit(withPlanID(planEntry(template, &contextargs.MetaInput{}, nil)))
} else {
targetBound = append(targetBound, template)
}
}
// entries are emitted as they are planned, so memory does not grow with
// the plan; only the first id per origin is kept
originIDs := make(map[string]string)
target.Iterate(func(value *contextargs.MetaInput) bool {
selection := e.selectionFor(value)
for _, template := range targetBound {
if selection != nil && !template.SelectedBy(selection) {
continue
}
entry := withPlanID(planEntry(template, value, selection))
if key, scoped := originScopeKey(template, value); scoped {
if id, seen := originIDs[key]; seen {
entry.AlternativeTo = id
} else {
originIDs[key] = entry.ID
}
}
emit(entry)
}
return true
})
}
func withPlanID(entry PlanEntry) PlanEntry {
entry.ID = planEntryID(entry)
return entry
}
// protocolCost is how many http requests one request of a protocol is worth
// in time. Measured per call against a CDN-fronted site, where an http
// request took about 15ms: a javascript call adds about 150ms of runtime, a
// headless page load with waitload took about 1s. Unlisted protocols cost
// about one round trip, like http.
var protocolCost = map[types.ProtocolType]int{
types.JavascriptProtocol: 10,
types.HeadlessProtocol: 60,
}
func planEntry(template *templates.Template, value *contextargs.MetaInput, selection protocols.TemplateSelection) PlanEntry {
entry := PlanEntry{
Target: value.Input,
IP: value.CustomIP,
Templates: planTemplates(template, selection),
Protocol: template.Type().String(),
Requests: template.TotalRequests,
Dynamic: template.Flow != "" || template.IsFuzzableRequest(),
}
weight := func(protocol types.ProtocolType) int { return max(protocolCost[protocol], 1) }
switch {
case template.CompiledWorkflow != nil:
// only the root of a workflow is certain to run; subtemplates depend
// on what it matches
entry.Requests = 0
for _, workflow := range template.CompiledWorkflow.Workflows {
for _, executer := range workflow.Executers {
requests := executer.Executer.Requests()
entry.Requests += requests
entry.Cost += requests * weight(executer.TemplateType)
}
}
entry.Dynamic = true
case isCluster(template):
cluster := template.Executer.(*templates.ClusterExecuter)
// TotalRequests of a cluster counts only its http and dns requests,
// so an ssl cluster would show none
entry.Requests = cluster.Requests()
entry.Cost = entry.Requests * weight(template.Type())
case len(template.RequestsQueue) > 0:
// each block is weighted by its own protocol, so a template mixing
// http with javascript is not priced as all http
for _, request := range template.RequestsQueue {
entry.Cost += request.Requests() * weight(request.Type())
}
default:
entry.Cost = entry.Requests * weight(template.Type())
}
return entry
}
// planTemplates lists the templates an entry runs, sorted so the entry id does
// not depend on clustering order.
func planTemplates(template *templates.Template, selection protocols.TemplateSelection) []string {
if cluster, ok := template.Executer.(*templates.ClusterExecuter); ok {
return slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
}
return []string{template.ID}
}
func isCluster(template *templates.Template) bool {
_, ok := template.Executer.(*templates.ClusterExecuter)
return ok
}
// planEntryID derives the id from what the entry runs, so planning the same
// scan again yields the same ids.
func planEntryID(entry PlanEntry) string {
sum := sha256.Sum256([]byte(entry.Target + "\x00" + entry.IP + "\x00" + strings.Join(entry.Templates, "\x00")))
return hex.EncodeToString(sum[:8])
}
// PlanRun limits a scan to the entries of a plan and records the ones that
// finish, so running the plan again resumes it.
type PlanRun interface {
// Skip reports whether the entry is not in the plan or already finished.
Skip(id string) bool
// Finished records an entry that ran without error.
Finished(id string)
}
// SetPlanRun makes the scan run only the entries run allows.
func (e *Engine) SetPlanRun(run PlanRun) {
e.planRun = run
}
// claimPlanned reports whether the pair is skipped under a plan run, and
// otherwise returns the func that records it finished. Without a plan run
// every pair runs.
func (e *Engine) claimPlanned(template *templates.Template, value *contextargs.MetaInput) (finished func(), skip bool) {
if e.planRun == nil {
return func() {}, false
}
// as in Plan, a self-contained template is not narrowed by any target
var selection protocols.TemplateSelection
if !template.SelfContained {
selection = e.selectionFor(value)
}
id := planEntryID(PlanEntry{Target: value.Input, IP: value.CustomIP, Templates: planTemplates(template, selection)})
if e.planRun.Skip(id) {
return nil, true
}
return func() { e.planRun.Finished(id) }, false
}
-215
View File
@@ -1,215 +0,0 @@
package core
import (
"context"
"sync"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
tmpltypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestPlanEntryCostWeightsByProtocol(t *testing.T) {
headless := &templates.Template{ID: "h", RequestsHeadless: []*headless.Request{{}}, TotalRequests: 2}
require.Equal(t, 120, planEntry(headless, &contextargs.MetaInput{Input: "t"}, nil).Cost)
plain := &templates.Template{ID: "p", TotalRequests: 2}
require.Equal(t, 2, planEntry(plain, &contextargs.MetaInput{Input: "t"}, nil).Cost, "unlisted protocols cost one request each")
}
// countedRequest is a request block with a fixed protocol and count.
type countedRequest struct {
protocols.Request
protocol tmpltypes.ProtocolType
requests int
}
func (r countedRequest) Type() tmpltypes.ProtocolType { return r.protocol }
func (r countedRequest) Requests() int { return r.requests }
func TestPlanEntryWeightsEachProtocolBlock(t *testing.T) {
// one http request and ten javascript calls: Type() reports http for the
// whole template, which would price it as 11
mixed := &templates.Template{
ID: "mixed",
RequestsHTTP: []*http.Request{{}},
TotalRequests: 11,
RequestsQueue: []protocols.Request{
countedRequest{protocol: tmpltypes.HTTPProtocol, requests: 1},
countedRequest{protocol: tmpltypes.JavascriptProtocol, requests: 10},
},
}
entry := planEntry(mixed, &contextargs.MetaInput{Input: "t"}, nil)
require.Equal(t, 11, entry.Requests)
require.Equal(t, 101, entry.Cost)
}
func TestPlanListsThePairsTheScanRuns(t *testing.T) {
options := &types.Options{BulkSize: 2, TemplateThreads: 2, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
engine := New(options)
engine.SetExecuterOptions(&protocols.ExecutorOptions{
Logger: engine.Logger,
Options: options,
ResumeCfg: types.NewResumeCfg(),
ProtocolType: tmpltypes.HTTPProtocol,
Progress: &progressRecorder{},
TargetScope: scopeByInput{
"php": {paths: []string{"thinkphp.yaml"}},
"none": {},
},
})
var (
mu sync.Mutex
runs []string
)
var templatesList []*templates.Template
for _, path := range []string{"thinkphp.yaml", "generic.yaml"} {
templatesList = append(templatesList, &templates.Template{
ID: path,
Path: path,
TotalRequests: 3,
Executer: &recordingExecuter{path: path, mu: &mu, runs: &runs},
})
}
selfContained := &templates.Template{ID: "self.yaml", Path: "self.yaml", SelfContained: true, TotalRequests: 1}
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "php"}, {Input: "none"}, {Input: "plain"}}}
var plan []PlanEntry
engine.Plan(append(templatesList, selfContained), targets, true, func(entry PlanEntry) { plan = append(plan, entry) })
require.Empty(t, plan[0].Target, "a self-contained template runs once, with no target")
require.Equal(t, []string{"self.yaml"}, plan[0].Templates)
var planned []string
ids := make(map[string]struct{})
for _, entry := range plan[1:] {
require.Equal(t, 3, entry.Requests)
planned = append(planned, entry.Templates[0]+"@"+entry.Target)
ids[entry.ID] = struct{}{}
}
require.Len(t, ids, len(plan)-1, "every entry has its own id")
var again []PlanEntry
engine.Plan(append(templatesList, selfContained), targets, true, func(entry PlanEntry) { again = append(again, entry) })
require.Equal(t, plan, again, "planning the same scan again yields the same entries and ids")
engine.ExecuteScanWithOpts(context.Background(), templatesList, targets, true)
require.ElementsMatch(t, runs, planned, "the plan lists exactly the pairs the scan runs")
require.ElementsMatch(t, []string{"thinkphp.yaml@php", "thinkphp.yaml@plain", "generic.yaml@plain"}, planned)
}
func TestPlanListsOriginAlternatives(t *testing.T) {
options := &types.Options{BulkSize: 1, TemplateThreads: 1, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
engine := New(options)
engine.SetExecuterOptions(&protocols.ExecutorOptions{
Logger: engine.Logger,
Options: options,
ResumeCfg: types.NewResumeCfg(),
ProtocolType: tmpltypes.HTTPProtocol,
Progress: &progressRecorder{},
})
root := &templates.Template{
ID: "root.yaml",
RequestsHTTP: []*http.Request{{Path: []string{"{{RootURL}}/health"}}},
TotalRequests: 1,
}
require.True(t, root.IsOriginScoped())
generic := &templates.Template{ID: "generic.yaml", TotalRequests: 1}
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{
{Input: "https://example.com/a"},
{Input: "https://example.com/b"},
{Input: "https://other.example/a"},
{Input: "https://example.com/a", CustomIP: "203.0.113.5"},
}}
var plan []PlanEntry
engine.Plan([]*templates.Template{generic, root}, targets, true, func(entry PlanEntry) {
plan = append(plan, entry)
})
var roots []PlanEntry
for _, entry := range plan {
if entry.Templates[0] == "root.yaml" {
roots = append(roots, entry)
}
}
require.Equal(t, []PlanEntry{
{ID: roots[0].ID, Target: "https://example.com/a", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
{ID: roots[1].ID, Target: "https://example.com/b", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1, AlternativeTo: roots[0].ID},
{ID: roots[2].ID, Target: "https://other.example/a", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
{ID: roots[3].ID, Target: "https://example.com/a", IP: "203.0.113.5", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
}, roots)
require.NotEqual(t, roots[0].ID, roots[3].ID, "a pinned address is its own attempt")
var again []PlanEntry
engine.Plan([]*templates.Template{generic, root}, targets, true, func(entry PlanEntry) {
again = append(again, entry)
})
require.Equal(t, plan, again)
}
// fixedPlanRun is a plan run over fixed ids that records the finished ones.
type fixedPlanRun struct {
mu sync.Mutex
allowed map[string]bool
finished []string
}
func (a *fixedPlanRun) Skip(id string) bool { return !a.allowed[id] }
func (a *fixedPlanRun) Finished(id string) {
a.mu.Lock()
defer a.mu.Unlock()
a.finished = append(a.finished, id)
}
func TestPlanRunRunsOnlyThePlannedPairs(t *testing.T) {
options := &types.Options{BulkSize: 2, TemplateThreads: 2, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
engine := New(options)
engine.SetExecuterOptions(&protocols.ExecutorOptions{
Logger: engine.Logger,
Options: options,
ResumeCfg: types.NewResumeCfg(),
ProtocolType: tmpltypes.HTTPProtocol,
Progress: &progressRecorder{},
TargetScope: scopeByInput{"php": {paths: []string{"thinkphp.yaml"}}},
})
var (
mu sync.Mutex
runs []string
)
var templatesList []*templates.Template
for _, path := range []string{"thinkphp.yaml", "generic.yaml"} {
templatesList = append(templatesList, &templates.Template{
ID: path,
Path: path,
TotalRequests: 1,
Executer: &recordingExecuter{path: path, mu: &mu, runs: &runs},
})
}
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "php"}, {Input: "plain"}}}
var plan []PlanEntry
engine.Plan(templatesList, targets, true, func(entry PlanEntry) { plan = append(plan, entry) })
require.Len(t, plan, 3)
// the first entry is left out of the plan, as if an agent removed it
run := &fixedPlanRun{allowed: map[string]bool{plan[1].ID: true, plan[2].ID: true}}
engine.SetPlanRun(run)
engine.ExecuteScanWithOpts(context.Background(), templatesList, targets, true)
var want []string
for _, entry := range plan[1:] {
want = append(want, entry.Templates[0]+"@"+entry.Target)
}
require.ElementsMatch(t, want, runs, "only the planned pairs run")
require.ElementsMatch(t, []string{plan[1].ID, plan[2].ID}, run.finished, "the scan derives the ids the plan wrote")
}
+21 -42
View File
@@ -38,11 +38,14 @@ func NewEntityParser(dir string) (*EntityParser, error) {
return parser.ParseFile(fset, filename, src, parser.ParseComments)
},
}
pkg, err := loadPackage(cfg, ".")
pkgs, err := packages.Load(cfg, ".")
if err != nil {
return nil, err
}
if len(pkgs) == 0 {
return nil, errors.New("no packages found")
}
pkg := pkgs[0]
return &EntityParser{
syntax: pkg.Syntax,
@@ -284,8 +287,8 @@ func (p *EntityParser) extractClassProperties(node *ast.StructType) []Property {
Description: field.Doc.Text(),
}
if strings.Contains(property.Type, ".") {
// external type found
property.Type = p.handleExternalType(property.Type)
// external struct found
property.Type = p.handleExternalStruct(property.Type)
}
properties = append(properties, property)
}
@@ -349,7 +352,7 @@ func (p *EntityParser) extractReturnType(fn *ast.FuncDecl) (out string) {
for _, result := range fn.Type.Results.List {
tmp := exprToString(result.Type)
if strings.Contains(tmp, ".") && !strings.HasPrefix(tmp, "goja.") {
tmp = p.handleExternalType(tmp) + " | null" // external object interfaces can always return null
tmp = p.handleExternalStruct(tmp) + " | null" // external object interfaces can always return null
}
returns = append(returns, tmp)
}
@@ -374,10 +377,8 @@ func (p *EntityParser) extractReturnType(fn *ast.FuncDecl) (out string) {
}
}
if sliceutil.Contains(returns, "error") {
// external types already record that they can be null
if !strings.HasSuffix(out, "| null") {
out += " | null"
}
// add | null to the return type
out = out + " | null"
return
}
}
@@ -415,18 +416,10 @@ func (p *EntityParser) extractParameters(fn *ast.FuncDecl) []Parameter {
return parameters
}
// handleExternalType returns the TypeScript type for typeName, a type from
// another package such as ssh.ClientConfig. A struct is queued for
// scrapeAndCreate, which describes its fields in an interface of the same name.
// Any other type, such as the net.Conn interface, has no fields to describe,
// so it becomes any. A typeName that does not resolve is queued too, so that
// scrapeAndCreate reports it.
func (p *EntityParser) handleExternalType(typeName string) string {
obj, err := p.lookupType(typeName)
if err == nil && !isStruct(obj.Type()) {
return "any"
}
// typeName is in format ssh.ClientConfig
// it first fetches all fields from the struct and creates a new object
// with that name and returns name of that object as type
func (p *EntityParser) handleExternalStruct(typeName string) string {
baseType := typeName[strings.LastIndex(typeName, ".")+1:]
p.newObjects[typeName] = &Entity{
Name: baseType,
@@ -520,7 +513,6 @@ func (p *EntityParser) extractVarsNConstants() {
// loadImportedPackages loads all imported packages
func (p *EntityParser) loadImportedPackages() error {
cfg := &packages.Config{Mode: packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesInfo}
// get all import statements
// iterate over all imports
for _, file := range p.syntax {
@@ -530,7 +522,7 @@ func (p *EntityParser) loadImportedPackages() error {
// remove the quotes from the path
path = path[1 : len(path)-1]
// load the package
pkg, err := loadPackage(cfg, path)
pkg, err := loadPackage(path)
if err != nil {
return err
}
@@ -551,31 +543,18 @@ func (p *EntityParser) loadImportedPackages() error {
return nil
}
// loadPackage loads the package that pattern names.
//
// packages.Load reports parse, type, and list errors in Package.Errors
// rather than in its returned error, so loadPackage converts them into an
// error. The parser assumes a complete syntax tree and type information.
func loadPackage(cfg *packages.Config, pattern string) (*packages.Package, error) {
pkgs, err := packages.Load(cfg, pattern)
// Load the package containing the type definition
// TODO: we don't support named imports yet
func loadPackage(pkgPath string) (*packages.Package, error) {
cfg := &packages.Config{Mode: packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesInfo}
pkgs, err := packages.Load(cfg, pkgPath)
if err != nil {
return nil, err
}
if len(pkgs) == 0 {
return nil, errors.New("no packages found")
}
pkg := pkgs[0]
if len(pkg.Errors) > 0 {
errs := make([]error, len(pkg.Errors))
for i, e := range pkg.Errors {
errs[i] = e
}
return nil, fmt.Errorf("package %s: %w", pkg.ID, errors.Join(errs...))
}
return pkg, nil
return pkgs[0], nil
}
// exprToString converts an expression to a string
@@ -1,33 +0,0 @@
package tsgen_test
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/pkg/js/devtools/tsgen"
)
func TestParseExternalTypes(t *testing.T) {
t.Parallel()
ep, err := tsgen.NewEntityParser(filepath.Join("testdata", "external"))
require.NoError(t, err)
require.NoError(t, ep.Parse())
entities := make(map[string]tsgen.Entity)
for _, e := range ep.GetEntities() {
entities[e.Name] = e
}
// net.Conn is an interface, so it has no fields to describe.
require.Equal(t, "any | null", entities["Dial"].Function.Returns)
require.Equal(t, []tsgen.Property{{Name: "Conn", Type: "any", Description: ""}}, entities["Session"].Object.Properties)
require.NotContains(t, entities, "Conn")
// url.URL is a struct, so its fields become an interface.
require.Equal(t, "URL | null", entities["Parse"].Function.Returns)
require.Equal(t, "interface", entities["URL"].Type)
require.NotEmpty(t, entities["URL"].Object.Properties)
}
-86
View File
@@ -1,86 +0,0 @@
package tsgen
import (
"go/ast"
"go/parser"
"go/token"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"golang.org/x/tools/go/packages"
)
// writeModule writes src as the only package of a new Go module and
// returns the module directory.
func writeModule(t *testing.T, src string) string {
t.Helper()
dir := t.TempDir()
files := map[string]string{
"go.mod": "module example.com/tsgentest\n\ngo 1.21\n",
"example.go": src,
}
for name, content := range files {
require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600))
}
return dir
}
func TestNewEntityParser(t *testing.T) {
const src = `package example
import "strings"
// Upper returns s in upper case.
func Upper(s string) string { return strings.ToUpper(s) }
`
p, err := NewEntityParser(writeModule(t, src))
require.NoError(t, err)
require.NoError(t, p.Parse())
entities := p.GetEntities()
names := make([]string, 0, len(entities))
for _, e := range entities {
names = append(names, e.Name)
}
require.Contains(t, names, "Upper")
}
func TestNewEntityParserPackageErrors(t *testing.T) {
tests := []struct {
name string
src string
}{
{name: "syntax error", src: "package example\n\nfunc Broken( {\n\treturn 42\n}\n"},
{name: "type error", src: "package example\n\nfunc Broken() int { return undefined }\n"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := NewEntityParser(writeModule(t, tt.src))
require.ErrorContains(t, err, "example.go:")
})
}
}
func TestLoadImportedPackagesMissing(t *testing.T) {
const src = `package example; import "example.com/tsgentest/missing"`
file, err := parser.ParseFile(token.NewFileSet(), "example.go", src, parser.ImportsOnly)
require.NoError(t, err)
p := &EntityParser{
syntax: []*ast.File{file},
imports: map[string]*packages.Package{},
}
var pkgErr packages.Error
require.ErrorAs(t, p.loadImportedPackages(), &pkgErr)
require.Contains(t, pkgErr.Msg, "example.com/tsgentest/missing")
}
+19 -40
View File
@@ -1,15 +1,13 @@
package tsgen
import (
"errors"
"fmt"
"go/types"
"regexp"
"strings"
)
// errUnresolvedType is returned by lookupType for a name it cannot resolve.
var errUnresolvedType = errors.New("unresolved type")
"github.com/projectdiscovery/utils/errkit"
)
// scrape.go scrapes all information of exported type from different package
@@ -17,13 +15,24 @@ func (p *EntityParser) scrapeAndCreate(typeName string) error {
if p.newObjects[typeName] == nil {
return nil
}
typeNameObj, err := p.lookupType(typeName)
if err != nil {
return err
// get package name
pkgName := strings.Split(typeName, ".")[0]
baseTypeName := strings.Split(typeName, ".")[1]
// get package
pkg, ok := p.imports[pkgName]
if !ok {
return errkit.Newf("package %v for type %v not found", pkgName, typeName)
}
// get type
obj := pkg.Types.Scope().Lookup(baseTypeName)
if obj == nil {
return errkit.Newf("type %v not found in package %+v", typeName, pkg)
}
// Ensure the object is a type name
typeNameObj, ok := obj.(*types.TypeName)
if !ok {
return errkit.Newf("%v is not a type name", typeName)
}
baseTypeName := typeNameObj.Name()
// Ensure the type is a named struct type
namedStruct, ok := typeNameObj.Type().Underlying().(*types.Struct)
if !ok {
@@ -63,36 +72,6 @@ func (p *EntityParser) scrapeAndCreate(typeName string) error {
return nil
}
// lookupType resolves typeName, such as net.Conn, in the packages imported by
// the parsed files.
func (p *EntityParser) lookupType(typeName string) (*types.TypeName, error) {
pkgName, baseTypeName, _ := strings.Cut(typeName, ".")
pkg := p.imports[pkgName]
if pkg == nil {
return nil, fmt.Errorf("%w: package %v for type %v not found", errUnresolvedType, pkgName, typeName)
}
obj := pkg.Types.Scope().Lookup(baseTypeName)
if obj == nil {
return nil, fmt.Errorf("%w: type %v not found in package %+v", errUnresolvedType, typeName, pkg)
}
typeNameObj, ok := obj.(*types.TypeName)
if !ok {
return nil, fmt.Errorf("%w: %v is not a type name", errUnresolvedType, typeName)
}
return typeNameObj, nil
}
// isStruct reports whether t is a struct type.
func isStruct(t types.Type) bool {
_, ok := t.Underlying().(*types.Struct)
return ok
}
type ExtObject struct {
builtIn map[string]string
nested map[string]map[string]*ExtObject // Changed to map of field names to ExtObject
-32
View File
@@ -1,32 +0,0 @@
// Package external is a tsgen fixture that references types from other
// packages, both structs and non-structs.
package external
import (
"context"
"net"
"net/url"
)
// Dial returns an external interface type.
func Dial(network, address string) (net.Conn, error) {
return net.Dial(network, address)
}
// Parse returns an external struct type.
func Parse(raw string) (*url.URL, error) {
return url.Parse(raw)
}
// Session holds an external interface type in an exported field.
type Session struct {
Conn net.Conn
}
type dialer struct{}
// DialContext is exported, but its receiver is not.
func (dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, network, address)
}
@@ -1 +0,0 @@
shared adapter package; not a public JavaScript module
+2
View File
@@ -8,6 +8,8 @@ import (
"fmt"
_ "github.com/microsoft/go-mssqldb"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
)
+16 -87
View File
@@ -9,58 +9,11 @@ import (
"net/url"
"strings"
gomssql "github.com/microsoft/go-mssqldb"
_ "github.com/microsoft/go-mssqldb"
"github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
)
// mssqlCustomDialer routes connection dialing through Nuclei's fastdialer.
// Implementing HostDialer ensures the driver delegates hostname resolution to the dialer
// rather than using the local OS net.LookupIP, keeping DNS resolution, scan proxies,
// and dial-time network policies strictly enforced.
type mssqlCustomDialer struct {
executionId string
host string
}
var _ gomssql.HostDialer = (*mssqlCustomDialer)(nil)
func (d *mssqlCustomDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
dialers := protocolstate.GetDialersWithId(d.executionId)
if dialers == nil {
return nil, fmt.Errorf("dialers not initialized for %s", d.executionId)
}
targetHost, _, err := net.SplitHostPort(addr)
if err != nil {
targetHost = addr
}
if !protocolstate.IsHostAllowed(d.executionId, targetHost) {
return nil, protocolstate.ErrHostDenied.Msgf(targetHost)
}
return dialers.Fastdialer.Dial(ctx, network, addr)
}
func (d *mssqlCustomDialer) HostName() string {
return d.host
}
func openDB(executionId, target, username, password, dbName string) (*sql.DB, error) {
host, _, err := net.SplitHostPort(target)
if err != nil {
host = target
}
connString := mssqlConnString(target, username, password, dbName)
connector, err := gomssql.NewConnector(connString)
if err != nil {
return nil, err
}
connector.Dialer = &mssqlCustomDialer{
executionId: executionId,
host: host,
}
return sql.OpenDB(connector), nil
}
type (
// Client is a client for MS SQL database.
// Internally client uses microsoft/go-mssqldb driver.
@@ -114,7 +67,9 @@ func connect(ctx context.Context, executionId string, host string, port int, use
target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
db, err := openDB(executionId, target, username, password, dbName)
connString := mssqlConnString(target, username, password, dbName)
db, err := sql.Open("sqlserver", connString)
if err != nil {
return false, err
}
@@ -124,7 +79,16 @@ func connect(ctx context.Context, executionId string, host string, port int, use
_, err = db.ExecContext(ctx, "select 1")
if err != nil {
if isNetworkError(err) {
switch {
case strings.Contains(err.Error(), "connect: connection refused"):
fallthrough
case strings.Contains(err.Error(), "no pg_hba.conf entry for host"):
fallthrough
case strings.Contains(err.Error(), "network unreachable"):
fallthrough
case strings.Contains(err.Error(), "reset"):
fallthrough
case strings.Contains(err.Error(), "i/o timeout"):
return false, err
}
return false, nil
@@ -132,42 +96,6 @@ func connect(ctx context.Context, executionId string, host string, port int, use
return true, nil
}
// isNetworkError reports whether an error indicates a network, dialer,
// timeout, or policy failure as opposed to a server-side authentication rejection.
func isNetworkError(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return true
}
var netErr net.Error
if errors.As(err, &netErr) {
return true
}
errStr := strings.ToLower(err.Error())
for _, marker := range []string{
"connection refused",
"actively refused",
"port closed",
"unable to open tcp connection",
"network unreachable",
"connection reset",
"i/o timeout",
"connection timed out",
"context canceled",
"deadline exceeded",
"dialers not initialized",
"network policy",
"host denied",
} {
if strings.Contains(errStr, marker) {
return true
}
}
return false
}
// IsMssql checks if the given host is running MS SQL database.
// If the host is running MS SQL database, it returns true.
// If the host is not running MS SQL database, it returns false.
@@ -228,8 +156,9 @@ func (c *MSSQLClient) ExecuteQuery(ctx context.Context, host string, port int, u
}
target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
connString := mssqlConnString(target, username, password, dbName)
db, err := openDB(executionId, target, username, password, dbName)
db, err := sql.Open("sqlserver", connString)
if err != nil {
return nil, err
}
-120
View File
@@ -1,18 +1,9 @@
package mssql
import (
"context"
"errors"
"net"
"strconv"
"testing"
"time"
gomssql "github.com/microsoft/go-mssqldb"
"github.com/microsoft/go-mssqldb/msdsn"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
@@ -42,114 +33,3 @@ func TestConnectionStringKeepsPlainDatabaseName(t *testing.T) {
require.Equal(t, "master", cfg.Database)
}
func TestOpenDBUsesCustomFastdialer(t *testing.T) {
execID := "mssql-opendb-test"
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
t.Cleanup(func() { protocolstate.Close(execID) })
db, err := openDB(execID, "127.0.0.1:1433", "user", "pass", "master")
require.NoError(t, err)
require.NotNil(t, db)
_ = db.Close()
}
func TestMssqlDialerEnforcesNetworkPolicy(t *testing.T) {
execID := "mssql-policy-test"
require.NoError(t, protocolstate.Init(&types.Options{
ExecutionId: execID,
ExcludeTargets: goflags.StringSlice{"denied.host"},
}))
t.Cleanup(func() { protocolstate.Close(execID) })
dialer := &mssqlCustomDialer{
executionId: execID,
host: "denied.host",
}
require.Equal(t, "denied.host", dialer.HostName())
var _ gomssql.HostDialer = dialer
_, err := dialer.DialContext(context.Background(), "tcp", "denied.host:1433")
require.Error(t, err)
require.Contains(t, err.Error(), "network policy")
uninitDialer := &mssqlCustomDialer{
executionId: "nonexistent-exec-id",
host: "example.com",
}
_, err = uninitDialer.DialContext(context.Background(), "tcp", "example.com:1433")
require.Error(t, err)
require.Contains(t, err.Error(), "dialers not initialized")
}
func TestConnectFailsFastOnClosedPort(t *testing.T) {
execID := "mssql-closed-port-test"
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
t.Cleanup(func() { protocolstate.Close(execID) })
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
_, portStr, err := net.SplitHostPort(ln.Addr().String())
require.NoError(t, err)
port, err := strconv.Atoi(portStr)
require.NoError(t, err)
_ = ln.Close()
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
ok, err := connect(ctx, execID, "127.0.0.1", port, "user", "pass", "master")
require.Error(t, err)
require.False(t, ok)
}
func TestConnectWithCanceledContext(t *testing.T) {
execID := "mssql-canceled-ctx-test"
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
t.Cleanup(func() { protocolstate.Close(execID) })
ctx, cancel := context.WithCancel(context.Background())
cancel() // Cancel immediately
ok, err := connect(ctx, execID, "127.0.0.1", 1433, "user", "pass", "master")
require.Error(t, err)
require.False(t, ok)
}
func TestIsNetworkError(t *testing.T) {
require.True(t, isNetworkError(errors.New("connect: connection refused")))
require.True(t, isNetworkError(errors.New("No connection could be made because the target machine actively refused it.")))
require.True(t, isNetworkError(errors.New("unable to open tcp connection with host '127.0.0.1:1433': cause=\"port closed or filtered\"")))
require.True(t, isNetworkError(errors.New("network unreachable")))
require.True(t, isNetworkError(errors.New("connection reset by peer")))
require.True(t, isNetworkError(errors.New("i/o timeout")))
require.True(t, isNetworkError(context.Canceled))
require.True(t, isNetworkError(context.DeadlineExceeded))
require.True(t, isNetworkError(protocolstate.ErrHostDenied.Msgf("127.0.0.1")))
// Server-side responses / authentication failures are not network errors
require.False(t, isNetworkError(nil))
require.False(t, isNetworkError(errors.New("mssql: Login failed for user 'sa'")))
require.False(t, isNetworkError(errors.New("mssql: Cannot open database \"secret\" requested by the login")))
require.False(t, isNetworkError(errors.New("mssql: Password has expired")))
require.False(t, isNetworkError(errors.New("mssql: Permission denied to database")))
require.False(t, isNetworkError(errors.New("mssql: Password must be reset")))
require.False(t, isNetworkError(errors.New("mssql: The query timed out")))
}
func TestExecuteQueryUsesFastdialer(t *testing.T) {
execID := "mssql-execute-query-test"
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: execID}))
t.Cleanup(func() { protocolstate.Close(execID) })
client := &MSSQLClient{}
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck // SA1029: executionId string key matches goja runtime injection
// Denied host should fail policy check immediately
_, err := client.ExecuteQuery(ctx, "denied.host", 1433, "user", "pass", "master", "SELECT 1")
require.Error(t, err)
// Closed port should fail
_, err = client.ExecuteQuery(ctx, "127.0.0.1", 65530, "user", "pass", "master", "SELECT 1")
require.Error(t, err)
}
@@ -1 +0,0 @@
shared adapter package; not a public JavaScript module
@@ -6,6 +6,7 @@ import (
"net/http"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
@@ -20,6 +21,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan/techgraph"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
@@ -62,6 +64,14 @@ type Service struct {
techTemplates []*templates.Template
ServiceOpts Options
hasResults *atomic.Bool
// tech-graph based selection (automatic scan v2). When selector is nil the
// service falls back to the legacy tag-overlap selection.
selector *techgraph.Selector
tier techgraph.Tier
universe map[string]*templates.Template
universeErr error
universeOnce sync.Once
}
// New takes options and returns a new automatic scan service
@@ -103,7 +113,7 @@ func New(opts Options) (*Service, error) {
if err != nil {
return nil, errors.Wrap(err, "could not get http client")
}
return &Service{
service := &Service{
opts: opts.ExecuterOpts,
store: opts.Store,
engine: opts.Engine,
@@ -115,7 +125,22 @@ func New(opts Options) (*Service, error) {
techTemplates: techDetectTemplates,
ServiceOpts: opts,
hasResults: &atomic.Bool{},
}, nil
}
tier, err := techgraph.ParseTier(opts.ExecuterOpts.Options.AutomaticScanCoverage)
if err != nil {
return nil, err
}
service.tier = tier
// load the embedded tech-graph for v2 selection; on failure we silently fall
// back to legacy tag-overlap so behaviour never regresses.
if graph, err := techgraph.Embedded(); err == nil {
service.selector = techgraph.NewSelector(graph)
} else {
gologger.Warning().Msgf("automatic scan: tech-graph unavailable, using legacy selection: %s", err)
}
return service, nil
}
// Close closes the service
@@ -179,7 +204,9 @@ func (s *Service) executeAutomaticScanOnTarget(ctx context.Context, input *conte
finalTags = append(finalTags, s.opts.Options.Tags...)
finalTags = sliceutil.Dedupe(finalTags)
if len(finalTags) == 0 {
// Lean has no baseline, so an empty signal set has nothing to run. Balanced
// and thorough still run the tech-agnostic baseline.
if len(finalTags) == 0 && (s.selector == nil || s.tier == techgraph.TierLean) {
gologger.Warning().Msgf("Skipping automatic scan since no tags were found on %v\n", input.Input)
return
}
@@ -187,9 +214,19 @@ func (s *Service) executeAutomaticScanOnTarget(ctx context.Context, input *conte
gologger.Print().Msgf("Final tags identified for %v: %+v\n", input.Input, finalTags)
}
finalTemplates, err := LoadTemplatesWithTags(s.ServiceOpts, s.templateDirs, finalTags, false)
if err != nil {
gologger.Error().Msgf("%v Error loading templates: %s\n", input.Input, err)
var finalTemplates []*templates.Template
if s.selector != nil {
finalTemplates = s.selectWithGraph(input, finalTags)
} else {
var err error
finalTemplates, err = LoadTemplatesWithTags(s.ServiceOpts, s.templateDirs, finalTags, false)
if err != nil {
gologger.Error().Msgf("%v Error loading templates: %s\n", input.Input, err)
return
}
}
if len(finalTemplates) == 0 {
gologger.Warning().Msgf("Skipping automatic scan since no templates were selected on %v\n", input.Input)
return
}
gologger.Info().Msgf("Executing %d templates on %v", len(finalTemplates), input.Input)
@@ -205,12 +242,137 @@ func (s *Service) executeAutomaticScanOnTarget(ctx context.Context, input *conte
s.hasResults.Store(tmp.Load())
}
// ensureUniverse lazily loads every selectable template once and indexes it by
// id. This is the pool from which graph-based selection draws (precise + fuzzy),
// avoiding a per-target loader pass.
func (s *Service) ensureUniverse() (map[string]*templates.Template, error) {
s.universeOnce.Do(func() {
all, err := s.store.LoadTemplates(s.templateDirs)
if err != nil {
s.universeErr = err
return
}
s.universe = make(map[string]*templates.Template, len(all))
for _, t := range all {
if t != nil {
s.universe[t.ID] = t
}
}
})
return s.universe, s.universeErr
}
// selectWithGraph resolves detected signals to a precise fire-set via the
// tech-graph, augmented (except at the lean tier) by legacy tag-overlap for any
// signals not present in the graph, so recall never falls below legacy behaviour.
func (s *Service) selectWithGraph(input *contextargs.MetaInput, finalTags []string) []*templates.Template {
universe, err := s.ensureUniverse()
if err != nil || len(universe) == 0 {
if len(finalTags) == 0 {
return nil
}
if err != nil {
gologger.Warning().Msgf("automatic scan: template load failed, using tag selection: %s", err)
}
tpls, loadErr := LoadTemplatesWithTags(s.ServiceOpts, s.templateDirs, finalTags, false)
if loadErr != nil {
gologger.Error().Msgf("%v Error loading templates: %s\n", input.Input, loadErr)
return nil
}
return tpls
}
techIDs, unresolved := s.selector.Resolve(finalTags)
selected := map[string]*templates.Template{}
graph := 0
for _, id := range s.selector.FireSet(s.tier, techIDs) {
if t, ok := universe[id]; ok {
selected[id] = t
graph++
}
}
added := 0
if s.tier != techgraph.TierLean && len(techIDs) > 0 {
candidates := make([]techgraph.Candidate, 0, len(universe))
for id, t := range universe {
if _, ok := selected[id]; ok {
continue
}
candidates = append(candidates, techgraph.Candidate{
ID: id,
Tags: t.Info.Tags.ToSlice(),
Category: categoryFromPath(t.Path),
})
}
for _, id := range s.selector.AbsentDependents(techIDs, candidates) {
if t, ok := universe[id]; ok {
selected[id] = t
added++
}
}
}
fuzzy := 0
if s.tier != techgraph.TierLean && len(unresolved) > 0 {
want := map[string]struct{}{}
for _, tag := range unresolved {
want[strings.ToLower(tag)] = struct{}{}
}
for id, t := range universe {
if _, ok := selected[id]; ok {
continue
}
for _, tag := range t.Info.Tags.ToSlice() {
if _, ok := want[strings.ToLower(tag)]; ok {
selected[id] = t
fuzzy++
break
}
}
}
}
out := make([]*templates.Template, 0, len(selected))
for _, t := range selected {
out = append(out, t)
}
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
if s.opts.Options.VerboseVerbose {
gologger.Print().Msgf("%v selection [tier=%s]: %d techs, %d graph + %d new + %d fuzzy = %d templates\n",
input.Input, s.tier, len(techIDs), graph, added, fuzzy, len(out))
}
if !s.opts.Options.DisableClustering {
out, _, _ = templates.ClusterTemplates(out, s.opts)
}
return out
}
type sharedProgress struct {
progress.Progress
}
func (p *sharedProgress) Init(hostCount int64, rulesCount int, requestCount int64) {}
// categoryFromPath returns the nuclei category directory for a template path
// (http/cves/2024/x.yaml -> cves). An empty result means the category is unknown
// and the template is not eligible for the post-graph supplement.
func categoryFromPath(path string) string {
parts := strings.Split(filepath.ToSlash(path), "/")
for i, part := range parts {
switch part {
case "http", "network", "dns", "ssl", "file", "headless", "javascript", "code", "cloud", "dast":
if i+1 < len(parts) {
return parts[i+1]
}
}
}
return ""
}
// getTagsUsingWappalyzer returns tags using wappalyzer by fingerprinting target
// and utilizing the mapping data
func (s *Service) getTagsUsingWappalyzer(input *contextargs.MetaInput) []string {
@@ -13,3 +13,9 @@ func TestNormalizeAppName(t *testing.T) {
appName = normalizeAppName("JBoss:2.3.5")
require.Equal(t, "jboss", appName, "could not get normalized name")
}
func TestCategoryFromPath(t *testing.T) {
require.Equal(t, "cves", categoryFromPath("/home/templates/http/cves/2024/CVE-2024-1.yaml"))
require.Equal(t, "fuzzing", categoryFromPath("http/fuzzing/xss.yaml"))
require.Equal(t, "", categoryFromPath("README.md"))
}
@@ -0,0 +1,73 @@
package techgraph
import "strings"
// excludedCategories never participate in automatic scan (opt-in/specialized).
var excludedCategories = map[string]struct{}{
"fuzzing": {},
"osint": {},
"token-spray": {},
"malware": {},
"cloud": {},
}
// baselineCategories are the only categories whose product-less templates may be
// considered for the tech-agnostic baseline. Product-specific categories (cves,
// vulnerabilities, exposed-panels, default-logins, cnvd) are intentionally absent:
// an unmappable template there must go to unmapped (fallback), never baseline.
var baselineCategories = map[string]struct{}{
"exposures": {},
"misconfiguration": {},
"miscellaneous": {},
"iot": {},
"takeovers": {}, // subdomain takeover signatures are run host-agnostically
}
// genericBaselineTags positively identify a tech-agnostic check.
var genericBaselineTags = map[string]struct{}{
"cors": {}, "crlf": {}, "open-redirect": {}, "ds_store": {}, "dsstore": {},
"phpinfo": {}, "directory-listing": {}, "listing": {}, "debug": {},
"robots": {}, "sitemap": {}, "git": {}, "svn": {}, "dotenv": {},
"host-header-injection": {}, "http-trace": {}, "options-method": {},
"well-known": {}, "trace": {},
}
// genericBaselineIDSubstrings positively identify generic checks by id.
var genericBaselineIDSubstrings = []string{
"git-config", "gitignore", "git-credentials", "svn-", ".env", "dotenv",
"phpinfo", "directory-listing", "dir-listing", "ds-store", "ds_store",
"robots-txt", "sitemap-", "cors-", "crlf-", "open-redirect", "trace-method",
"options-method", "well-known", "backup-files", "sql-dump",
}
// classifyBaseline decides whether an otherwise-unmapped template is a genuinely
// generic, tech-agnostic baseline check. It is a positive allowlist: anything not
// recognised as generic is left for the caller to mark unmapped.
func classifyBaseline(info templateInfo) (Tier, bool) {
// tech-agnostic "generic" buckets (e.g. vulnerabilities/generic/* holds
// cors/sqli/xss probes that apply to any target) qualify regardless of
// category. The logs/files/backups subdir heuristic was removed: it leaked
// product-specific exposures (bitrix, magento, oracle-ebs, ...) into baseline.
if info.Subdir == "generic" {
return TierThorough, true
}
if _, ok := baselineCategories[info.Category]; !ok {
return "", false
}
// takeovers are wholesale host-agnostic.
if info.Category == "takeovers" {
return TierBalanced, true
}
for _, tag := range info.Tags {
if _, ok := genericBaselineTags[normToken(tag)]; ok {
return TierBalanced, true
}
}
idl := strings.ToLower(info.ID)
for _, sub := range genericBaselineIDSubstrings {
if strings.Contains(idl, sub) {
return TierBalanced, true
}
}
return "", false
}
@@ -0,0 +1,353 @@
package techgraph
import (
"crypto/sha256"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"time"
)
// Build walks the templates root and produces the tech-graph artifact.
func Build(root string) (*Graph, error) {
root = filepath.Clean(root)
g := &Graph{
Version: ArtifactVersion,
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
Techs: map[string]*Tech{},
Synonyms: map[string]string{},
}
infos, hash, parseErr, err := collect(root)
if err != nil {
return nil, err
}
g.Stats.ParseError = parseErr
g.SourceHash = fmt.Sprintf("sha256:%x", hash)
var deferred []templateInfo
// Pass 1: build tech nodes from authoritative signals (detection anchors,
// cpe/product, platform). Everything else is deferred so it can be resolved
// against the fully-built node set in pass 2.
for _, info := range infos {
g.Stats.Total++
// Excluded categories never enter detection or the fire-set. Gating
// this before detection classification keeps opt-in categories such as
// cloud out of the automatic-scan detector list.
if _, ok := excludedCategories[info.Category]; ok {
g.Stats.Excluded++
continue
}
if isDetection(info) && isDetectionSeverity(info) {
id := techID(info.Vendor, info.Product)
if id != "" {
g.anchorTech(id, info)
}
g.Detection = append(g.Detection, DetectionEntry{ID: info.ID, Tech: id})
g.Stats.Detection++
continue
}
if p, ok := detectPlatform(info); ok {
g.addDependent(p, info, "platform")
g.Stats.Dependents++
g.Stats.DependentsPlat++
continue
}
if id := techID(info.Vendor, info.Product); id != "" {
g.addDependent(id, info, "cpe")
g.Stats.Dependents++
g.Stats.DependentsCPE++
continue
}
deferred = append(deferred, info)
}
// Pass 2: resolve deferred templates to a known tech BEFORE considering
// baseline. Order: directory grouping -> tag -> id token. Only genuinely
// generic leftovers become baseline; the rest are unmapped (runtime fallback).
aliasIdx := g.buildAliasIndex()
for _, info := range deferred {
if id, ok := g.resolveToken(info.Subdir, aliasIdx); ok {
g.addDependent(id, info, "dir")
g.Stats.Dependents++
g.Stats.DependentsDir++
continue
}
if id, ok := reconcileByTag(g, info, aliasIdx); ok {
g.addDependent(id, info, "tag")
g.Stats.Dependents++
g.Stats.DependentsTag++
continue
}
if id, ok := reconcileByID(g, info, aliasIdx); ok {
g.addDependent(id, info, "id")
g.Stats.Dependents++
g.Stats.DependentsID++
continue
}
if tier, ok := classifyBaseline(info); ok {
g.Baseline = append(g.Baseline, BaselineEntry{
ID: info.ID, Category: info.Category, Severity: info.Severity, Tier: tier,
})
g.Stats.Baseline++
continue
}
g.Stats.Unmapped++
}
// ship validated seed synonyms in the artifact so the runtime selector (which
// only reads the artifact) can resolve acronyms like aem -> adobe:experience_manager.
for alias, id := range seedSynonyms {
if _, ok := g.Techs[id]; ok {
g.Synonyms[alias] = id
}
}
g.Stats.Techs = len(g.Techs)
g.sortStable()
return g, nil
}
// collect walks the tree once and returns parsed infos plus a content hash.
func collect(root string) ([]templateInfo, []byte, int, error) {
var infos []templateInfo
var parseErr int
hasher := sha256.New()
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return nil
}
if d.IsDir() {
if skipDir(d.Name()) {
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(path, ".yaml") && !strings.HasSuffix(path, ".yml") {
return nil
}
info, ok := extractTemplate(path, root)
if !ok {
parseErr++
return nil
}
if _, err := fmt.Fprintf(hasher, "%s|%s|%s|%s\n", info.ID, info.Vendor, info.Product, info.CPE); err != nil {
return err
}
infos = append(infos, info)
return nil
})
if err != nil {
return nil, nil, 0, err
}
sort.Slice(infos, func(i, j int) bool { return infos[i].ID < infos[j].ID })
return infos, hasher.Sum(nil), parseErr, nil
}
// anchorTech creates/updates a tech node's metadata without adding a fireable
// template (used by detector templates that define a tech).
func (g *Graph) anchorTech(id string, info templateInfo) *Tech {
tech, ok := g.Techs[id]
if !ok {
tech = &Tech{ID: id, Vendor: normToken(info.Vendor), Product: normToken(info.Product), CPE: info.CPE}
g.Techs[id] = tech
}
if tech.CPE == "" && info.CPE != "" {
tech.CPE = info.CPE
}
mergeAliases(tech, aliasesFor(info.Vendor, info.Product, info.Tags))
for _, a := range tech.Aliases {
if existing, ok := g.Synonyms[a]; !ok || existing == id {
g.Synonyms[a] = id
}
}
return tech
}
// addDependent registers a fireable template under its canonical tech.
//
// Only "cpe" attachments contribute aliases (the template's own vendor/product
// genuinely define the tech). Platform/dir/tag/id attachments must NOT pollute
// the node's aliases with the sub-product's tokens (e.g. a wp-plugin's name must
// not become a wordpress alias), which would corrupt reconciliation.
func (g *Graph) addDependent(id string, info templateInfo, source string) {
var tech *Tech
if source == "cpe" {
tech = g.anchorTech(id, info)
} else {
tech = g.ensureTech(id, info)
}
ref := TemplateRef{ID: info.ID, Category: info.Category, Severity: info.Severity, Source: source}
if source == "platform" || source == "dir" {
ref.Product = normToken(info.Product)
}
tech.Templates = append(tech.Templates, ref)
}
// ensureTech returns an existing node, or creates a bare one without merging the
// template's aliases (used for non-cpe attachments).
func (g *Graph) ensureTech(id string, info templateInfo) *Tech {
if tech, ok := g.Techs[id]; ok {
return tech
}
vendor, product := "", ""
if v, p, ok := splitTechID(id); ok {
vendor, product = v, p
}
tech := &Tech{ID: id, Vendor: vendor, Product: product}
g.Techs[id] = tech
return tech
}
// splitTechID reverses techID for bare node creation.
func splitTechID(id string) (vendor, product string, ok bool) {
if i := strings.IndexByte(id, ':'); i > 0 {
return id[:i], id[i+1:], true
}
return "", id, true
}
// buildAliasIndex maps a single alias/product token to a unique tech id. Tokens
// that resolve to more than one tech, are too short, or are generic are dropped
// so tag reconciliation stays conservative.
func (g *Graph) buildAliasIndex() map[string]string {
counts := map[string]map[string]struct{}{}
add := func(tok, id string) {
tok = normToken(tok)
if len(tok) < 5 || genericTokens[tok] {
return
}
if counts[tok] == nil {
counts[tok] = map[string]struct{}{}
}
counts[tok][id] = struct{}{}
}
for id, tech := range g.Techs {
add(id, id)
add(tech.Product, id)
for _, a := range tech.Aliases {
add(a, id)
}
}
idx := map[string]string{}
for tok, ids := range counts {
if len(ids) == 1 {
for id := range ids {
idx[tok] = id
}
}
}
return idx
}
// resolveToken maps a single token to a tech id via the seed synonym table
// first (validated against existing nodes), then the alias index.
func (g *Graph) resolveToken(tok string, idx map[string]string) (string, bool) {
tok = normToken(tok)
if tok == "" || genericTokens[tok] {
return "", false
}
if id, ok := seedSynonyms[tok]; ok {
if _, exists := g.Techs[id]; exists {
return id, true
}
}
if id, ok := idx[tok]; ok {
return id, true
}
return "", false
}
// reconcileByTag attaches a product-less template to a known tech when its tags
// resolve to exactly one unique tech node.
func reconcileByTag(g *Graph, info templateInfo, idx map[string]string) (string, bool) {
matches := map[string]struct{}{}
for _, tag := range info.Tags {
if id, ok := g.resolveToken(tag, idx); ok {
matches[id] = struct{}{}
}
}
if len(matches) == 1 {
for id := range matches {
return id, true
}
}
return "", false
}
// reconcileByID attaches a template when its id tokens resolve to exactly one
// unique tech node (e.g. "aem-jcr-exposure" -> adobe:experience_manager).
//
// This is the riskiest signal (id tokens are common English words), so it is
// guarded: a token must be >=6 chars AND also appear in the template name, which
// requires the product to be the template's actual subject rather than an
// incidental id suffix like "-dashboard"/"-gateway".
func reconcileByID(g *Graph, info templateInfo, idx map[string]string) (string, bool) {
name := strings.ToLower(info.Name)
matches := map[string]struct{}{}
for _, tok := range strings.Split(strings.ToLower(info.ID), "-") {
if len(tok) < 6 || !strings.Contains(name, tok) {
continue
}
if id, ok := g.resolveToken(tok, idx); ok {
matches[id] = struct{}{}
}
}
if len(matches) == 1 {
for id := range matches {
return id, true
}
}
return "", false
}
func mergeAliases(tech *Tech, aliases []string) {
have := map[string]struct{}{}
for _, a := range tech.Aliases {
have[a] = struct{}{}
}
for _, a := range aliases {
if a == "" {
continue
}
if _, ok := have[a]; ok {
continue
}
have[a] = struct{}{}
tech.Aliases = append(tech.Aliases, a)
}
}
func (g *Graph) sortStable() {
for _, tech := range g.Techs {
sort.Strings(tech.Aliases)
sort.Slice(tech.Templates, func(i, j int) bool {
return tech.Templates[i].ID < tech.Templates[j].ID
})
}
sort.Slice(g.Baseline, func(i, j int) bool { return g.Baseline[i].ID < g.Baseline[j].ID })
sort.Slice(g.Detection, func(i, j int) bool { return g.Detection[i].ID < g.Detection[j].ID })
}
// WriteFile serialises the graph as indented JSON.
func (g *Graph) WriteFile(path string) error {
data, err := json.MarshalIndent(g, "", " ")
if err != nil {
return err
}
return os.WriteFile(path, append(data, '\n'), 0o644)
}
func skipDir(name string) bool {
switch name {
case ".git", ".github", "helpers", "profiles", "workflows":
return true
}
return false
}
@@ -0,0 +1,121 @@
package techgraph
import "strings"
// seedSynonyms maps common acronyms/aliases to a canonical tech id. This is the
// curated seed of the synonym table that the offline AI step later expands.
// Targets are validated against existing nodes at resolve time, so a stale entry
// simply has no effect.
var seedSynonyms = map[string]string{
"aem": "adobe:experience_manager",
"coldfusion": "adobe:coldfusion",
"iis": "microsoft:iis",
"k8s": "kubernetes:kubernetes",
"springboot": "vmware:spring_boot",
"spring-boot": "vmware:spring_boot",
"wls": "oracle:weblogic_server",
"weblogic": "oracle:weblogic_server",
"adfs": "microsoft:active_directory_federation_services",
// platform roots: ambiguous in the alias index (collide with sub-product
// nodes like keydatas:wordpress) so pin them to the canonical platform node.
"wordpress": "wordpress",
"drupal": "drupal",
"joomla": "joomla",
"magento": "magento",
}
// genericTokens are tags/aliases that are not products and must never be used as
// a tag/id/dir-reconciliation key (they would incorrectly attach templates to random
// techs). This list intentionally includes common English words that also happen
// to be CPE product names (dashboard, gateway, console, ...), platform/OS tokens
// (linux, windows, android, ...), and category labels (generic, audit, ...),
// since those were the dominant source of false attachments.
var genericTokens = map[string]bool{
// vuln classes / scan vocabulary
"cve": true, "cves": true, "vuln": true, "vulnerability": true, "tech": true,
"detect": true, "detection": true, "panel": true, "panels": true, "login": true, "cms": true,
"exposure": true, "exposures": true, "misconfig": true, "misconfiguration": true,
"rce": true, "lfi": true, "xss": true, "sqli": true, "ssrf": true, "ssti": true,
"redirect": true, "disclosure": true, "default": true, "auth": true, "bypass": true,
"injection": true, "traversal": true, "config": true, "files": true, "file": true,
"backup": true, "takeover": true, "oast": true, "intrusive": true, "unauth": true,
"router": true, "iot": true, "network": true, "http": true, "json": true, "api": true,
"server": true, "service": true, "framework": true, "plugin": true, "theme": true,
"webserver": true, "database": true, "cloud": true, "devops": true, "tools": true,
"edb": true, "kev": true, "packetstorm": true, "seclists": true, "osint": true,
// category labels
"generic": true, "audit": true, "miscellaneous": true, "misc": true, "enumeration": true,
"vulnerabilities": true, "exposed": true, "keys": true, "secret": true, "secrets": true,
// platform / OS / cloud-provider tokens (not products)
"linux": true, "windows": true, "macos": true, "unix": true, "android": true, "ios": true,
"aws": true, "azure": true, "gcp": true, "alibaba": true, "kubernetes": true, "k8s": true,
"docker": true, "mobile": true,
// common English words that collide with product names
"dashboard": true, "gateway": true, "path": true, "console": true, "firewall": true,
"monitor": true, "control": true, "permissions": true, "oauth": true, "analytics": true,
"enterprise": true, "report": true, "reports": true, "help": true, "printer": true,
"core": true, "commerce": true, "anchor": true, "nova": true, "element": true,
"webclient": true, "memcached": true, "manager": true, "management": true, "admin": true,
"client": true, "agent": true, "portal": true, "gateway2": true, "status": true,
"user": true, "users": true, "account": true, "token": true, "tokens": true,
"web": true, "app": true, "application": true, "system": true, "platform": true,
}
// normToken lowercases and trims a vendor/product token. CPE uses '_' for
// spaces; we keep it as-is so the id stays stable and machine-joinable.
func normToken(s string) string {
s = strings.ToLower(strings.TrimSpace(s))
// CPE wildcards / NA markers are not real tokens.
if s == "*" || s == "-" {
return ""
}
// URLs / paths / multi-word strings are not clean product tokens (guards
// against junk nodes like "https://apisix.apache.org").
if strings.ContainsAny(s, "/ ") {
return ""
}
return s
}
// techID builds the canonical technology id from vendor/product. When vendor is
// empty or equal to product we collapse to a single token to avoid duplicate
// nodes like "wordpress:wordpress" vs "wordpress".
func techID(vendor, product string) string {
vendor = normToken(vendor)
product = normToken(product)
switch {
case product == "":
return ""
case vendor == "" || vendor == product:
return product
default:
return vendor + ":" + product
}
}
// aliasesFor returns alias tokens that runtime detection (wappalyzer names,
// tags) might surface for a tech, so they can be reconciled to the canonical id.
// Pillar 1 derives these from product/vendor tokens; AI augments this offline.
func aliasesFor(vendor, product string, tags []string) []string {
seen := map[string]struct{}{}
var out []string
add := func(s string) {
s = normToken(s)
if s == "" {
return
}
if _, ok := seen[s]; ok {
return
}
seen[s] = struct{}{}
out = append(out, s)
}
// Only whole product/vendor tokens are safe aliases. Splitting on separators
// produces ambiguous fragments (manager, server, token, web, ...) that cause
// false tag/id reconciliation, so we deliberately avoid it.
add(product)
if vendor != product {
add(vendor)
}
return out
}
@@ -0,0 +1,59 @@
// Command techgraph-gen builds the offline tech-graph.json artifact from a local
// nuclei-templates checkout. See pkg/protocols/common/automaticscan/AUTOMATIC_SCAN_V2.md.
package main
import (
"flag"
"fmt"
"os"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan/techgraph"
)
func main() {
var (
templates = flag.String("templates", "", "path to nuclei-templates directory (required)")
output = flag.String("output", "tech-graph.json", "output artifact path")
htmlOut = flag.String("html", "", "also write a standalone visualizer HTML with data inlined")
)
flag.Parse()
if *templates == "" {
fmt.Fprintln(os.Stderr, "error: -templates is required")
flag.Usage()
os.Exit(2)
}
graph, err := techgraph.Build(*templates)
if err != nil {
fmt.Fprintf(os.Stderr, "error: build failed: %v\n", err)
os.Exit(1)
}
if err := graph.WriteFile(*output); err != nil {
fmt.Fprintf(os.Stderr, "error: write failed: %v\n", err)
os.Exit(1)
}
if *htmlOut != "" {
html, err := techgraph.StandaloneHTML(graph)
if err != nil {
fmt.Fprintf(os.Stderr, "error: html render failed: %v\n", err)
os.Exit(1)
}
if err := os.WriteFile(*htmlOut, []byte(html), 0o644); err != nil {
fmt.Fprintf(os.Stderr, "error: html write failed: %v\n", err)
os.Exit(1)
}
fmt.Printf("standalone visualizer written to %s\n", *htmlOut)
}
s := graph.Stats
fmt.Printf("tech-graph written to %s\n", *output)
fmt.Printf(" templates: %d (parse-skipped %d)\n", s.Total, s.ParseError)
fmt.Printf(" dependents: %d (cpe %d, platform %d, dir %d, tag %d, id %d) across %d techs\n",
s.Dependents, s.DependentsCPE, s.DependentsPlat, s.DependentsDir, s.DependentsTag, s.DependentsID, s.Techs)
fmt.Printf(" baseline: %d\n", s.Baseline)
fmt.Printf(" detection: %d\n", s.Detection)
fmt.Printf(" excluded: %d\n", s.Excluded)
fmt.Printf(" unmapped: %d\n", s.Unmapped)
fmt.Printf(" source: %s\n", graph.SourceHash)
}
@@ -0,0 +1,39 @@
package techgraph
import (
_ "embed"
"encoding/json"
"strings"
)
// embeddedGraph is the prebuilt tech-graph artifact shipped with nuclei. Run the
// techgraph-gen command to regenerate it against a nuclei-templates checkout.
//
//go:embed tech-graph.json
var embeddedGraph []byte
//go:embed visualizer.html
var visualizerHTML string
// Embedded returns the tech-graph artifact compiled into the binary.
func Embedded() (*Graph, error) {
g := &Graph{}
if err := json.Unmarshal(embeddedGraph, g); err != nil {
return nil, err
}
return g, nil
}
// EmbeddedRaw returns the raw embedded artifact bytes.
func EmbeddedRaw() []byte { return embeddedGraph }
// StandaloneHTML returns the visualizer with the given graph inlined, so it can
// be opened directly in a browser without a server or sidecar file.
func StandaloneHTML(g *Graph) (string, error) {
data, err := json.Marshal(g)
if err != nil {
return "", err
}
inject := "<script>window.__TECHGRAPH__=" + string(data) + ";</script>\n</head>"
return strings.Replace(visualizerHTML, "</head>", inject, 1), nil
}
@@ -0,0 +1,165 @@
package techgraph
import (
"os"
"path/filepath"
"strings"
"gopkg.in/yaml.v3"
)
// templateInfo is the subset of a nuclei template we need for graph building.
// We deliberately avoid the full template parser: only the info block is read,
// which keeps the generator fast and dependency-light over ~13k files.
type templateInfo struct {
ID string
Name string
Severity string
Category string // derived from directory (cves, exposed-panels, ...)
Subdir string // directory under the category (e.g. "aem" in misconfiguration/aem/)
Tags []string
Vendor string
Product string
CPE string
}
// rawTemplate mirrors the YAML structure for lenient decoding.
type rawTemplate struct {
ID string `yaml:"id"`
Info struct {
Name string `yaml:"name"`
Severity string `yaml:"severity"`
Tags any `yaml:"tags"`
Metadata map[string]any `yaml:"metadata"`
Classification struct {
CPE string `yaml:"cpe"`
} `yaml:"classification"`
} `yaml:"info"`
}
// extractTemplate reads a single template file and returns its info subset.
// ok is false when the file is not a parseable template (e.g. workflow, helper).
func extractTemplate(path, root string) (templateInfo, bool) {
data, err := os.ReadFile(path)
if err != nil {
return templateInfo{}, false
}
var raw rawTemplate
if err := yaml.Unmarshal(data, &raw); err != nil {
return templateInfo{}, false
}
if raw.ID == "" {
return templateInfo{}, false
}
category, subdir := derivePath(path, root)
info := templateInfo{
ID: raw.ID,
Name: raw.Info.Name,
Severity: strings.ToLower(strings.TrimSpace(raw.Info.Severity)),
Category: category,
Subdir: subdir,
Tags: coerceStringSlice(raw.Info.Tags),
CPE: strings.TrimSpace(raw.Info.Classification.CPE),
}
if raw.Info.Metadata != nil {
info.Vendor = firstNonEmpty(coerceStringSlice(raw.Info.Metadata["vendor"]))
info.Product = firstNonEmpty(coerceStringSlice(raw.Info.Metadata["product"]))
}
// Prefer CPE for vendor/product when metadata is missing.
if cv, cp, ok := parseCPE(info.CPE); ok {
if info.Vendor == "" {
info.Vendor = cv
}
if info.Product == "" {
info.Product = cp
}
}
return info, true
}
// derivePath returns the nuclei category and the immediate subdirectory under
// it, e.g. http/misconfiguration/aem/x.yaml -> ("misconfiguration", "aem").
// The subdir is often a product grouping and is used as a tech signal.
func derivePath(path, root string) (category, subdir string) {
rel, err := filepath.Rel(root, path)
if err != nil {
rel = path
}
parts := strings.Split(filepath.ToSlash(rel), "/")
switch {
case len(parts) >= 4:
// protocol/category/subdir/.../file
return parts[1], parts[2]
case len(parts) == 3:
return parts[1], ""
case len(parts) == 2:
return parts[0], ""
default:
return "", ""
}
}
// parseCPE extracts vendor and product from CPE 2.3 (cpe:2.3:a:vendor:product:..)
// or CPE 2.2 URI (cpe:/a:vendor:product:..) forms.
func parseCPE(cpe string) (vendor, product string, ok bool) {
cpe = strings.TrimSpace(cpe)
if cpe == "" {
return "", "", false
}
switch {
case strings.HasPrefix(cpe, "cpe:2.3:"):
parts := strings.Split(cpe, ":")
// cpe:2.3:part:vendor:product:...
if len(parts) >= 5 {
return normToken(parts[3]), normToken(parts[4]), true
}
case strings.HasPrefix(cpe, "cpe:/"):
parts := strings.Split(strings.TrimPrefix(cpe, "cpe:/"), ":")
// part:vendor:product:...
if len(parts) >= 3 {
return normToken(parts[1]), normToken(parts[2]), true
}
}
return "", "", false
}
// coerceStringSlice normalises tags/metadata values that may be a string, a
// comma-separated string, or a YAML list into a clean slice.
func coerceStringSlice(v any) []string {
var out []string
add := func(s string) {
for _, part := range strings.Split(s, ",") {
part = strings.TrimSpace(part)
if part != "" {
out = append(out, part)
}
}
}
switch t := v.(type) {
case string:
add(t)
case []any:
for _, item := range t {
if s, ok := item.(string); ok {
add(s)
}
}
case []string:
for _, s := range t {
add(s)
}
}
return out
}
func firstNonEmpty(values []string) string {
for _, v := range values {
if strings.TrimSpace(v) != "" {
return strings.TrimSpace(v)
}
}
return ""
}
@@ -0,0 +1,77 @@
package techgraph
// platforms are technologies with large plugin/extension ecosystems whose
// add-on templates carry the add-on as `product` (often with the platform as
// `vendor` or only as a tag). Runtime detection sees the platform, not each
// plugin, so those templates must be reachable from the platform node.
var platforms = map[string]string{
"wordpress": "wordpress",
"joomla": "joomla",
"drupal": "drupal",
"magento": "magento",
"prestashop": "prestashop",
"opencart": "opencart",
"moodle": "moodle",
"typo3": "typo3",
"dotnetnuke": "dotnetnuke",
"concrete5": "concrete5",
"shopware": "shopware",
"vbulletin": "vbulletin",
"mybb": "mybb",
"phpbb": "phpbb",
}
// platformTagAliases maps secondary tag tokens to a platform key.
var platformTagAliases = map[string]string{
"wp": "wordpress",
"wp-plugin": "wordpress",
"wp-theme": "wordpress",
}
// detectPlatform returns the platform node id when a template belongs to a
// platform's add-on ecosystem (and is not the platform core itself).
func detectPlatform(info templateInfo) (string, bool) {
vendor := normToken(info.Vendor)
product := normToken(info.Product)
if p, ok := platforms[vendor]; ok && product != "" && product != p {
return p, true
}
for _, tag := range info.Tags {
t := normToken(tag)
if p, ok := platforms[t]; ok {
if product != "" && product != p {
return p, true
}
}
if p, ok := platformTagAliases[t]; ok {
if product != p {
return p, true
}
}
}
return "", false
}
// isDetection reports whether a template is a phase-1 detector.
func isDetection(info templateInfo) bool {
for _, tag := range info.Tags {
switch normToken(tag) {
case "tech", "detect", "favicon":
return true
}
}
return false
}
// isDetectionSeverity guards the detection bucket against vuln/exposure templates
// that merely carry a "tech"/"detect" tag. Real fingerprint templates are
// informational; medium+ severity indicates an actual finding, not detection.
func isDetectionSeverity(info templateInfo) bool {
switch info.Severity {
case "", "info", "low":
return true
default:
return false
}
}
@@ -0,0 +1,221 @@
package techgraph
import (
"fmt"
"sort"
"strings"
)
// Selector resolves detected technology signals into a concrete fire-set of
// template ids using a tech-graph artifact. It is pure and deterministic.
type Selector struct {
g *Graph
byAlias map[string]string // alias/synonym/id/product -> canonical tech id
}
// NewSelector builds a selector with an alias index for fast signal resolution.
func NewSelector(g *Graph) *Selector {
s := &Selector{g: g, byAlias: make(map[string]string)}
// synonyms first (authoritative), then ids/products/aliases without clobbering.
for a, id := range g.Synonyms {
s.byAlias[normToken(a)] = id
}
for id, t := range g.Techs {
if _, ok := s.byAlias[normToken(id)]; !ok {
s.byAlias[normToken(id)] = id
}
if t.Product != "" {
if _, ok := s.byAlias[normToken(t.Product)]; !ok {
s.byAlias[normToken(t.Product)] = id
}
}
for _, a := range t.Aliases {
if _, ok := s.byAlias[normToken(a)]; !ok {
s.byAlias[normToken(a)] = id
}
}
}
return s
}
// Resolve maps detected signal tokens (wappalyzer names, detection tags, cpe
// products) to canonical tech ids. Tokens that match no node are returned as
// unresolved so the caller can apply tag-overlap fallback.
func (s *Selector) Resolve(signals []string) (techIDs, unresolved []string) {
seenTech := map[string]struct{}{}
seenUnres := map[string]struct{}{}
for _, sig := range signals {
tok := normToken(sig)
if tok == "" || genericTokens[tok] {
continue
}
if id, ok := s.byAlias[tok]; ok {
if _, dup := seenTech[id]; !dup {
seenTech[id] = struct{}{}
techIDs = append(techIDs, id)
}
continue
}
if _, dup := seenUnres[tok]; !dup {
seenUnres[tok] = struct{}{}
unresolved = append(unresolved, tok)
}
}
sort.Strings(techIDs)
sort.Strings(unresolved)
return techIDs, unresolved
}
// FireSet returns the precise template ids to execute for the given techs at the
// requested coverage tier (dependents + tier-gated baseline).
func (s *Selector) FireSet(tier Tier, techIDs []string) []string {
ids := map[string]struct{}{}
for _, id := range techIDs {
if t, ok := s.g.Techs[id]; ok {
for _, r := range t.Templates {
ids[r.ID] = struct{}{}
}
}
}
for _, b := range s.g.Baseline {
if tierIncludesBaseline(tier, b.Tier) {
ids[b.ID] = struct{}{}
}
}
out := make([]string, 0, len(ids))
for id := range ids {
out = append(out, id)
}
sort.Strings(out)
return out
}
// BaselineIDs returns just the tier-gated baseline ids.
func (s *Selector) BaselineIDs(tier Tier) []string {
var out []string
for _, b := range s.g.Baseline {
if tierIncludesBaseline(tier, b.Tier) {
out = append(out, b.ID)
}
}
sort.Strings(out)
return out
}
// tierIncludesBaseline gates baseline membership by coverage tier:
// - lean: no baseline (dependents only)
// - balanced: balanced baseline only
// - thorough: balanced + thorough baseline
func tierIncludesBaseline(scan, entry Tier) bool {
switch scan {
case TierLean:
return false
case TierThorough:
return true
default: // balanced
return entry == TierBalanced
}
}
// NormalizeTier coerces arbitrary input to a valid tier, defaulting to balanced.
// Case and surrounding whitespace are ignored.
func NormalizeTier(s string) Tier {
tier, err := ParseTier(s)
if err != nil {
return TierBalanced
}
return tier
}
// ParseTier validates a coverage tier. Empty input defaults to balanced.
// Unknown values return an error instead of silently changing behavior.
func ParseTier(s string) (Tier, error) {
switch Tier(strings.ToLower(strings.TrimSpace(s))) {
case "", TierBalanced:
return TierBalanced, nil
case TierLean:
return TierLean, nil
case TierThorough:
return TierThorough, nil
default:
return "", fmt.Errorf("invalid automatic scan coverage %q (allowed: lean, balanced, thorough)", s)
}
}
// Candidate is a loaded template that the artifact may not know about yet.
type Candidate struct {
ID string
Tags []string
Category string
}
// AbsentDependents returns ids of templates the graph does not contain whose
// tags resolve to exactly one of the detected techs. Ambiguous tags and
// excluded categories are left out, so this only recovers templates added
// after the artifact was generated.
func (s *Selector) AbsentDependents(detected []string, candidates []Candidate) []string {
if s == nil || s.g == nil || len(detected) == 0 || len(candidates) == 0 {
return nil
}
want := make(map[string]struct{}, len(detected))
for _, id := range detected {
want[id] = struct{}{}
}
known := s.knownIDs()
var out []string
for _, c := range candidates {
if c.ID == "" {
continue
}
if _, ok := known[c.ID]; ok {
continue
}
// An unknown category is skipped: excluded categories such as fuzzing
// and cloud must not be pulled in just because the path did not parse.
if _, excluded := excludedCategories[c.Category]; excluded || c.Category == "" {
continue
}
matches := map[string]struct{}{}
for _, tag := range c.Tags {
tok := normToken(tag)
if len(tok) < 5 || genericTokens[tok] {
continue
}
id, ok := s.byAlias[tok]
if !ok {
continue
}
matches[id] = struct{}{}
}
if len(matches) != 1 {
continue
}
var only string
for id := range matches {
only = id
}
if _, ok := want[only]; !ok {
continue
}
out = append(out, c.ID)
}
sort.Strings(out)
return out
}
// knownIDs is every template id the artifact already classifies.
func (s *Selector) knownIDs() map[string]struct{} {
ids := make(map[string]struct{})
for _, tech := range s.g.Techs {
for _, ref := range tech.Templates {
ids[ref.ID] = struct{}{}
}
}
for _, b := range s.g.Baseline {
ids[b.ID] = struct{}{}
}
for _, d := range s.g.Detection {
ids[d.ID] = struct{}{}
}
return ids
}
@@ -0,0 +1,91 @@
package techgraph
import (
"reflect"
"testing"
)
func testGraph() *Graph {
return &Graph{
Techs: map[string]*Tech{
"wordpress": {ID: "wordpress", Product: "wordpress", Aliases: []string{"wordpress"},
Templates: []TemplateRef{{ID: "wp-cve-1"}, {ID: "wp-plugin-1", Source: "platform"}}},
"atlassian:confluence": {ID: "atlassian:confluence", Product: "confluence", Aliases: []string{"confluence"},
Templates: []TemplateRef{{ID: "confluence-cve-1"}}},
},
Baseline: []BaselineEntry{
{ID: "git-config", Tier: TierBalanced},
{ID: "noisy-osinty", Tier: TierThorough},
},
Synonyms: map[string]string{"aem": "adobe:experience_manager"},
}
}
func TestResolve(t *testing.T) {
s := NewSelector(testGraph())
tech, unres := s.Resolve([]string{"WordPress", "confluence", "php", "cve"})
if len(tech) != 2 {
t.Fatalf("want 2 techs, got %v", tech)
}
if len(unres) != 1 || unres[0] != "php" {
t.Fatalf("want [php] unresolved, got %v", unres)
}
}
func TestFireSetTiers(t *testing.T) {
s := NewSelector(testGraph())
techs := []string{"wordpress"}
lean := s.FireSet(TierLean, techs)
if want := []string{"wp-cve-1", "wp-plugin-1"}; !reflect.DeepEqual(lean, want) {
t.Fatalf("lean: want %v, got %v", want, lean)
}
bal := s.FireSet(TierBalanced, techs)
if want := []string{"git-config", "wp-cve-1", "wp-plugin-1"}; !reflect.DeepEqual(bal, want) {
t.Fatalf("balanced: want %v, got %v", want, bal)
}
tho := s.FireSet(TierThorough, techs)
if want := []string{"git-config", "noisy-osinty", "wp-cve-1", "wp-plugin-1"}; !reflect.DeepEqual(tho, want) {
t.Fatalf("thorough: want %v, got %v", want, tho)
}
}
func TestNormalizeTier(t *testing.T) {
for in, want := range map[string]Tier{"lean": TierLean, " THOROUGH ": TierThorough, "": TierBalanced, "x": TierBalanced} {
if got := NormalizeTier(in); got != want {
t.Fatalf("NormalizeTier(%q)=%q want %q", in, got, want)
}
}
}
func TestParseTier(t *testing.T) {
if _, err := ParseTier("not-a-tier"); err == nil {
t.Fatal("expected error for unknown tier")
}
got, err := ParseTier(" Lean ")
if err != nil || got != TierLean {
t.Fatalf("ParseTier(Lean)=%q %v", got, err)
}
}
func TestAbsentDependents(t *testing.T) {
s := NewSelector(testGraph())
candidates := []Candidate{
{ID: "new-wp", Tags: []string{"wordpress"}, Category: "cves"},
{ID: "ambiguous", Tags: []string{"wordpress", "confluence"}, Category: "cves"},
{ID: "fuzzed", Tags: []string{"wordpress"}, Category: "fuzzing"},
{ID: "wp-cve-1", Tags: []string{"wordpress"}, Category: "cves"},
{ID: "short", Tags: []string{"wp"}, Category: "cves"},
{ID: "other-tech", Tags: []string{"wordpress"}, Category: "cves"},
}
got := s.AbsentDependents([]string{"wordpress"}, candidates[:5])
if want := []string{"new-wp"}; !reflect.DeepEqual(got, want) {
t.Fatalf("want %v, got %v", want, got)
}
if got := s.AbsentDependents([]string{"atlassian:confluence"}, candidates[5:]); len(got) != 0 {
t.Fatalf("wordpress tag must not attach to confluence, got %v", got)
}
if got := s.AbsentDependents([]string{"wordpress"}, []Candidate{{ID: "no-path", Tags: []string{"wordpress"}}}); len(got) != 0 {
t.Fatalf("unknown category must not be added, got %v", got)
}
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,84 @@
// Package techgraph builds the offline technology->template mapping artifact
// (tech-graph.json) consumed by automatic scan v2. See AUTOMATIC_SCAN_V2.md.
package techgraph
// ArtifactVersion is the schema version of the emitted artifact.
const ArtifactVersion = 1
// Tier gates how aggressively a template is selected at runtime.
type Tier string
const (
TierLean Tier = "lean"
TierBalanced Tier = "balanced"
TierThorough Tier = "thorough"
)
// Graph is the top-level tech-graph artifact.
type Graph struct {
Version int `json:"version"`
GeneratedAt string `json:"generated_at"`
SourceHash string `json:"source_hash,omitempty"`
Techs map[string]*Tech `json:"techs"`
Baseline []BaselineEntry `json:"baseline"`
Detection []DetectionEntry `json:"detection"`
Synonyms map[string]string `json:"synonyms"`
Stats Stats `json:"stats"`
}
// DetectionEntry is a phase-1 detector template (tagged tech/detect/favicon).
// Running these fingerprints the target before selection.
type DetectionEntry struct {
ID string `json:"id"`
Tech string `json:"tech,omitempty"` // canonical tech this detector anchors, if any
}
// Tech is a single canonical technology and the templates that depend on it.
type Tech struct {
ID string `json:"id"`
Vendor string `json:"vendor,omitempty"`
Product string `json:"product,omitempty"`
CPE string `json:"cpe,omitempty"`
Aliases []string `json:"aliases,omitempty"`
Templates []TemplateRef `json:"templates"`
}
// TemplateRef references a template that depends on a tech.
type TemplateRef struct {
ID string `json:"id"`
Category string `json:"category,omitempty"`
Severity string `json:"severity,omitempty"`
// Product is the original sub-product when attached to a platform node via
// platform grouping (e.g. plugin "wp-optimize" under platform "wordpress").
Product string `json:"product,omitempty"`
// Source records how the template was attached: "cpe", "platform", "tag".
Source string `json:"source,omitempty"`
// VersionRange is reserved for the AI version-parsing milestone. Empty means
// "fire regardless of detected version".
VersionRange string `json:"version_range,omitempty"`
}
// BaselineEntry is a tech-agnostic template that runs on every applicable target.
type BaselineEntry struct {
ID string `json:"id"`
Category string `json:"category,omitempty"`
Severity string `json:"severity,omitempty"`
Tier Tier `json:"tier"`
}
// Stats summarises a build run.
type Stats struct {
Total int `json:"total"`
Dependents int `json:"dependents"`
DependentsCPE int `json:"dependents_cpe"`
DependentsPlat int `json:"dependents_platform"`
DependentsDir int `json:"dependents_dir"`
DependentsTag int `json:"dependents_tag"`
DependentsID int `json:"dependents_id"`
Baseline int `json:"baseline"`
Detection int `json:"detection"`
Excluded int `json:"excluded"`
Unmapped int `json:"unmapped"`
Techs int `json:"techs"`
ParseError int `json:"parse_error"`
}
@@ -0,0 +1,225 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>nuclei tech-graph visualizer</title>
<style>
:root {
--bg: #0c0f17; --panel: #141926; --panel2: #1b2233; --line: #263149;
--txt: #e6edf6; --muted: #8b97ad; --accent: #5b8cff;
--cpe:#36c98a; --platform:#b07cff; --dir:#3bc4c4; --tag:#5b8cff; --id:#f0a23c;
--crit:#ff5470; --high:#ff8a3d; --med:#f5c451; --low:#5b8cff; --info:#7d8aa3;
}
* { box-sizing: border-box; }
body { margin:0; background:var(--bg); color:var(--txt);
font:14px/1.5 ui-sans-serif,system-ui,-apple-system,"Segoe UI",Roboto,sans-serif; }
header { padding:18px 24px; border-bottom:1px solid var(--line);
display:flex; align-items:center; gap:16px; flex-wrap:wrap;
background:linear-gradient(180deg,#10141f,#0c0f17); position:sticky; top:0; z-index:5; }
h1 { font-size:16px; margin:0; font-weight:600; letter-spacing:.3px; }
h1 span { color:var(--accent); }
.meta { color:var(--muted); font-size:12px; }
.btn { background:var(--panel2); color:var(--txt); border:1px solid var(--line);
padding:7px 13px; border-radius:8px; cursor:pointer; font-size:13px; }
.btn:hover { border-color:var(--accent); }
main { padding:20px 24px; max-width:1240px; margin:0 auto; }
.cards { display:grid; grid-template-columns:repeat(auto-fit,minmax(140px,1fr)); gap:12px; margin-bottom:18px; }
.card { background:var(--panel); border:1px solid var(--line); border-radius:12px; padding:14px 16px; }
.card .n { font-size:24px; font-weight:700; }
.card .l { color:var(--muted); font-size:12px; text-transform:uppercase; letter-spacing:.5px; }
.srcbar { display:flex; height:10px; border-radius:6px; overflow:hidden; margin:4px 0 16px; border:1px solid var(--line); }
.srcbar div { height:100%; }
.legend { display:flex; gap:14px; flex-wrap:wrap; font-size:12px; color:var(--muted); margin-bottom:18px; }
.legend i { display:inline-block; width:10px; height:10px; border-radius:3px; margin-right:5px; vertical-align:middle; }
.tabs { display:flex; gap:6px; border-bottom:1px solid var(--line); margin-bottom:16px; }
.tab { padding:9px 15px; cursor:pointer; color:var(--muted); border-bottom:2px solid transparent; }
.tab.active { color:var(--txt); border-bottom-color:var(--accent); }
.toolbar { display:flex; gap:10px; margin-bottom:14px; flex-wrap:wrap; align-items:center; }
input[type=search], select { background:var(--panel2); border:1px solid var(--line); color:var(--txt);
padding:8px 12px; border-radius:8px; font-size:13px; }
input[type=search] { flex:1; min-width:220px; }
.tech { background:var(--panel); border:1px solid var(--line); border-radius:10px; margin-bottom:8px; overflow:hidden; }
.tech > summary { padding:11px 15px; cursor:pointer; display:flex; align-items:center; gap:12px; list-style:none; }
.tech > summary::-webkit-details-marker { display:none; }
.tech .id { font-weight:600; }
.tech .cnt { margin-left:auto; color:var(--muted); font-size:12px; background:var(--panel2);
padding:2px 9px; border-radius:20px; }
.cpe { color:var(--muted); font-size:11px; font-family:ui-monospace,monospace; }
.tpls { border-top:1px solid var(--line); padding:6px 15px 12px; }
.row { display:flex; align-items:center; gap:8px; padding:5px 0; border-bottom:1px dashed #1e2740; font-size:13px; }
.row:last-child { border-bottom:none; }
.row .tid { font-family:ui-monospace,monospace; font-size:12px; }
.badge { font-size:10px; padding:1px 7px; border-radius:20px; text-transform:uppercase; letter-spacing:.4px; font-weight:600; }
.b-cpe{background:rgba(54,201,138,.16);color:var(--cpe)} .b-platform{background:rgba(176,124,255,.16);color:var(--platform)}
.b-dir{background:rgba(59,196,196,.16);color:var(--dir)} .b-tag{background:rgba(91,140,255,.16);color:var(--tag)}
.b-id{background:rgba(240,162,60,.16);color:var(--id)}
.sev { width:7px; height:7px; border-radius:50%; flex:none; }
.s-critical{background:var(--crit)} .s-high{background:var(--high)} .s-medium{background:var(--med)}
.s-low{background:var(--low)} .s-info{background:var(--info)} .s-unknown{background:#444}
.cat { color:var(--muted); font-size:11px; }
.drop { border:2px dashed var(--line); border-radius:14px; padding:60px; text-align:center; color:var(--muted); }
.drop.hot { border-color:var(--accent); color:var(--txt); }
.muted { color:var(--muted); }
.hidden { display:none; }
.alias { color:var(--muted); font-size:11px; }
</style>
</head>
<body>
<header>
<h1>nuclei <span>tech-graph</span> visualizer</h1>
<span class="meta" id="meta"></span>
<button class="btn" id="pick" style="margin-left:auto">Load tech-graph.json</button>
<input type="file" id="file" accept="application/json,.json" class="hidden" />
</header>
<main>
<div id="empty" class="drop">
Drop <b>tech-graph.json</b> here, or click <b>Load</b>.<br/>
<span class="muted">Generate it with <code>techgraph-gen -templates &lt;dir&gt; -output tech-graph.json</code></span>
</div>
<div id="app" class="hidden">
<div class="cards" id="cards"></div>
<div class="srcbar" id="srcbar"></div>
<div class="legend" id="legend"></div>
<div class="tabs">
<div class="tab active" data-tab="techs">Technologies</div>
<div class="tab" data-tab="baseline">Baseline</div>
<div class="tab" data-tab="detection">Detection</div>
<div class="tab" data-tab="synonyms">Synonyms</div>
</div>
<div class="toolbar">
<input type="search" id="q" placeholder="Search tech, template id, alias, cpe…" />
<select id="sort">
<option value="count">Sort: most templates</option>
<option value="name">Sort: name</option>
</select>
<select id="srcFilter">
<option value="">Source: all</option>
<option value="cpe">cpe</option><option value="platform">platform</option>
<option value="dir">dir</option><option value="tag">tag</option><option value="id">id</option>
</select>
<span class="muted" id="resultCount"></span>
</div>
<div id="view"></div>
</div>
</main>
<script>
const SEV = ["critical","high","medium","low","info"];
const SRC = ["cpe","platform","dir","tag","id"];
const SRC_COLOR = {cpe:"--cpe",platform:"--platform",dir:"--dir",tag:"--tag",id:"--id"};
let G = null;
const $ = s => document.querySelector(s);
const el = (t,c,h) => { const e=document.createElement(t); if(c)e.className=c; if(h!=null)e.innerHTML=h; return e; };
const esc = s => String(s==null?"":s).replace(/[&<>]/g,m=>({"&":"&amp;","<":"&lt;",">":"&gt;"}[m]));
function load(json){
G = json;
$("#empty").classList.add("hidden");
$("#app").classList.remove("hidden");
$("#meta").textContent = `v${json.version} · ${json.generated_at||""} · ${json.source_hash? json.source_hash.slice(0,18)+"…":""}`;
renderCards(); renderTabs(); render();
}
function renderCards(){
const s = G.stats || {};
const cards = [
["templates", s.total], ["dependents", s.dependents], ["techs", s.techs],
["baseline", s.baseline], ["detection", s.detection], ["excluded", s.excluded], ["unmapped", s.unmapped],
];
$("#cards").innerHTML = cards.map(([l,n])=>`<div class="card"><div class="n">${n??0}</div><div class="l">${l}</div></div>`).join("");
// source bar from per-source dependent counts
const bySrc = {cpe:s.dependents_cpe,platform:s.dependents_platform,dir:s.dependents_dir,tag:s.dependents_tag,id:s.dependents_id};
const tot = SRC.reduce((a,k)=>a+(bySrc[k]||0),0)||1;
$("#srcbar").innerHTML = SRC.map(k=>`<div style="width:${100*(bySrc[k]||0)/tot}%;background:var(${SRC_COLOR[k]})" title="${k}: ${bySrc[k]||0}"></div>`).join("");
$("#legend").innerHTML = SRC.map(k=>`<span><i style="background:var(${SRC_COLOR[k]})"></i>${k} ${bySrc[k]||0}</span>`).join("")
+ ` <span style="margin-left:auto" class="muted">dependent attachment source</span>`;
}
let TAB = "techs";
function renderTabs(){
document.querySelectorAll(".tab").forEach(t=>{
t.onclick = ()=>{ TAB=t.dataset.tab; document.querySelectorAll(".tab").forEach(x=>x.classList.toggle("active",x===t)); render(); };
});
}
["#q","#sort","#srcFilter"].forEach(s=>document.addEventListener("input",e=>{ if(e.target.matches(s)) render(); }));
function render(){
const q = $("#q").value.trim().toLowerCase();
const v = $("#view"); v.innerHTML="";
if(TAB==="techs") return renderTechs(q,v);
if(TAB==="baseline") return renderList(G.baseline||[], q, v, b=>[b.id,b.category], b=>rowHTML(b.id,null,b.category,b.severity,b.tier));
if(TAB==="detection") return renderList(G.detection||[], q, v, d=>[d.id,d.tech], d=>rowHTML(d.id,null,"detector → "+(d.tech||"—")));
if(TAB==="synonyms"){
const ents=Object.entries(G.synonyms||{}).filter(([a,t])=>!q||a.includes(q)||t.includes(q)).sort();
$("#resultCount").textContent = ents.length+" synonyms";
ents.forEach(([a,t])=>{ const r=el("div","row"); r.innerHTML=`<span class="tid">${esc(a)}</span><span class="muted">→</span><span class="badge b-tag">${esc(t)}</span>`; v.appendChild(r); });
}
}
function renderTechs(q,v){
const srcF = $("#srcFilter").value;
let techs = Object.values(G.techs||{});
techs = techs.filter(t=>{
const tpls = t.templates||[];
if(srcF && !tpls.some(x=>x.source===srcF)) return false;
if(!q) return true;
if(t.id.toLowerCase().includes(q) || (t.cpe||"").toLowerCase().includes(q)) return true;
if((t.aliases||[]).some(a=>a.includes(q))) return true;
return tpls.some(x=>x.id.toLowerCase().includes(q));
});
techs.sort($("#sort").value==="name"
? (a,b)=>a.id.localeCompare(b.id)
: (a,b)=>(b.templates||[]).length-(a.templates||[]).length);
$("#resultCount").textContent = `${techs.length} techs · ${techs.reduce((n,t)=>n+(t.templates||[]).length,0)} templates`;
const frag=document.createDocumentFragment();
for(const t of techs.slice(0,400)){
const d=el("details","tech");
const tpls=(t.templates||[]);
d.innerHTML=`<summary><span class="id">${esc(t.id)}</span>`
+(t.cpe?`<span class="cpe">${esc(t.cpe)}</span>`:"")
+(t.aliases&&t.aliases.length?`<span class="alias">${esc(t.aliases.slice(0,4).join(", "))}</span>`:"")
+`<span class="cnt">${tpls.length}</span></summary>`;
const body=el("div","tpls");
let list=tpls; if(srcF) list=list.filter(x=>x.source===srcF);
if(q) list=list.filter(x=>x.id.toLowerCase().includes(q)||t.id.toLowerCase().includes(q)||(t.aliases||[]).some(a=>a.includes(q)));
body.innerHTML = list.slice(0,300).map(x=>rowHTML(x.id,x.source,x.category,x.severity)).join("");
d.appendChild(body); frag.appendChild(d);
}
v.appendChild(frag);
if(techs.length>400) v.appendChild(el("div","muted",`Showing first 400 of ${techs.length}. Refine your search.`));
}
function renderList(items,q,v,keys,rowFn){
const f=items.filter(it=>!q||keys(it).some(k=>String(k||"").toLowerCase().includes(q)));
$("#resultCount").textContent = f.length+" items";
v.innerHTML = `<div class="tech"><div class="tpls">`+f.slice(0,1000).map(rowFn).join("")+`</div></div>`;
}
function rowHTML(id,source,category,severity,extra){
const sev=(severity||"unknown").toLowerCase();
return `<div class="row"><span class="sev s-${SEV.includes(sev)?sev:"unknown"}"></span>`
+ `<span class="tid">${esc(id)}</span>`
+ (source?`<span class="badge b-${source}">${esc(source)}</span>`:"")
+ (category?`<span class="cat">${esc(category)}</span>`:"")
+ (extra?`<span class="cat">· ${esc(extra)}</span>`:"")
+ `</div>`;
}
// ---- loading: file picker, drag/drop, auto-fetch ----
$("#pick").onclick=()=>$("#file").click();
$("#file").onchange=e=>{ const f=e.target.files[0]; if(f) f.text().then(t=>load(JSON.parse(t))); };
const dz=$("#empty");
["dragover","dragenter"].forEach(ev=>dz.addEventListener(ev,e=>{e.preventDefault();dz.classList.add("hot");}));
["dragleave","drop"].forEach(ev=>dz.addEventListener(ev,e=>{e.preventDefault();dz.classList.remove("hot");}));
dz.addEventListener("drop",e=>{ const f=e.dataTransfer.files[0]; if(f) f.text().then(t=>load(JSON.parse(t))); });
if(window.__TECHGRAPH__){ load(window.__TECHGRAPH__); }
else { fetch("tech-graph.json").then(r=>r.ok?r.json():Promise.reject()).then(load).catch(()=>{}); }
</script>
</body>
</html>
@@ -1,8 +1,6 @@
package eventcreator
import (
"sync/atomic"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
@@ -12,29 +10,6 @@ import (
"golang.org/x/text/language"
)
// RecordFunc receives the event of a request before its operators run, which
// is what a replay needs to run them again.
type RecordFunc func(request protocols.Request, event output.InternalEvent)
// Every protocol builds its result here, so a recorder set here covers all
// of them. Scans read it concurrently, hence the atomic.
var record atomic.Pointer[RecordFunc]
// SetRecord installs fn, nil to stop recording, and returns the callback it
// replaced so the caller can put it back.
func SetRecord(fn RecordFunc) RecordFunc {
var previous *RecordFunc
if fn == nil {
previous = record.Swap(nil)
} else {
previous = record.Swap(&fn)
}
if previous == nil {
return nil
}
return *previous
}
// CreateEvent wraps the outputEvent with the result of the operators defined on the request
func CreateEvent(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool) *output.InternalWrappedEvent {
return CreateEventWithAdditionalOptions(request, outputEvent, isResponseDebug, nil)
@@ -44,9 +19,6 @@ func CreateEvent(request protocols.Request, outputEvent output.InternalEvent, is
// and enables extending the resulting event with additional attributes or values.
func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool,
addAdditionalOptions func(internalWrappedEvent *output.InternalWrappedEvent)) *output.InternalWrappedEvent {
if fn := record.Load(); fn != nil {
(*fn)(request, outputEvent)
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
// Dump response variables if ran in debug mode
@@ -1,26 +0,0 @@
package eventcreator
import (
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/stretchr/testify/require"
)
func TestSetRecordRestoresTheCallbackItReplaced(t *testing.T) {
var calls []string
first := RecordFunc(func(protocols.Request, output.InternalEvent) { calls = append(calls, "first") })
second := RecordFunc(func(protocols.Request, output.InternalEvent) { calls = append(calls, "second") })
require.Nil(t, SetRecord(first))
previous := SetRecord(second)
(*record.Load())(nil, nil)
// a second runner stopping puts the first one's recorder back, not nil
SetRecord(previous)
(*record.Load())(nil, nil)
require.Equal(t, []string{"second", "first"}, calls)
SetRecord(nil)
require.Nil(t, record.Load())
}
@@ -16,7 +16,6 @@ type Dialers struct {
HTTPClientPool *HTTPPool
PerHostRateLimitPool any // *hostratelimit.PerHostRateLimitPool
HTTPToHTTPSPortTracker any // *httpclientpool.HTTPToHTTPSPortTracker
SharedResponses any // *httpclientpool.sharedResponses
HTTPDesyncHosts *ExpiringSet
NetworkPolicy *networkpolicy.NetworkPolicy
LocalFileAccessAllowed bool
@@ -7,10 +7,10 @@ import (
"strings"
"sync"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/networkpolicy"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/proto"
"github.com/projectdiscovery/utils/errkit"
stringsutil "github.com/projectdiscovery/utils/strings"
urlutil "github.com/projectdiscovery/utils/url"
@@ -15,7 +15,7 @@ func TestMemGuardianGoroutineLeak(t *testing.T) {
defer goleak.VerifyNone(t,
goleak.IgnoreAnyContainingPkg("go.opencensus.io/stats/view"),
goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/rod"),
goleak.IgnoreAnyContainingPkg("github.com/go-rod/rod"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/server"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/ratelimit"),
// expirable LRU cache creates a background goroutine for TTL expiration that persists
@@ -104,7 +104,7 @@ func TestMemGuardianReset(t *testing.T) {
defer goleak.VerifyNone(t,
goleak.IgnoreAnyContainingPkg("go.opencensus.io/stats/view"),
goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/rod"),
goleak.IgnoreAnyContainingPkg("github.com/go-rod/rod"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/server"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/ratelimit"),
// expirable LRU cache creates a background goroutine for TTL expiration that persists
-50
View File
@@ -1,10 +1,8 @@
package dns
import (
"context"
"net"
"strings"
"time"
"github.com/miekg/dns"
"github.com/pkg/errors"
@@ -18,7 +16,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool"
fileutil "github.com/projectdiscovery/utils/file"
iputil "github.com/projectdiscovery/utils/ip"
)
// Request contains a DNS protocol request to be made from a template
@@ -207,48 +204,6 @@ func resolverHost(resolver string) string {
return r
}
// pinResolverHost replaces a resolver host name with an address that the
// network policy allows. The policy check passes when one address of the name
// is allowed, and the dns client resolves the name again when it dials, so it
// could otherwise reach a denied address of the same name.
// The lookup stops after timeout, so a slow system resolver cannot hold the
// request.
func pinResolverHost(executionID, resolver, host string, timeout time.Duration) (string, error) {
// DoH and DoT resolvers dial through the scan dialer, which checks every
// address, and DoT verifies the certificate against the name.
if iputil.IsIP(host) || strings.HasPrefix(resolver, "doh:") || strings.HasPrefix(resolver, "dot:") {
return resolver, nil
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
addrs, err := net.DefaultResolver.LookupHost(ctx, host)
if err != nil {
return "", errors.Wrapf(err, "could not resolve dns resolver %s", resolver)
}
prefix, address := resolver, ""
if index := strings.LastIndex(resolver, host); index != -1 {
prefix, address = resolver[:index], resolver[index:]
}
port := "53"
if _, p, err := net.SplitHostPort(address); err == nil && p != "" {
port = p
}
for _, addr := range addrs {
if protocolstate.IsHostAllowed(executionID, addr) {
return prefix + net.JoinHostPort(addr, port), nil
}
}
return "", errors.Errorf("dns resolver %s is blocked by network policy", resolver)
}
// resolverLookupTimeout is the scan timeout, or ten seconds when it is not set.
func resolverLookupTimeout(options *protocols.ExecutorOptions) time.Duration {
if options.Options.Timeout > 0 {
return time.Duration(options.Options.Timeout) * time.Second
}
return 10 * time.Second
}
func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadata map[string]interface{}) (*dnsclientpool.Client, error) {
dnsClientOptions := &dnsclientpool.Configuration{
Retries: request.Retries,
@@ -282,11 +237,6 @@ func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadat
if !protocolstate.IsHostAllowed(options.Options.ExecutionId, host) {
return nil, errors.Errorf("dns resolver %s is blocked by network policy", resolver)
}
pinned, err := pinResolverHost(options.Options.ExecutionId, resolver, host, resolverLookupTimeout(options))
if err != nil {
return nil, err
}
resolver = pinned
}
resolvers = append(resolvers, resolver)
}
@@ -93,7 +93,7 @@ func Get(options *types.Options, configuration *Configuration) (*Client, error)
if (configuration.Retries <= 1) && len(configuration.Resolvers) == 0 {
return cachedClient(resolversFor(options))
}
if len(options.InternalResolversList) == 0 && hasPolicyResolver(configuration.Resolvers) {
if len(options.InternalResolversList) == 0 && hasDohResolver(configuration.Resolvers) {
return getPolicyClient(options, configuration)
}
hash := configuration.Hash()
+16 -74
View File
@@ -3,14 +3,12 @@ package dnsclientpool
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"net"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/miekg/dns"
"github.com/pkg/errors"
@@ -21,8 +19,8 @@ import (
)
// Client sends the DNS queries of a template. It sends queries for template
// DoH and DoT resolvers itself, because the retryabledns clients dial and
// follow redirects without the network policy of the scan.
// DoH resolvers itself, because the retryabledns DoH client dials and follows
// redirects without the network policy of the scan.
type Client struct {
*retryabledns.Client
@@ -31,51 +29,31 @@ type Client struct {
index uint32
doh *doh.Client
transport *http.Transport
dial func(ctx context.Context, network, address string) (net.Conn, error)
timeout time.Duration
}
// resolver is one template resolver. url is set for a DoH resolver and dot
// for a DoT resolver, and both are empty for a resolver that retryabledns
// queries over UDP or TCP.
// resolver is one template resolver. url is empty for a resolver that
// retryabledns queries over UDP, TCP or DoT.
type resolver struct {
url string
method doh.Method
dot string
}
var (
// policyClients holds the clients with DoH or DoT resolvers of each execution,
// policyClients holds the clients with DoH resolvers of each execution,
// because each one dials with the dialer of its execution.
policyClients = map[string]map[string]*Client{}
policyMu sync.Mutex
// dotRootCAs verifies DoT certificates, and nil uses the system roots.
dotRootCAs *x509.CertPool
)
func hasPolicyResolver(resolvers []string) bool {
func hasDohResolver(resolvers []string) bool {
for _, r := range resolvers {
if strings.HasPrefix(r, "doh:") || strings.HasPrefix(r, "dot:") {
if strings.HasPrefix(r, "doh:") {
return true
}
}
return false
}
// parseDotResolver returns the host and port of a resolver such as
// dot:dns.example or dot:dns.example:853, in the same way as retryabledns.
func parseDotResolver(r string) (resolver, bool) {
if !strings.HasPrefix(r, "dot:") {
return resolver{}, false
}
address := strings.TrimPrefix(r, "dot:")
if _, _, err := net.SplitHostPort(address); err != nil {
address = net.JoinHostPort(address, "853")
}
return resolver{dot: address}, true
}
// parseDohResolver returns the URL and method of a resolver such as
// doh:https://dns.example/dns-query:get, in the same way as retryabledns.
func parseDohResolver(r string) (resolver, bool) {
@@ -113,14 +91,10 @@ func getPolicyClient(options *types.Options, configuration *Configuration) (*Cli
client.resolvers = append(client.resolvers, parsed)
continue
}
if parsed, ok := parseDotResolver(r); ok {
client.resolvers = append(client.resolvers, parsed)
continue
}
client.resolvers = append(client.resolvers, resolver{})
networkResolvers = append(networkResolvers, r)
}
// Trace and the UDP and TCP resolvers still go through retryabledns.
// Trace and the non DoH resolvers still go through retryabledns.
if len(networkResolvers) == 0 {
networkResolvers = defaultResolvers
}
@@ -137,16 +111,14 @@ func getPolicyClient(options *types.Options, configuration *Configuration) (*Cli
// The scan dialer checks every address it connects to, so redirects
// and names that resolve to a denied address are refused too. It also
// goes through the scan proxy when one is set.
client.dial = func(ctx context.Context, network, address string) (net.Conn, error) {
dialers := protocolstate.GetDialersWithId(executionID)
if dialers == nil || dialers.Fastdialer == nil {
return nil, fmt.Errorf("dialers not initialized for %s", executionID)
}
return dialers.Fastdialer.Dial(ctx, network, address)
}
client.timeout = options.GetTimeouts().DialTimeout
client.transport = &http.Transport{
DialContext: client.dial,
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
dialers := protocolstate.GetDialersWithId(executionID)
if dialers == nil || dialers.Fastdialer == nil {
return nil, fmt.Errorf("dialers not initialized for %s", executionID)
}
return dialers.Fastdialer.Dial(ctx, network, address)
},
// same as the retryabledns DoH client
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec G402
}
@@ -177,8 +149,6 @@ func (c *Client) Do(msg *dns.Msg) (*dns.Msg, error) {
r := c.resolvers[index%uint32(len(c.resolvers))]
if r.url != "" {
resp, err = c.doh.QueryWithDOHMsg(r.method, doh.Resolver{URL: r.url}, msg)
} else if r.dot != "" {
resp, err = c.queryDoT(r.dot, msg)
} else {
resp, err = c.Client.Do(msg)
}
@@ -189,35 +159,7 @@ func (c *Client) Do(msg *dns.Msg) (*dns.Msg, error) {
return resp, retryabledns.ErrRetriesExceeded
}
// queryDoT sends a DNS message over TLS to address. It dials through the scan
// dialer and verifies the certificate against the resolver host, as the
// retryabledns DoT client does.
func (c *Client) queryDoT(address string, msg *dns.Msg) (*dns.Msg, error) {
host, _, err := net.SplitHostPort(address)
if err != nil {
return nil, err
}
timeout := c.timeout
if timeout <= 0 {
timeout = retryabledns.DefaultOptions.Timeout
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
conn, err := c.dial(ctx, "tcp", address)
if err != nil {
return nil, err
}
tlsConn := tls.Client(conn, &tls.Config{ServerName: host, RootCAs: dotRootCAs, MinVersion: tls.VersionTLS12})
defer func() { _ = tlsConn.Close() }()
if err := tlsConn.HandshakeContext(ctx); err != nil {
return nil, err
}
dnsClient := &dns.Client{Net: "tcp-tls", Timeout: timeout}
resp, _, err := dnsClient.ExchangeWithConn(msg, &dns.Conn{Conn: tlsConn})
return resp, err
}
// Close drops the clients with DoH or DoT resolvers of an execution.
// Close drops the clients with DoH resolvers of an execution.
func Close(executionID string) {
policyMu.Lock()
clients := policyClients[executionID]
@@ -1,91 +0,0 @@
package dnsclientpool
import (
"crypto/tls"
"net"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/miekg/dns"
"github.com/stretchr/testify/require"
)
// newDotListener accepts TCP connections on 127.0.0.1, counts them and closes
// them, so a test sees each DoT dial that reaches the address.
func newDotListener(t *testing.T, hits *atomic.Int32) string {
t.Helper()
l, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = l.Close() })
go func() {
for {
conn, err := l.Accept()
if err != nil {
return
}
hits.Add(1)
_ = conn.Close()
}
}()
_, port, err := net.SplitHostPort(l.Addr().String())
require.NoError(t, err)
return port
}
func TestDotResolverObeysNetworkPolicy(t *testing.T) {
for _, host := range []string{"127.0.0.1", "localhost"} {
t.Run("denied address "+host, func(t *testing.T) {
var hits atomic.Int32
port := newDotListener(t, &hits)
// localhost passes the resolver check through ::1, while 127.0.0.1 is denied
options := dohOptions(t, []string{"127.0.0.1"})
client, err := Get(options, &Configuration{Retries: 2, Resolvers: []string{"dot:" + net.JoinHostPort(host, port)}})
require.NoError(t, err)
_, _ = client.Do(txtQuery())
require.Zero(t, hits.Load(), "DoT query reached a denied address")
})
}
t.Run("allowed address", func(t *testing.T) {
var hits atomic.Int32
port := newDotListener(t, &hits)
options := dohOptions(t, nil)
client, err := Get(options, &Configuration{Retries: 1, Resolvers: []string{"dot:127.0.0.1:" + port}})
require.NoError(t, err)
_, _ = client.Do(txtQuery())
require.Equal(t, int32(1), hits.Load())
})
}
func TestDotResolverAnswers(t *testing.T) {
// httptest gives a certificate for 127.0.0.1 and a pool that trusts it.
certs := httptest.NewTLSServer(http.NotFoundHandler())
t.Cleanup(certs.Close)
previous := dotRootCAs
dotRootCAs = certs.Client().Transport.(*http.Transport).TLSClientConfig.RootCAs
t.Cleanup(func() { dotRootCAs = previous })
l, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: certs.TLS.Certificates, MinVersion: tls.VersionTLS12})
require.NoError(t, err)
server := &dns.Server{Listener: l, Net: "tcp-tls", Handler: dns.HandlerFunc(func(w dns.ResponseWriter, q *dns.Msg) {
m := new(dns.Msg)
m.SetReply(q)
rr, _ := dns.NewRR(`example.com. 60 IN TXT "dot-answer"`)
m.Answer = append(m.Answer, rr)
_ = w.WriteMsg(m)
})}
go func() { _ = server.ActivateAndServe() }()
t.Cleanup(func() { _ = server.Shutdown() })
options := dohOptions(t, nil)
client, err := Get(options, &Configuration{Retries: 2, Resolvers: []string{"dot:" + l.Addr().String()}})
require.NoError(t, err)
resp, err := client.Do(txtQuery())
require.NoError(t, err)
require.Len(t, resp.Answer, 1)
require.Contains(t, resp.Answer[0].String(), "dot-answer")
}
+1 -1
View File
@@ -195,7 +195,7 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad
// perform trace if necessary (excluded from duration — only the query RTT is measured)
var traceData *retryabledns.TraceData
if request.Trace {
traceData, err = trace(request.options.Options.ExecutionId, request.dnsClient, domain, request.question, request.TraceMaxRecursion)
traceData, err = request.dnsClient.Trace(domain, request.question, request.TraceMaxRecursion)
if err != nil {
request.options.Output.Request(request.options.TemplatePath, domain, "dns", err)
}
-119
View File
@@ -1,119 +0,0 @@
package dns
import (
"context"
"net"
"sync/atomic"
"testing"
"time"
"github.com/miekg/dns"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
)
// startDNSServer answers every query with a TXT record, or with the A records
// in addrs when the question is for name, and counts the queries.
func startDNSServer(t *testing.T, name string, addrs []string, hits *atomic.Int32) string {
t.Helper()
pc, err := net.ListenPacket("udp", "127.0.0.1:0")
require.NoError(t, err)
server := &dns.Server{PacketConn: pc, Handler: dns.HandlerFunc(func(w dns.ResponseWriter, q *dns.Msg) {
hits.Add(1)
m := new(dns.Msg)
m.SetReply(q)
question := q.Question[0]
switch {
case question.Name == name && question.Qtype == dns.TypeA:
for _, addr := range addrs {
rr, _ := dns.NewRR(name + " 0 IN A " + addr)
m.Answer = append(m.Answer, rr)
}
case question.Name == name:
default:
rr, _ := dns.NewRR(question.Name + ` 0 IN TXT "answer"`)
m.Answer = append(m.Answer, rr)
}
_ = w.WriteMsg(m)
})}
go func() { _ = server.ActivateAndServe() }()
t.Cleanup(func() { _ = server.Shutdown() })
l, err := net.Listen("tcp", pc.LocalAddr().String())
require.NoError(t, err)
tcpServer := &dns.Server{Listener: l, Handler: server.Handler}
go func() { _ = tcpServer.ActivateAndServe() }()
t.Cleanup(func() { _ = tcpServer.Shutdown() })
return pc.LocalAddr().String()
}
// useSystemResolver sends the lookups of the process to the DNS server at address.
func useSystemResolver(t *testing.T, address string) {
t.Helper()
previous := net.DefaultResolver
net.DefaultResolver = &net.Resolver{
PreferGo: true,
Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, "udp", address)
},
}
t.Cleanup(func() { net.DefaultResolver = previous })
}
func TestDNSResolverNameObeysNetworkPolicy(t *testing.T) {
// The resolver name has an allowed address and a denied address. The
// policy check passes because one address is allowed, so the query must
// not go to the denied one.
var deniedHits, systemHits atomic.Int32
denied := startDNSServer(t, "", nil, &deniedHits)
_, port, err := net.SplitHostPort(denied)
require.NoError(t, err)
system := startDNSServer(t, "resolver.nuclei-test.", []string{"127.0.0.1", "127.0.0.2"}, &systemHits)
useSystemResolver(t, system)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ExcludeTargets = []string{"127.0.0.1"}
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: "dns-resolver-policy",
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}, Name: "test"},
})
for _, resolver := range []string{"resolver.nuclei-test:" + port, "tcp:resolver.nuclei-test:" + port} {
t.Run(resolver, func(t *testing.T) {
deniedHits.Store(0)
request := &Request{
RequestType: DNSRequestTypeHolder{DNSRequestType: TXT},
Class: "INET",
Retries: 2,
Name: "{{FQDN}}",
Resolvers: []string{resolver},
}
require.NoError(t, request.Compile(executerOpts))
msg := new(dns.Msg)
msg.SetQuestion("example.com.", dns.TypeTXT)
_, _ = request.dnsClient.Do(msg)
require.Zero(t, deniedHits.Load(), "dns query reached a denied address")
})
}
}
func TestDNSResolverNameLookupStopsAtTimeout(t *testing.T) {
// The system resolver never answers, so the lookup that pins the resolver
// name must stop at the timeout.
silent, err := net.ListenPacket("udp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = silent.Close() })
useSystemResolver(t, silent.LocalAddr().String())
start := time.Now()
_, err = pinResolverHost(t.Name(), "silent.nuclei-test:53", "silent.nuclei-test", time.Second)
require.Error(t, err)
require.Less(t, time.Since(start), 4*time.Second, "the resolver lookup did not stop at the timeout")
}
-88
View File
@@ -1,88 +0,0 @@
package dns
import (
"context"
"math/rand"
"net"
"github.com/miekg/dns"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool"
"github.com/projectdiscovery/retryabledns"
sliceutil "github.com/projectdiscovery/utils/slice"
)
// traceNameServerPort is the port of the name servers that a trace queries.
var traceNameServerPort = "53"
// trace follows the delegation of host from the root servers in the same way
// as retryabledns.Client.Trace, but it skips name server addresses that the
// network policy denies. The name servers come from the answers, so the owner
// of a zone could otherwise send the queries of a trace to internal addresses.
func trace(executionID string, client *dnsclientpool.Client, host string, requestType uint16, maxRecursion int) (*retryabledns.TraceData, error) {
var traceData retryabledns.TraceData
host = dns.CanonicalName(host)
servers := retryabledns.RootDNSServersIPv4
seenNS := make(map[string]struct{})
seenCName := make(map[string]int)
for i := 1; i < maxRecursion; i++ {
responses, err := client.QueryParallel(host, requestType, servers)
if err != nil {
return nil, err
}
for _, server := range servers {
seenNS[server] = struct{}{}
}
if len(responses) == 0 {
return &traceData, nil
}
for _, dnsData := range responses {
if dnsData != nil && len(dnsData.Resolver) > 0 {
traceData.DNSData = append(traceData.DNSData, dnsData)
}
}
var nameServers []string
var nextCname string
for _, dnsData := range responses {
if dnsData == nil {
continue
}
for _, ns := range dnsData.NS {
ips, err := net.DefaultResolver.LookupIP(context.Background(), "ip4", ns)
if err != nil {
continue
}
for _, ip := range ips {
if !protocolstate.IsHostAllowed(executionID, ip.String()) {
continue
}
nameServers = append(nameServers, net.JoinHostPort(ip.String(), traceNameServerPort))
}
}
if nextCname == "" && len(dnsData.CNAME) > 0 {
nextCname = dnsData.CNAME[0]
}
}
nameServers = sliceutil.Dedupe(nameServers)
if len(nameServers) == 0 {
break
}
server := nameServers[rand.Intn(len(nameServers))] // #nosec G404
if _, ok := seenNS[server]; ok && nextCname == "" {
break
}
servers = []string{server}
if nextCname != "" {
seenCName[nextCname]++
if seenCName[nextCname] > retryabledns.DefaultMaxPerCNAMEFollows {
break
}
host = nextCname
}
}
return &traceData, nil
}
-124
View File
@@ -1,124 +0,0 @@
package dns
import (
"context"
"net"
"sync/atomic"
"testing"
"github.com/miekg/dns"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/retryabledns"
)
func serveTraceDNS(t *testing.T, address string, handler dns.HandlerFunc) string {
t.Helper()
pc, err := net.ListenPacket("udp", address)
require.NoError(t, err)
server := &dns.Server{PacketConn: pc, Handler: handler}
go func() { _ = server.ActivateAndServe() }()
t.Cleanup(func() { _ = server.Shutdown() })
return pc.LocalAddr().String()
}
func TestDNSTraceObeysNetworkPolicy(t *testing.T) {
// The zone of the traced name delegates to a name server on 127.0.0.1.
// The trace must query it when the policy allows the address, so that the
// test can see a trace that ignores the policy, and must skip it when the
// policy denies the address.
t.Run("allowed", func(t *testing.T) {
require.NotZero(t, runTraceAgainstLoopbackNameServer(t, nil), "trace did not query the allowed name server")
})
t.Run("denied", func(t *testing.T) {
require.Zero(t, runTraceAgainstLoopbackNameServer(t, []string{"127.0.0.1"}), "trace queried a denied name server")
})
}
func runTraceAgainstLoopbackNameServer(t *testing.T, excludeTargets []string) int32 {
t.Helper()
var internalHits atomic.Int32
internal := serveTraceDNS(t, "127.0.0.1:0", func(w dns.ResponseWriter, q *dns.Msg) {
internalHits.Add(1)
m := new(dns.Msg)
m.SetReply(q)
rr, _ := dns.NewRR(q.Question[0].Name + ` 0 IN TXT "internal"`)
m.Answer = append(m.Answer, rr)
_ = w.WriteMsg(m)
})
_, port, err := net.SplitHostPort(internal)
require.NoError(t, err)
previousPort := traceNameServerPort
traceNameServerPort = port
t.Cleanup(func() { traceNameServerPort = previousPort })
// public stands in for the root servers, the scan resolvers and the system resolver
public := serveTraceDNS(t, "127.0.0.1:0", func(w dns.ResponseWriter, q *dns.Msg) {
m := new(dns.Msg)
m.SetReply(q)
question := q.Question[0]
var rr dns.RR
switch {
case question.Name == "ns.nuclei-test." && question.Qtype == dns.TypeA:
rr, _ = dns.NewRR("ns.nuclei-test. 0 IN A 127.0.0.1")
case question.Name == "ns.nuclei-test.":
case question.Name == "trace.nuclei-test.":
rr, _ = dns.NewRR("trace.nuclei-test. 0 IN NS ns.nuclei-test.")
default:
rr, _ = dns.NewRR(question.Name + ` 0 IN TXT "public"`)
}
if rr != nil {
m.Answer = append(m.Answer, rr)
}
_ = w.WriteMsg(m)
})
previousRoots := retryabledns.RootDNSServersIPv4
retryabledns.RootDNSServersIPv4 = []string{public}
t.Cleanup(func() { retryabledns.RootDNSServersIPv4 = previousRoots })
previousResolver := net.DefaultResolver
net.DefaultResolver = &net.Resolver{
PreferGo: true,
Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, "udp", public)
},
}
t.Cleanup(func() { net.DefaultResolver = previousResolver })
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ExcludeTargets = excludeTargets
options.InternalResolversList = []string{public}
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: "dns-trace-policy",
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Info}, Name: "test"},
})
request := &Request{
RequestType: DNSRequestTypeHolder{DNSRequestType: TXT},
Class: "INET",
Name: "{{FQDN}}",
// One retry keeps the client keyed by this test's resolvers, so a
// client cached by an earlier run cannot point at a closed server.
Retries: 1,
Trace: true,
TraceMaxRecursion: 5,
}
require.NoError(t, request.Compile(executerOpts))
var event output.InternalEvent
input := contextargs.NewWithInput(context.Background(), "trace.nuclei-test")
err = request.ExecuteWithResults(input, nil, nil, func(e *output.InternalWrappedEvent) {
event = e.InternalEvent
})
require.NoError(t, err)
require.Contains(t, event["trace"], "ns.nuclei-test", "trace did not query the root servers")
return internalHits.Load()
}
+3 -3
View File
@@ -7,10 +7,10 @@ import (
"strings"
"sync"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher"
"github.com/go-rod/rod/lib/launcher/flags"
"github.com/pkg/errors"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/launcher"
"github.com/projectdiscovery/rod/lib/launcher/flags"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
+2 -2
View File
@@ -5,8 +5,8 @@ package engine
import (
"encoding/base64"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/proto"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto"
)
// NewHijack create hijack from page.
+2 -2
View File
@@ -5,11 +5,11 @@ import (
"errors"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/utils"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/utils"
)
// Instance is an isolated browser instance opened for doing operations with it.
@@ -5,9 +5,9 @@ import (
"testing"
"time"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/cdp"
"github.com/projectdiscovery/rod/lib/proto"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/cdp"
"github.com/go-rod/rod/lib/proto"
"github.com/stretchr/testify/require"
)
+2 -2
View File
@@ -11,6 +11,8 @@ import (
"sync"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
@@ -19,8 +21,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/proto"
"github.com/projectdiscovery/utils/errkit"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -11,6 +11,10 @@ import (
"sync"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/input"
"github.com/go-rod/rod/lib/proto"
"github.com/go-rod/rod/lib/utils"
"github.com/kitabisa/go-ci"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
@@ -20,10 +24,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
filepathutil "github.com/projectdiscovery/nuclei/v3/pkg/utils/filepath"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/input"
"github.com/projectdiscovery/rod/lib/proto"
"github.com/projectdiscovery/rod/lib/utils"
contextutil "github.com/projectdiscovery/utils/context"
"github.com/projectdiscovery/utils/errkit"
fileutil "github.com/projectdiscovery/utils/file"
+3 -3
View File
@@ -11,12 +11,12 @@ import (
"testing"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/cdp"
"github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/cdp"
"github.com/projectdiscovery/rod/lib/proto"
"github.com/stretchr/testify/require"
)
+3 -3
View File
@@ -10,12 +10,12 @@ import (
"testing"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher"
"github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/networkpolicy"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/launcher"
"github.com/projectdiscovery/rod/lib/proto"
"github.com/stretchr/testify/require"
)
+2 -2
View File
@@ -6,10 +6,10 @@ import (
"net/http/httputil"
"strings"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/rod"
"github.com/projectdiscovery/rod/lib/proto"
)
// routingRuleHandler handles proxy rule for actions related to request/response modification
@@ -6,7 +6,7 @@ import (
"runtime"
"testing"
"github.com/projectdiscovery/rod/lib/launcher/flags"
"github.com/go-rod/rod/lib/launcher/flags"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
-103
View File
@@ -1,103 +0,0 @@
package http
import (
"context"
"sync"
"sync/atomic"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
)
// bodySizeEstimate is the largest request body a template has built so far.
type bodySizeEstimate struct {
known atomic.Bool
max atomic.Int64
learning sync.Mutex
}
type bodyReservation struct {
budget *httpbody.Budget
size int64
release func()
unlock func()
}
func noop() {}
// reserveBody reserves the template's largest body size before the body is
// built. Until a size is known, builds run one at a time.
func (request *Request) reserveBody(ctx context.Context) (*bodyReservation, error) {
r := &bodyReservation{budget: request.options.RequestBodyBudget, release: noop, unlock: noop}
estimate := request.bodySize
if r.budget == nil || estimate == nil {
return r, nil
}
if !estimate.known.Load() {
estimate.learning.Lock()
if !estimate.known.Load() {
r.unlock = estimate.learning.Unlock
return r, nil
}
estimate.learning.Unlock()
}
r.size = estimate.max.Load()
release, err := r.budget.Acquire(ctx, r.size)
if err != nil {
return nil, err
}
r.release = release
return r, nil
}
// makeReserved builds the next request inside a body reservation, waiting
// for budget until waitCtx ends. On success the caller owns the reservation
// and calls done once the send ends.
func (request *Request) makeReserved(waitCtx context.Context, generator *requestGenerator, input *contextargs.Context, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, *bodyReservation, error) {
reservation, err := request.reserveBody(waitCtx)
if err != nil {
return nil, nil, err
}
generated, err := generator.Make(input.Context(), input, data, payloads, dynamicValues)
if err == nil {
err = reservation.built(waitCtx, request, generated.bodyLen())
}
if err != nil {
reservation.done()
return nil, nil, err
}
generated.bodyReserved = true
return generated, reservation, nil
}
// built grows the reservation to the body's real size. It releases before
// acquiring again: waiting while holding part of the budget can deadlock.
func (r *bodyReservation) built(ctx context.Context, request *Request, size int64) error {
if estimate := request.bodySize; estimate != nil {
for {
seen := estimate.max.Load()
if size <= seen || estimate.max.CompareAndSwap(seen, size) {
break
}
}
estimate.known.Store(true)
}
r.unlock()
r.unlock = noop
if size <= r.size {
return nil
}
r.release()
r.release = noop
release, err := r.budget.Acquire(ctx, size)
if err != nil {
return err
}
r.release, r.size = release, size
return nil
}
func (r *bodyReservation) done() {
r.unlock()
r.release()
}
@@ -1,125 +0,0 @@
//go:build !race
package http
import (
"bufio"
"context"
"fmt"
"io"
"net"
"net/http"
"runtime"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
)
// Threads and pipeline build bodies ahead of the send. Built bodies must stay
// within the budget while the server holds the first request.
func TestRequestBodyBudgetBoundsBodiesBuiltAhead(t *testing.T) {
const bodySize = 8 << 20
const payloads = 12
for _, mode := range []string{"threads", "pipeline"} {
t.Run(mode, func(t *testing.T) {
if mode == "pipeline" {
t.Skip("needs projectdiscovery/rawhttp#551: the pipeline client hangs after 10s idle")
}
arrived := make(chan struct{}, payloads)
unblock := make(chan struct{})
var received atomic.Int32
// net/http drops pipelined connections with large bodies, so serve
// Content-Length framed requests in order by hand.
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer func() { _ = listener.Close() }()
go func() {
for {
conn, err := listener.Accept()
if err != nil {
return
}
go func() {
defer func() { _ = conn.Close() }()
reader := bufio.NewReader(conn)
for {
req, err := http.ReadRequest(reader)
if err != nil {
return
}
_, _ = io.Copy(io.Discard, req.Body)
received.Add(1)
arrived <- struct{}{}
<-unblock
if _, err := io.WriteString(conn, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"); err != nil {
return
}
}
}()
}
}()
target := "http://" + listener.Addr().String()
options := testutils.DefaultOptions.Copy()
testutils.Init(options)
values := make([]string, payloads)
for i := range values {
values[i] = fmt.Sprint(i)
}
request := &Request{
ID: "body-budget-" + mode,
Payloads: map[string]interface{}{"v": values},
}
body := `{{repeat("A", 8388608)}}`
if mode == "threads" {
request.Threads = 8
request.Path = []string{"{{BaseURL}}/?v={{v}}"}
request.Method = HTTPMethodTypeHolder{MethodType: HTTPPost}
request.Body = body
} else {
request.Pipeline = true
request.PipelineConcurrentConnections = 1
request.PipelineRequestsPerConnection = payloads
request.Raw = []string{"POST /?v={{v}} HTTP/1.1\nHost: {{Hostname}}\nContent-Length: 8388608\n\n" + body}
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOpts.RequestBodyBudget = httpbody.NewBudget(bodySize)
require.NoError(t, request.Compile(executerOpts))
var before runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&before)
done := make(chan error, 1)
go func() {
done <- request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), target), make(output.InternalEvent), make(output.InternalEvent), func(*output.InternalWrappedEvent) {})
}()
<-arrived
time.Sleep(300 * time.Millisecond)
var held runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&held)
close(unblock)
require.NoError(t, <-done)
grown := int64(held.HeapAlloc) - int64(before.HeapAlloc)
t.Logf("heap held: %d MB", grown>>20)
require.Less(t, grown, int64(4*bodySize), "bodies built ahead of the send must stay within the budget")
require.EqualValues(t, payloads, received.Load())
})
}
}
-200
View File
@@ -1,200 +0,0 @@
package http
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
)
// The third request queues ~2.4s, so it only beats the 3s timeout if the
// timeout starts after admission.
func TestRequestBodyBudgetQueuesHeavyRequests(t *testing.T) {
const bodySize = 1 << 20
const requests = 3
var inFlight, maxInFlight atomic.Int32
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := inFlight.Add(1)
defer inFlight.Add(-1)
for {
seen := maxInFlight.Load()
if n <= seen || maxInFlight.CompareAndSwap(seen, n) {
break
}
}
body, _ := io.ReadAll(r.Body)
time.Sleep(1200 * time.Millisecond)
if len(body) == bodySize {
_, _ = io.WriteString(w, "full body")
}
}))
defer ts.Close()
options := testutils.DefaultOptions.Copy()
options.SetTimeouts(&types.Timeouts{DialTimeout: time.Second, HttpTimeout: 3 * time.Second})
testutils.Init(options)
request := &Request{
ID: "body-budget",
Path: []string{"{{BaseURL}}"},
Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
Body: `{{repeat("A", 1048576)}}`,
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Part: "body",
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"full body"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOpts.RequestBodyBudget = httpbody.NewBudget(bodySize)
require.NoError(t, request.Compile(executerOpts))
var matched atomic.Int32
var wg sync.WaitGroup
for range requests {
wg.Go(func() {
err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), ts.URL), make(output.InternalEvent), make(output.InternalEvent), func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matched.Add(1)
}
})
require.NoError(t, err)
})
}
wg.Wait()
require.EqualValues(t, 1, maxInFlight.Load(), "budget must hold heavy requests to one in flight")
require.EqualValues(t, requests, matched.Load(), "queued requests must not time out while waiting")
}
func TestReserveBodyQueuesBuildsOnLearnedSize(t *testing.T) {
const bodySize = 4 << 20
request := &Request{
options: &protocols.ExecutorOptions{RequestBodyBudget: httpbody.NewBudget(6 << 20)},
bodySize: &bodySizeEstimate{},
}
first, err := request.reserveBody(t.Context())
require.NoError(t, err)
blocked := make(chan struct{})
go func() {
second, err := request.reserveBody(t.Context())
require.NoError(t, err)
second.done()
close(blocked)
}()
select {
case <-blocked:
t.Fatal("second build started before the first recorded its size")
case <-time.After(20 * time.Millisecond):
}
require.NoError(t, first.built(t.Context(), request, bodySize))
first.done()
<-blocked
held, err := request.reserveBody(t.Context())
require.NoError(t, err)
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
defer cancel()
_, err = request.reserveBody(ctx)
require.ErrorIs(t, err, context.DeadlineExceeded)
held.done()
next, err := request.reserveBody(t.Context())
require.NoError(t, err)
next.done()
}
func TestReserveBodyGrowsToLargerBody(t *testing.T) {
request := &Request{
options: &protocols.ExecutorOptions{RequestBodyBudget: httpbody.NewBudget(6 << 20)},
bodySize: &bodySizeEstimate{},
}
learn, err := request.reserveBody(t.Context())
require.NoError(t, err)
require.NoError(t, learn.built(t.Context(), request, 1<<20))
learn.done()
grown, err := request.reserveBody(t.Context())
require.NoError(t, err)
require.NoError(t, grown.built(t.Context(), request, 5<<20))
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
defer cancel()
_, err = request.reserveBody(ctx)
require.ErrorIs(t, err, context.DeadlineExceeded)
grown.done()
}
// Stop-at-first-match ends the workers with tasks still queued. Their
// reservations must go back to the shared budget.
func TestRequestBodyBudgetStopAtFirstMatchReleasesQueued(t *testing.T) {
const budget = 16 << 20
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
time.Sleep(100 * time.Millisecond)
_, _ = io.WriteString(w, "match")
}))
defer ts.Close()
options := testutils.DefaultOptions.Copy()
testutils.Init(options)
values := make([]string, 64)
for i := range values {
values[i] = fmt.Sprint(i)
}
request := &Request{
ID: "body-budget-stop-first",
Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
Path: []string{"{{BaseURL}}/?v={{v}}"},
Body: `{{repeat("A", 1048576)}}`,
Threads: 4,
Payloads: map[string]interface{}{"v": values},
StopAtFirstMatch: true,
Operators: operators.Operators{Matchers: []*matchers.Matcher{{
Part: "body",
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"match"},
}}},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOpts.RequestBodyBudget = httpbody.NewBudget(budget)
require.NoError(t, request.Compile(executerOpts))
err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), ts.URL), make(output.InternalEvent), make(output.InternalEvent), func(*output.InternalWrappedEvent) {})
require.NoError(t, err)
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
defer cancel()
release, err := executerOpts.RequestBodyBudget.Acquire(ctx, budget)
require.NoError(t, err, "every reservation must be released")
release()
}
-3
View File
@@ -82,9 +82,6 @@ type generatedRequest struct {
dynamicValues map[string]interface{}
interactshURLs []string
customCancelFunction context.CancelFunc
// sendTimeout starts after the body budget admits the request.
sendTimeout time.Duration
bodyReserved bool
// requestURLPattern tracks unmodified request url pattern without values ( it is used for constant vuln_hash)
// ex: {{BaseURL}}/api/exp?param={{randstr}}
requestURLPattern string
-20
View File
@@ -147,10 +147,6 @@ type Request struct {
rawhttpClient *rawhttp.Client
replayClient *http.Client // set when -replay-proxy is given
dialer *fastdialer.Dialer
bodySize *bodySizeEstimate
// sharesResponse lets other templates reuse a response this request gets,
// and this request reuse theirs (see httpclientpool.WithSharedResponse)
sharesResponse bool
// description: |
// SelfContained specifies if the request is self-contained.
@@ -420,7 +416,6 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
return errors.Wrap(err, "could not get dialer")
}
request.dialer = dialer
request.bodySize = &bodySizeEstimate{}
request.options = options
for _, option := range request.options.Options.CustomHeaders {
@@ -590,24 +585,9 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
if shouldDisableKeepAliveForHTTPProxy(request, options) {
request.connConfiguration.Connection.DisableKeepAlive = true
}
request.sharesResponse = canShareResponse(request, options.Flow)
return nil
}
// canShareResponse reports whether a response another template got for the
// same request is as good as sending it again. A shared response arrives at
// once, so nothing that measures time may take one: race requests, timing
// analyzers, and a request whose operators or flow read the duration. The
// operators are searched whole, so a new field that can read it, as LLM
// matcher inputs do, is covered without being listed.
func canShareResponse(request *Request, flow string) bool {
if request.Race || request.Analyzer != nil || strings.Contains(flow, "duration") {
return false
}
operators, err := json.Marshal(request.Operators)
return err == nil && !bytes.Contains(operators, []byte("duration"))
}
// RebuildGenerator rebuilds the generator for the request
func (request *Request) RebuildGenerator() error {
generator, err := generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options)
@@ -514,11 +514,7 @@ func wrappedGet(options *types.Options, configuration *Configuration, host strin
}
}
tracking := &connTrackingTransport{base: transport}
if shared := getSharedResponses(options); shared != nil {
return &sharingTransport{base: tracking, shared: shared}, nil
}
return tracking, nil
return &connTrackingTransport{base: transport}, nil
}
redirectFlow := configuration.RedirectFlow
maxRedirects := configuration.MaxRedirects
@@ -175,9 +175,7 @@ func TestIdleConnPoolSizeIsWiredToTransport(t *testing.T) {
client, err := Get(opts, &Configuration{}, "example.com")
require.NoError(t, err)
sharing, ok := client.HTTPClient.Transport.(*sharingTransport)
require.True(t, ok, "expected the shared-response transport wrapper")
tracking, ok := sharing.base.(*connTrackingTransport)
tracking, ok := client.HTTPClient.Transport.(*connTrackingTransport)
require.True(t, ok, "expected the connection-tracking transport wrapper")
transport, ok := tracking.base.(*http.Transport)
require.True(t, ok, "expected an *http.Transport underneath")
@@ -1,281 +0,0 @@
package httpclientpool
import (
"bytes"
"container/list"
"context"
"io"
"maps"
"net/http"
"slices"
"strings"
"sync"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/useragent"
)
// Templates send many requests another template already sent to the same
// target, byte for byte: clustering merges only single-request templates. A
// request marked with WithSharedResponse is sent once per scan and every
// other template sending the same bytes gets a copy of the response. The key
// is the request as it goes on the wire, cookies included, and the client
// still processes each copy, so redirects and cookies behave as if it was
// sent again.
const (
// sharedResponseMaxBody keeps large downloads out of the store; they are
// sent each time
sharedResponseMaxBody = 1 << 20
sharedResponsesMaxBytes = 64 << 20
)
type sharedResponseKey struct{}
// WithSharedResponse marks the request of templateID as one whose response
// other templates may reuse. A template sending the same request again gets
// it sent again: a repeat within one template is intended.
func WithSharedResponse(ctx context.Context, templateID string) context.Context {
return context.WithValue(ctx, sharedResponseKey{}, templateID)
}
type sharedResponses struct {
mu sync.Mutex
entries map[string]*sharedEntry
// order holds stored entries oldest first, for eviction
order *list.List
bytes int
maxBytes int
}
type sharedEntry struct {
key string
templateID string
ready chan struct{}
response *storedResponse
}
type storedResponse struct {
response *http.Response
body []byte
}
func getSharedResponses(options *types.Options) *sharedResponses {
dialers := protocolstate.GetDialersWithId(options.ExecutionId)
if dialers == nil {
return nil
}
dialers.Lock()
defer dialers.Unlock()
if dialers.SharedResponses == nil {
dialers.SharedResponses = &sharedResponses{entries: make(map[string]*sharedEntry), order: list.New(), maxBytes: sharedResponsesMaxBytes}
}
shared, _ := dialers.SharedResponses.(*sharedResponses)
return shared
}
// ResetSharedResponses forgets the responses of earlier scans, so a scan
// that reuses the execution, as an SDK engine does, sends its requests again.
func ResetSharedResponses(options *types.Options) {
shared := getSharedResponses(options)
if shared == nil {
return
}
shared.mu.Lock()
defer shared.mu.Unlock()
clear(shared.entries)
shared.order.Init()
shared.bytes = 0
}
type sharingTransport struct {
base http.RoundTripper
shared *sharedResponses
}
func (t *sharingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
templateID, marked := req.Context().Value(sharedResponseKey{}).(string)
if !marked || (req.Method != http.MethodGet && req.Method != http.MethodHead) || hasBody(req) {
return t.base.RoundTrip(req)
}
key := sharedRequestKey(req)
t.shared.mu.Lock()
entry, found := t.shared.entries[key]
if !found {
entry = &sharedEntry{key: key, templateID: templateID, ready: make(chan struct{})}
t.shared.entries[key] = entry
t.shared.mu.Unlock()
return t.send(req, entry)
}
t.shared.mu.Unlock()
if entry.templateID == templateID {
return t.base.RoundTrip(req)
}
select {
case <-entry.ready:
case <-req.Context().Done():
return nil, req.Context().Err()
}
if entry.response == nil {
// the first send failed or was too large to keep
return t.base.RoundTrip(req)
}
return entry.response.copyFor(req), nil
}
func (t *sharingTransport) CloseIdleConnections() {
type closeIdler interface{ CloseIdleConnections() }
if ci, ok := t.base.(closeIdler); ok {
ci.CloseIdleConnections()
}
}
// send sends the first request of key and stores its response for the rest.
func (t *sharingTransport) send(req *http.Request, entry *sharedEntry) (*http.Response, error) {
defer close(entry.ready)
resp, err := t.base.RoundTrip(req)
if err != nil {
t.forget(entry)
return resp, err
}
body, err := io.ReadAll(io.LimitReader(resp.Body, sharedResponseMaxBody+1))
if err != nil || len(body) > sharedResponseMaxBody || !shareableStatus(resp.StatusCode) {
t.forget(entry)
// The bytes already taken off the body have to go back in front of it.
// A short read is still this caller's response: a Content-Length that
// is longer than the body is what some matchers look for. It is not
// stored, and neither is a body too large to keep or a refusal.
resp.Body = struct {
io.Reader
io.Closer
}{io.MultiReader(bytes.NewReader(body), resp.Body), resp.Body}
return resp, nil
}
_ = resp.Body.Close()
entry.response = &storedResponse{response: resp, body: body}
t.store(entry)
return entry.response.copyFor(req), nil
}
// shareableStatus reports whether code is an answer a later template can
// reuse. 404 and 500 are answers templates match on. 429 and the gateway
// errors mean the server did not answer, so the next template must send.
func shareableStatus(code int) bool {
switch code {
case http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
return false
default:
return true
}
}
// hasBody reports whether req sends a body. retryablehttp gives every
// request one, empty for a GET, so its content is what counts.
func hasBody(req *http.Request) bool {
if req.Body == nil || req.Body == http.NoBody {
return false
}
if req.ContentLength > 0 || req.GetBody == nil {
return true
}
body, err := req.GetBody()
if err != nil {
return true
}
defer func() { _ = body.Close() }()
var first [1]byte
n, _ := body.Read(first[:])
return n > 0
}
func (t *sharingTransport) forget(entry *sharedEntry) {
t.shared.mu.Lock()
defer t.shared.mu.Unlock()
if t.shared.entries[entry.key] == entry {
delete(t.shared.entries, entry.key)
}
}
func (t *sharingTransport) store(entry *sharedEntry) {
t.shared.mu.Lock()
defer t.shared.mu.Unlock()
// a reset while it was in flight dropped it
if t.shared.entries[entry.key] != entry {
return
}
t.shared.order.PushBack(entry)
t.shared.bytes += entry.size()
for t.shared.bytes > t.shared.maxBytes && t.shared.order.Len() > 1 {
oldest := t.shared.order.Remove(t.shared.order.Front()).(*sharedEntry)
t.shared.bytes -= oldest.size()
if t.shared.entries[oldest.key] == oldest {
delete(t.shared.entries, oldest.key)
}
}
}
// sharedEntryOverhead stands for the response struct and bookkeeping, so
// bodyless responses, as every HEAD gets, still count toward the cap.
const sharedEntryOverhead = 512
func (e *sharedEntry) size() int {
size := sharedEntryOverhead + len(e.key) + len(e.response.body)
for name, values := range e.response.response.Header {
size += len(name)
for _, value := range values {
size += len(value)
}
}
return size
}
func (s *storedResponse) copyFor(req *http.Request) *http.Response {
resp := *s.response
resp.Header = s.response.Header.Clone()
resp.Trailer = s.response.Trailer.Clone()
resp.Body = io.NopCloser(bytes.NewReader(s.body))
// A HEAD response has an empty body and a Content-Length for the body it
// did not send. Matchers read that length, so a reused HEAD keeps it.
if req.Method != http.MethodHead {
resp.ContentLength = int64(len(s.body))
}
resp.Request = req
return &resp
}
// randomUserAgents are the values nuclei picks a User-Agent from for every
// request a template does not set one for. Two templates sending the same
// request almost never pick the same one, and either could have picked the
// other's, so a picked User-Agent is left out of the key.
var randomUserAgents = sync.OnceValue(func() map[string]struct{} {
values := make(map[string]struct{}, len(useragent.UserAgents))
for _, userAgent := range useragent.UserAgents {
values[userAgent.Raw] = struct{}{}
}
return values
})
// sharedRequestKey is the request as the server receives it, apart from the
// order of headers and a picked User-Agent.
func sharedRequestKey(req *http.Request) string {
var key strings.Builder
key.WriteString(req.Method)
key.WriteByte(' ')
key.WriteString(req.URL.String())
key.WriteByte('\n')
key.WriteString(req.Host)
for _, name := range slices.Sorted(maps.Keys(req.Header)) {
values := req.Header[name]
if _, picked := randomUserAgents()[req.Header.Get("User-Agent")]; name == "User-Agent" && len(values) == 1 && picked {
continue
}
for _, value := range values {
key.WriteByte('\n')
key.WriteString(name)
key.WriteString(": ")
key.WriteString(value)
}
}
return key.String()
}
@@ -1,308 +0,0 @@
package httpclientpool
import (
"container/list"
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/useragent"
"github.com/stretchr/testify/require"
)
func newSharingTransport(base http.RoundTripper) *sharingTransport {
return &sharingTransport{base: base, shared: &sharedResponses{entries: make(map[string]*sharedEntry), order: list.New(), maxBytes: sharedResponsesMaxBytes}}
}
func sharedGet(t *testing.T, transport http.RoundTripper, method, url, templateID string, header http.Header) string {
t.Helper()
ctx := context.Background()
if templateID != "" {
ctx = WithSharedResponse(ctx, templateID)
}
req, err := http.NewRequestWithContext(ctx, method, url, nil)
require.NoError(t, err)
for name, values := range header {
req.Header[name] = values
}
resp, err := transport.RoundTrip(req)
require.NoError(t, err)
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
return string(body)
}
func TestSharedResponsesSendEachRequestOncePerScan(t *testing.T) {
var hits atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := hits.Add(1)
_, _ = io.WriteString(w, r.URL.Path+strings.Repeat("!", int(n)))
}))
defer server.Close()
transport := newSharingTransport(http.DefaultTransport)
first := sharedGet(t, transport, http.MethodGet, server.URL+"/a", "one", nil)
require.Equal(t, first, sharedGet(t, transport, http.MethodGet, server.URL+"/a", "two", nil), "another template gets the stored response")
require.EqualValues(t, 1, hits.Load())
// nuclei builds a GET with an empty body
empty, err := retryablehttp.NewRequestWithContext(WithSharedResponse(context.Background(), "four"), http.MethodGet, server.URL+"/a", strings.NewReader(""))
require.NoError(t, err)
resp, err := transport.RoundTrip(empty.Request)
require.NoError(t, err)
_ = resp.Body.Close()
require.EqualValues(t, 1, hits.Load(), "an empty body is no body")
sharedGet(t, transport, http.MethodGet, server.URL+"/a", "one", nil)
require.EqualValues(t, 2, hits.Load(), "a template repeating its own request sends it")
sharedGet(t, transport, http.MethodGet, server.URL+"/a", "", nil)
require.EqualValues(t, 3, hits.Load(), "an unmarked request is always sent")
sharedGet(t, transport, http.MethodGet, server.URL+"/a", "three", http.Header{"Cookie": {"session=1"}})
require.EqualValues(t, 4, hits.Load(), "other cookies make another request")
picked := func(i int) http.Header { return http.Header{"User-Agent": {useragent.UserAgents[i].Raw}} }
sharedGet(t, transport, http.MethodGet, server.URL+"/b", "one", picked(0))
sharedGet(t, transport, http.MethodGet, server.URL+"/b", "two", picked(1))
require.EqualValues(t, 5, hits.Load(), "a User-Agent nuclei picked at random is not part of the request")
sharedGet(t, transport, http.MethodGet, server.URL+"/b", "three", http.Header{"User-Agent": {"custom-scanner"}})
require.EqualValues(t, 6, hits.Load(), "one a template sets is")
for range 2 {
// A new request each time. Clone shares the body, so a second clone
// of one request can send ContentLength with an already-read body.
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), "one"), http.MethodPost, server.URL+"/a", strings.NewReader("x"))
require.NoError(t, err)
resp, err := transport.RoundTrip(req)
require.NoError(t, err)
_ = resp.Body.Close()
}
require.EqualValues(t, 8, hits.Load(), "a request with a body is always sent")
}
func TestSharedResponsesWaitForTheRequestInFlight(t *testing.T) {
var hits atomic.Int32
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
<-release
_, _ = io.WriteString(w, "slow")
}))
defer server.Close()
transport := newSharingTransport(http.DefaultTransport)
var wg sync.WaitGroup
bodies := make([]string, 5)
for i := range bodies {
wg.Add(1)
go func() {
defer wg.Done()
bodies[i] = sharedGet(t, transport, http.MethodGet, server.URL, "template-"+string(rune('a'+i)), nil)
}()
}
require.Eventually(t, func() bool { return hits.Load() > 0 }, 5*time.Second, time.Millisecond)
close(release)
wg.Wait()
require.EqualValues(t, 1, hits.Load())
require.Equal(t, []string{"slow", "slow", "slow", "slow", "slow"}, bodies)
}
func TestSharedResponsesSendLargeBodiesEachTime(t *testing.T) {
var hits atomic.Int32
large := strings.Repeat("x", sharedResponseMaxBody+10)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
_, _ = io.WriteString(w, large)
}))
defer server.Close()
transport := newSharingTransport(http.DefaultTransport)
require.Equal(t, large, sharedGet(t, transport, http.MethodGet, server.URL, "one", nil), "the whole body still arrives")
require.Equal(t, large, sharedGet(t, transport, http.MethodGet, server.URL, "two", nil))
require.EqualValues(t, 2, hits.Load())
}
type failOnce struct {
failed atomic.Bool
base http.RoundTripper
}
func (f *failOnce) RoundTrip(req *http.Request) (*http.Response, error) {
if f.failed.CompareAndSwap(false, true) {
return nil, errors.New("connection reset")
}
return f.base.RoundTrip(req)
}
func TestSharedResponsesDoNotKeepFailures(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.WriteString(w, "ok")
}))
defer server.Close()
transport := newSharingTransport(&failOnce{base: http.DefaultTransport})
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), "one"), http.MethodGet, server.URL, nil)
require.NoError(t, err)
_, err = transport.RoundTrip(req)
require.Error(t, err)
require.Equal(t, "ok", sharedGet(t, transport, http.MethodGet, server.URL, "two", nil), "the next template sends it again")
}
func TestSharedResponsesStartEmptyEachScan(t *testing.T) {
var hits atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
_, _ = io.WriteString(w, "ok")
}))
defer server.Close()
opts := newTestOptions(t, "test-shared-responses-reset")
transport := &sharingTransport{base: http.DefaultTransport, shared: getSharedResponses(opts)}
sharedGet(t, transport, http.MethodGet, server.URL, "one", nil)
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
require.EqualValues(t, 1, hits.Load())
// an SDK engine runs its next scan on the same execution
ResetSharedResponses(opts)
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
require.EqualValues(t, 2, hits.Load(), "a new scan does not see the responses of the last one")
}
func TestSharedResponsesCapCountsBodylessResponses(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
defer server.Close()
transport := newSharingTransport(http.DefaultTransport)
transport.shared.maxBytes = 20 * sharedEntryOverhead
for i := range 40 {
sharedGet(t, transport, http.MethodHead, fmt.Sprintf("%s/%d", server.URL, i), "one", nil)
}
require.LessOrEqual(t, transport.shared.bytes, transport.shared.maxBytes)
require.Less(t, len(transport.shared.entries), 20, "HEAD responses are evicted too")
}
func TestSharedResponsesKeepHeadContentLength(t *testing.T) {
var hits atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.Header().Set("Content-Length", "1234")
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
transport := newSharingTransport(http.DefaultTransport)
length := func(templateID string) int64 {
t.Helper()
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), templateID), http.MethodHead, server.URL, nil)
require.NoError(t, err)
resp, err := transport.RoundTrip(req)
require.NoError(t, err)
t.Cleanup(func() { _ = resp.Body.Close() })
return resp.ContentLength
}
require.Equal(t, int64(1234), length("one"))
require.Equal(t, int64(1234), length("two"), "a reused HEAD keeps the length the server advertised")
require.EqualValues(t, 1, hits.Load())
}
func TestSharedResponsesDoNotKeepRefusals(t *testing.T) {
for _, status := range []int{http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout} {
t.Run(http.StatusText(status), func(t *testing.T) {
var hits atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
http.Error(w, "no", status)
}))
defer server.Close()
transport := newSharingTransport(http.DefaultTransport)
sharedGet(t, transport, http.MethodGet, server.URL, "one", nil)
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
require.EqualValues(t, 2, hits.Load())
})
}
}
func TestSharedResponsesKeepNotFound(t *testing.T) {
var hits atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
http.NotFound(w, r)
}))
defer server.Close()
transport := newSharingTransport(http.DefaultTransport)
sharedGet(t, transport, http.MethodGet, server.URL, "one", nil)
sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
require.EqualValues(t, 1, hits.Load())
}
// shortBody yields a prefix and then ErrUnexpectedEOF, which is what a
// response with a Content-Length longer than its body looks like.
type shortBody struct {
prefix []byte
sent bool
}
func (s *shortBody) Read(p []byte) (int, error) {
if !s.sent {
s.sent = true
n := copy(p, s.prefix)
return n, io.ErrUnexpectedEOF
}
return 0, io.ErrUnexpectedEOF
}
func (s *shortBody) Close() error { return nil }
type countingTransport struct {
hits atomic.Int32
}
func (c *countingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
c.hits.Add(1)
return &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{"Content-Length": {"50000"}},
ContentLength: 50000,
Body: &shortBody{prefix: []byte("fourteen bytes")},
Request: req,
}, nil
}
func TestSharedResponsesDoNotKeepShortReads(t *testing.T) {
base := &countingTransport{}
transport := newSharingTransport(base)
send := func(templateID string) *http.Response {
t.Helper()
req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), templateID), http.MethodGet, "http://example.com", nil)
require.NoError(t, err)
resp, err := transport.RoundTrip(req)
require.NoError(t, err)
return resp
}
first := send("one")
body, err := io.ReadAll(first.Body)
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
require.Equal(t, "fourteen bytes", string(body))
require.Equal(t, int64(50000), first.ContentLength)
_ = first.Body.Close()
second := send("two")
_ = second.Body.Close()
require.EqualValues(t, 2, base.hits.Load(), "a short read is not reused")
}
+1 -2
View File
@@ -15,7 +15,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
convUtil "github.com/projectdiscovery/utils/conversion"
)
// Match matches a generic data response again a given matcher
@@ -226,7 +225,7 @@ func (request *Request) savedRequest(dump string) string {
if request.options == nil || request.options.Options == nil {
return dump
}
return string(capDumpBody(convUtil.Bytes(dump), request.options.Options.RequestSaveSize))
return string(capDumpBody([]byte(dump), request.options.Options.RequestSaveSize))
}
// eventResponse returns the full response of event. Events matched after the
+80 -121
View File
@@ -331,7 +331,6 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
// bounded worker-pool to avoid spawning one goroutine per payload
type task struct {
req *generatedRequest
body *bodyReservation
updatedInput *contextargs.Context
hasInteractMarkers bool
}
@@ -339,87 +338,72 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
var workersWg sync.WaitGroup
currentWorkers := maxWorkers
tasks := make(chan task, maxWorkers)
// Workers exit on cancel without draining, and the budget is shared, so
// release what they left queued.
stopWorkers := func() {
close(tasks)
workersWg.Wait()
for t := range tasks {
t.body.done()
}
}
spawnWorker := func(ctx context.Context) {
workersWg.Add(1)
go func() {
defer workersWg.Done()
for t := range tasks {
if func() (exit bool) {
defer t.body.done()
select {
case <-ctx.Done():
return true
default:
}
if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
return false
}
if err := spmHandler.Acquire(); err != nil {
return false
}
if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
spmHandler.Release()
return false
}
// Full URL, so https and an explicit port stay on the same key the observation uses.
// The handler context ends the wait when stop-at-first-match fires.
waitCtx, cancelWait := stopWithHandler(ctx, spmHandler)
err := request.rateLimitTake(waitCtx, hostBackoffKey(t.req, t.updatedInput))
cancelWait()
if err != nil {
select {
case <-spmHandler.Done():
spmHandler.Release()
return false
case <-ctx.Done():
spmHandler.Release()
return false
case spmHandler.ResultChan <- err:
spmHandler.Release()
return false
}
}
if ctx.Err() != nil || spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
spmHandler.Release()
return false
}
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
needsRequestEvent := hasInteractMatchers && request.NeedsRequestCondition()
err = request.executeRequest(t.updatedInput, t.req, make(map[string]interface{}), hasInteractMatchers, func(event *output.InternalWrappedEvent) {
if (t.hasInteractMarkers || needsRequestEvent) && request.options.Interactsh != nil {
requestData := &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
Event: event,
Operators: request.CompiledOperators,
MatchFunc: request.Match,
ExtractFunc: request.Extract,
}
allOASTUrls := httputils.GetInteractshURLSFromEvent(event.InternalEvent)
allOASTUrls = append(allOASTUrls, t.req.interactshURLs...)
request.options.RegisterInteractshRequest(sliceutil.Dedupe(allOASTUrls), requestData)
}
wrappedCallback(event)
}, 0)
select {
case <-ctx.Done():
return
default:
}
if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
continue
}
if err := spmHandler.Acquire(); err != nil {
continue
}
if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
spmHandler.Release()
continue
}
// Full URL, so https and an explicit port stay on the same key the observation uses.
// The handler context ends the wait when stop-at-first-match fires.
waitCtx, cancelWait := stopWithHandler(ctx, spmHandler)
err := request.rateLimitTake(waitCtx, hostBackoffKey(t.req, t.updatedInput))
cancelWait()
if err != nil {
select {
case <-spmHandler.Done():
spmHandler.Release()
return false
case spmHandler.ResultChan <- err:
request.recordHostResultAndCancelIfUnresponsive(t.updatedInput, err, spmHandler.Cancel)
continue
case <-ctx.Done():
spmHandler.Release()
continue
case spmHandler.ResultChan <- err:
spmHandler.Release()
continue
}
return false
}() {
return
}
if ctx.Err() != nil || spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
spmHandler.Release()
continue
}
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
needsRequestEvent := hasInteractMatchers && request.NeedsRequestCondition()
err = request.executeRequest(t.updatedInput, t.req, make(map[string]interface{}), hasInteractMatchers, func(event *output.InternalWrappedEvent) {
if (t.hasInteractMarkers || needsRequestEvent) && request.options.Interactsh != nil {
requestData := &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
Event: event,
Operators: request.CompiledOperators,
MatchFunc: request.Match,
ExtractFunc: request.Extract,
}
allOASTUrls := httputils.GetInteractshURLSFromEvent(event.InternalEvent)
allOASTUrls = append(allOASTUrls, t.req.interactshURLs...)
request.options.RegisterInteractshRequest(sliceutil.Dedupe(allOASTUrls), requestData)
}
wrappedCallback(event)
}, 0)
select {
case <-spmHandler.Done():
spmHandler.Release()
continue
case spmHandler.ResultChan <- err:
request.recordHostResultAndCancelIfUnresponsive(t.updatedInput, err, spmHandler.Cancel)
spmHandler.Release()
}
}
}()
@@ -438,7 +422,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
select {
case <-input.Context().Done():
stopWorkers()
close(tasks)
workersWg.Wait()
return input.Context().Err()
default:
}
@@ -446,7 +431,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
// resize check point - nop if there are no changes
if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency {
if err := spmHandler.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil {
stopWorkers()
close(tasks)
workersWg.Wait()
return err
}
// if payload concurrency increased, add more workers
@@ -463,22 +449,15 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
break
}
// Queued tasks hold budget and are only released once the workers
// stop, so the wait has to end with them.
generatedHttpRequest, reservation, err := request.makeReserved(workerCtx, generator, input, inputData, payloads, dynamicValues)
ctx := input.Context()
generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
if err != nil {
if err == types.ErrNoMoreRequests {
break
}
if workerCtx.Err() != nil {
stopWorkers()
if spmHandler.FoundFirstMatch() {
return nil
}
return context.Cause(workerCtx)
}
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
stopWorkers()
close(tasks)
workersWg.Wait()
return err
}
hasInteractMarkers := interactsh.HasMarkers(inputData) || len(generatedHttpRequest.interactshURLs) > 0
@@ -488,32 +467,33 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
if request.isUnresponsiveAddress(updatedInput) {
// skip on unresponsive host no need to continue
reservation.done()
spmHandler.Cancel()
stopWorkers()
close(tasks)
workersWg.Wait()
return nil
}
select {
case <-spmHandler.Done():
reservation.done()
stopWorkers()
close(tasks)
workersWg.Wait()
spmHandler.Wait()
if spmHandler.FoundFirstMatch() {
return nil
}
return multierr.Combine(spmHandler.CombinedResults()...)
case <-workerCtx.Done():
reservation.done()
stopWorkers()
close(tasks)
workersWg.Wait()
if spmHandler.FoundFirstMatch() {
return nil
}
return context.Cause(workerCtx)
case tasks <- task{req: generatedHttpRequest, body: reservation, updatedInput: updatedInput, hasInteractMarkers: hasInteractMarkers}:
case tasks <- task{req: generatedHttpRequest, updatedInput: updatedInput, hasInteractMarkers: hasInteractMarkers}:
}
request.options.Progress.IncrementRequests()
}
stopWorkers()
close(tasks)
workersWg.Wait()
spmHandler.Wait()
if spmHandler.FoundFirstMatch() {
// ignore any context cancellation and in-transit execution errors
@@ -602,7 +582,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
}
ctx := input.Context()
generatedHttpRequest, reservation, err := request.makeReserved(ctx, generator, input, inputData, payloads, dynamicValues)
generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
if err != nil {
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
return err
@@ -613,18 +593,15 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
if request.isUnresponsiveAddress(updatedInput) {
// skip on unresponsive host no need to continue
reservation.done()
spmHandler.Cancel()
return nil
}
generatedHttpRequest.pipelinedClient = pipeClient
if err := spmHandler.Acquire(); err != nil {
reservation.done()
break
}
go func(httpRequest *generatedRequest, body *bodyReservation, requestInput *contextargs.Context) {
go func(httpRequest *generatedRequest, requestInput *contextargs.Context) {
defer spmHandler.Release()
defer body.done()
if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(requestInput) {
// skip if first match is found
return
@@ -649,7 +626,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
request.recordHostResultAndCancelIfUnresponsive(requestInput, err, spmHandler.Cancel)
return
}
}(generatedHttpRequest, reservation, updatedInput)
}(generatedHttpRequest, updatedInput)
request.options.Progress.IncrementRequests()
}
spmHandler.Wait()
@@ -699,17 +676,16 @@ executionLoop:
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
ctx := input.Context()
generatedHttpRequest, reservation, err := request.makeReserved(ctx, generator, input, data, payloads, dynamicValue)
ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, request.options.Options.GetTimeouts().HttpTimeout, ErrHttpEngineRequestDeadline)
defer cancel()
generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
if err != nil {
if err == types.ErrNoMoreRequests {
return false, true, nil, nil
}
return false, true, nil, err
}
defer reservation.done()
if generatedHttpRequest.customCancelFunction == nil {
generatedHttpRequest.sendTimeout = request.options.Options.GetTimeouts().HttpTimeout
}
// ideally if http template used a custom port or hostname
// we would want to update it in input but currently templateCtx logic
// is closely tied to contextargs.Context so we are temporarily creating
@@ -1084,20 +1060,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
return fmt.Errorf("hostErrorsCache : host %s is unresponsive", input.MetaInput.Input)
}
// Race requests must all be in flight together to reach the open gate.
if !generatedRequest.original.Race && !generatedRequest.bodyReserved {
releaseBody, err := request.options.RequestBodyBudget.Acquire(input.Context(), generatedRequest.bodyLen())
if err != nil {
return err
}
defer releaseBody()
}
if generatedRequest.sendTimeout > 0 && generatedRequest.request != nil {
ctx, cancel := context.WithTimeoutCause(generatedRequest.request.Context(), generatedRequest.sendTimeout, ErrHttpEngineRequestDeadline)
defer cancel()
generatedRequest.request = generatedRequest.request.WithContext(ctx)
}
// wrap one more callback for validation and fixing event
callback := func(event *output.InternalWrappedEvent) {
// validateNFixEvent performs necessary validation on generated event
@@ -1340,9 +1302,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
}
}
if request.sharesResponse {
generatedRequest.request.WithContext(httpclientpool.WithSharedResponse(generatedRequest.request.Context(), request.options.TemplateID))
}
resp, err = executingClient.Do(generatedRequest.request)
sent = true
+1 -1
View File
@@ -1400,7 +1400,7 @@ func TestExecuteParallelHTTP_GoroutineLeaks(t *testing.T) {
defer goleak.VerifyNone(t,
goleak.IgnoreAnyContainingPkg("go.opencensus.io/stats/view"),
goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/rod"),
goleak.IgnoreAnyContainingPkg("github.com/go-rod/rod"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/server"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/client"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/ratelimit"),
-20
View File
@@ -1,20 +0,0 @@
package http
import (
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
"github.com/stretchr/testify/require"
)
func TestCanShareResponseKeepsTimingChecksOnTheirOwnRequest(t *testing.T) {
require.True(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Words: []string{"admin"}}}}}, ""))
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{DSL: []string{"duration>=5"}}}}}, ""))
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Part: "duration"}}}}, ""))
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Extractors: []*extractors.Extractor{{DSL: []string{"duration_2"}}}}}, ""))
require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Type: matchers.MatcherTypeHolder{MatcherType: matchers.LLMMatcher}, Inputs: []string{"{{duration_1}}"}}}}}, ""), "llm inputs read the event too")
require.False(t, canShareResponse(&Request{}, "http(1) && template.duration > 5"))
require.False(t, canShareResponse(&Request{Race: true}, ""))
}
+1 -22
View File
@@ -7,7 +7,6 @@ import (
"fmt"
"io"
"net/http"
"slices"
"strings"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
@@ -83,9 +82,6 @@ func dumpRequest(req *retryablehttp.Request, maxBody int) (requestDump, error) {
}
capture := &cappedWriter{headerEnd: -1, maxBody: maxBody}
if maxBody <= 0 {
capture.buf = make([]byte, 0, len(body)+dumpHeaderRoom)
}
if err := clone.Write(capture); err != nil {
return requestDump{}, err
}
@@ -93,7 +89,7 @@ func dumpRequest(req *retryablehttp.Request, maxBody int) (requestDump, error) {
// http.Transport asks for gzip unless the caller set an encoding.
if clone.Header.Get("Accept-Encoding") == "" && clone.Header.Get("Range") == "" && clone.Method != http.MethodHead && capture.headerEnd >= 2 {
end := capture.headerEnd - 2
data = slices.Insert(data, end, acceptEncodingGzip...)
data = append(data[:end:end], append([]byte("Accept-Encoding: gzip\r\n"), data[end:]...)...)
}
return requestDump{data: appendTruncationMarker(data, capture.dropped), truncated: capture.dropped}, nil
}
@@ -127,10 +123,6 @@ func capDumpBody(dump []byte, maxBody int) []byte {
var headerTerminator = []byte("\r\n\r\n")
var acceptEncodingGzip = []byte("Accept-Encoding: gzip\r\n")
const dumpHeaderRoom = 8 * 1024
// cappedWriter keeps the request line and headers in full and at most
// maxBody bytes of the body that follows, counting the rest.
type cappedWriter struct {
@@ -185,19 +177,6 @@ func getHTTPProjectCacheScope(requestDump requestDump, body []byte, scheme, host
return scoped.Bytes()
}
func (g *generatedRequest) bodyLen() int64 {
if g.rawRequest != nil {
return int64(len(g.rawRequest.Data))
}
if g.request == nil || g.request.Body == nil {
return 0
}
if body, ok := g.request.Body.(*httpbody.Body); ok {
return int64(body.Len())
}
return g.request.ContentLength
}
// body returns the request body without copying it.
func (g *generatedRequest) body() []byte {
if g.request != nil {
-5
View File
@@ -44,7 +44,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
unitutils "github.com/projectdiscovery/utils/unit"
)
@@ -107,8 +106,6 @@ type ExecutorOptions struct {
Progress progress.Progress
// RateLimiter is a rate-limiter for limiting sent number of requests.
RateLimiter *ratelimit.Limiter
// RequestBodyBudget bounds request-body bytes in flight; nil is no limit.
RequestBodyBudget *httpbody.Budget
// Catalog is a template catalog implementation for nuclei
Catalog catalog.Catalog
// ProjectFile is the project file for nuclei
@@ -398,7 +395,6 @@ func (e *ExecutorOptions) Copy() *ExecutorOptions {
IssuesClient: e.IssuesClient,
Progress: e.Progress,
RateLimiter: e.RateLimiter,
RequestBodyBudget: e.RequestBodyBudget,
Catalog: e.Catalog,
ProjectFile: e.ProjectFile,
Browser: e.Browser,
@@ -710,7 +706,6 @@ func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) {
e.IssuesClient = n.IssuesClient
e.Progress = n.Progress
e.RateLimiter = n.RateLimiter
e.RequestBodyBudget = n.RequestBodyBudget
e.Catalog = n.Catalog
e.ProjectFile = n.ProjectFile
e.Browser = n.Browser
-34
View File
@@ -23,7 +23,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
@@ -35,7 +34,6 @@ import (
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
"github.com/projectdiscovery/tlsx/pkg/tlsx/openssl"
"github.com/projectdiscovery/utils/errkit"
iputil "github.com/projectdiscovery/utils/ip"
stringsutil "github.com/projectdiscovery/utils/strings"
)
@@ -204,21 +202,6 @@ func (request *Request) GetID() string {
return request.ID
}
// allowedAddress returns the first address of host that the network policy allows.
func (request *Request) allowedAddress(host string) (string, error) {
dnsData, err := request.dialer.GetDNSData(host)
if err != nil {
return "", err
}
addresses := append(append([]string{}, dnsData.A...), dnsData.AAAA...)
for _, address := range addresses {
if protocolstate.IsHostAllowed(request.options.Options.ExecutionId, address) {
return address, nil
}
}
return "", protocolstate.ErrHostDenied.Msgf(host)
}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
hostPort := input.MetaInput.Input
@@ -272,23 +255,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
} else {
hostIp = host
}
if !iputil.IsIP(hostIp) {
// The openssl scan mode runs the openssl binary, which resolves a host
// name again by itself, so it gets an address the policy allows.
address, err := request.allowedAddress(hostIp)
if err != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
return errkit.Wrap(err, "could not resolve host")
}
hostIp = address
} else if !protocolstate.IsHostAllowed(request.options.Options.ExecutionId, hostIp) {
// A custom dialer can allow an address that the execution policy denies.
err := protocolstate.ErrHostDenied.Msgf(hostIp)
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
return err
}
if err := request.options.HostBackoffWait(input.Context(), addressToDial); err != nil {
return err
-179
View File
@@ -1,179 +0,0 @@
package ssl
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"math/big"
"net"
"slices"
"sync/atomic"
"testing"
"time"
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/tlsx/pkg/tlsx/openssl"
)
func selfSignedCertificate(t *testing.T, name string) tls.Certificate {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
require.NoError(t, err)
template := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{CommonName: name},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour),
}
der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
require.NoError(t, err)
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}
}
// serveTLS counts the TLS handshakes that reach address.
func serveTLS(t *testing.T, address, name string, handshakes *atomic.Int32) string {
t.Helper()
listener, err := tls.Listen("tcp", address, &tls.Config{Certificates: []tls.Certificate{selfSignedCertificate(t, name)}})
require.NoError(t, err)
t.Cleanup(func() { _ = listener.Close() })
go func() {
for {
conn, err := listener.Accept()
if err != nil {
return
}
go func() {
defer func() { _ = conn.Close() }()
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
if conn.(*tls.Conn).Handshake() == nil {
handshakes.Add(1)
}
}()
}
}()
return listener.Addr().String()
}
func TestSSLOpenSSLObeysNetworkPolicy(t *testing.T) {
if !openssl.IsAvailable() {
t.Skip("openssl binary is not installed")
}
addrs, err := net.LookupHost("localhost")
if err != nil || len(addrs) < 2 {
t.Skip("localhost does not resolve to both loopback addresses")
}
// The policy allows 127.0.0.1 and denies ::1, so localhost passes the
// dial check. The openssl binary must not connect to ::1.
var allowedHandshakes, deniedHandshakes atomic.Int32
allowed := serveTLS(t, "127.0.0.1:0", "allowed", &allowedHandshakes)
_, port, err := net.SplitHostPort(allowed)
require.NoError(t, err)
serveTLS(t, net.JoinHostPort("::1", port), "denied", &deniedHandshakes)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ExcludeTargets = []string{"::1"}
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "ssl-openssl-policy"})
executerOpts.IsMultiProtocol = true
request := &Request{ID: "ssl-openssl-policy", Address: "{{Host}}:{{Port}}", ScanMode: "openssl"}
require.NoError(t, request.Compile(executerOpts))
// The scan resolves names with the fastdialer, not the system resolver. On
// Windows the hosts file has no localhost entry, so the name does not resolve.
if !resolvesToBothLoopbacks(request) {
t.Skip("the scan dialer does not resolve localhost to both loopback addresses")
}
var event output.InternalEvent
input := contextargs.NewWithInput(context.Background(), net.JoinHostPort("localhost", port))
err = request.ExecuteWithResults(input, nil, nil, func(e *output.InternalWrappedEvent) {
event = e.InternalEvent
})
require.NoError(t, err)
require.Equal(t, "allowed", event["subject_cn"])
require.Zero(t, deniedHandshakes.Load(), "openssl connected to a denied address")
}
func resolvesToBothLoopbacks(request *Request) bool {
data, err := request.dialer.GetDNSData("localhost")
if err != nil {
return false
}
return slices.Contains(data.A, "127.0.0.1") && slices.Contains(data.AAAA, "::1")
}
func TestSSLOpenSSLLiteralIPObeysExecutionPolicy(t *testing.T) {
if !openssl.IsAvailable() {
t.Skip("openssl binary is not installed")
}
// The input pins ::1 as its custom ip, and a custom dialer with no deny
// list would connect to it. The execution policy denies it, so the openssl
// binary must not connect to it.
var deniedHandshakes atomic.Int32
denied := serveTLS(t, "[::1]:0", "denied", &deniedHandshakes)
_, port, err := net.SplitHostPort(denied)
require.NoError(t, err)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ExcludeTargets = []string{"::1"}
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "ssl-openssl-literal-ip"})
executerOpts.IsMultiProtocol = true
customDialer, err := fastdialer.NewDialer(fastdialer.DefaultOptions)
require.NoError(t, err)
t.Cleanup(customDialer.Close)
executerOpts.CustomFastdialer = customDialer
request := &Request{ID: "ssl-openssl-literal-ip", Address: "{{Host}}:{{Port}}", ScanMode: "openssl"}
require.NoError(t, request.Compile(executerOpts))
input := contextargs.NewWithInput(context.Background(), net.JoinHostPort("localhost", port))
input.MetaInput.CustomIP = "::1"
err = request.ExecuteWithResults(input, nil, nil, func(*output.InternalWrappedEvent) {})
require.Error(t, err)
require.Zero(t, deniedHandshakes.Load(), "openssl connected to an address the execution policy denies")
}
func TestSSLOpenSSLCustomIPObeysNetworkPolicy(t *testing.T) {
if !openssl.IsAvailable() {
t.Skip("openssl binary is not installed")
}
// The input pins the denied address ::1 as its custom ip, so the request
// skips the name lookup. The openssl binary must still not connect to it.
var deniedHandshakes atomic.Int32
denied := serveTLS(t, "[::1]:0", "denied", &deniedHandshakes)
_, port, err := net.SplitHostPort(denied)
require.NoError(t, err)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ExcludeTargets = []string{"::1"}
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "ssl-openssl-custom-ip"})
executerOpts.IsMultiProtocol = true
request := &Request{ID: "ssl-openssl-custom-ip", Address: "{{Host}}:{{Port}}", ScanMode: "openssl"}
require.NoError(t, request.Compile(executerOpts))
input := contextargs.NewWithInput(context.Background(), net.JoinHostPort("localhost", port))
input.MetaInput.CustomIP = "::1"
_ = request.ExecuteWithResults(input, nil, nil, func(*output.InternalWrappedEvent) {})
require.Zero(t, deniedHandshakes.Load(), "openssl connected to a denied custom ip")
}
-93
View File
@@ -1,93 +0,0 @@
package replay
import (
"encoding/base64"
"unicode/utf8"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
)
// Matchers type-assert some values, such as an int status code or a []string
// of dns answers, and binary responses are not valid UTF-8, so plain JSON
// would change what they see. Those values are wrapped in a one-key object
// naming their Go type; everything else is stored as JSON.
const (
tagBytes = "$bytes"
tagBinary = "$binary"
tagInt = "$int"
tagStrings = "$strings"
)
func encodeEvent(event output.InternalEvent) map[string]any {
encoded := make(map[string]any, len(event))
for key, value := range event {
switch v := value.(type) {
case string:
if utf8.ValidString(v) {
encoded[key] = v
} else {
encoded[key] = map[string]string{tagBinary: base64.StdEncoding.EncodeToString([]byte(v))}
}
case []byte:
encoded[key] = map[string]string{tagBytes: base64.StdEncoding.EncodeToString(v)}
case int:
encoded[key] = map[string]int{tagInt: v}
case []string:
encoded[key] = map[string][]string{tagStrings: v}
default:
// values JSON cannot hold, such as funcs, are not something a
// matcher reads
if _, err := json.Marshal(v); err == nil {
encoded[key] = v
}
}
}
return encoded
}
func decodeEvent(raw map[string]json.Message) (output.InternalEvent, error) {
event := make(output.InternalEvent, len(raw))
for key, message := range raw {
value, err := decodeValue(message)
if err != nil {
return nil, err
}
event[key] = value
}
return event, nil
}
func decodeValue(message json.Message) (any, error) {
var tagged map[string]json.Message
if json.Unmarshal(message, &tagged) == nil && len(tagged) == 1 {
for tag, inner := range tagged {
switch tag {
case tagBytes, tagBinary:
var text string
if err := json.Unmarshal(inner, &text); err != nil {
return nil, err
}
data, err := base64.StdEncoding.DecodeString(text)
if err != nil {
return nil, err
}
if tag == tagBinary {
return string(data), nil
}
return data, nil
case tagInt:
var number int
err := json.Unmarshal(inner, &number)
return number, err
case tagStrings:
var list []string
err := json.Unmarshal(inner, &list)
return list, err
}
}
}
var value any
err := json.Unmarshal(message, &value)
return value, err
}
-39
View File
@@ -1,39 +0,0 @@
package replay
import (
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
"github.com/stretchr/testify/require"
)
func TestEventRoundTripKeepsWhatMatchersRead(t *testing.T) {
event := output.InternalEvent{
"body": "<title>admin</title>",
"raw": "\x00\xffBIN\xfe",
"data": []byte{0, 1, 2},
"status_code": 200,
"answer": []string{"a", "b"},
"duration": 1.5,
"matched": true,
"info": map[string]any{"name": "x"},
"callback": func() {},
}
data, err := json.Marshal(encodeEvent(event))
require.NoError(t, err)
var raw map[string]json.Message
require.NoError(t, json.Unmarshal(data, &raw))
decoded, err := decodeEvent(raw)
require.NoError(t, err)
require.Equal(t, "<title>admin</title>", decoded["body"])
require.Equal(t, "\x00\xffBIN\xfe", decoded["raw"], "binary responses keep every byte")
require.Equal(t, []byte{0, 1, 2}, decoded["data"])
require.Equal(t, 200, decoded["status_code"], "the status matcher reads an int")
require.Equal(t, []string{"a", "b"}, decoded["answer"])
require.Equal(t, 1.5, decoded["duration"])
require.Equal(t, true, decoded["matched"])
require.Equal(t, map[string]any{"name": "x"}, decoded["info"])
require.NotContains(t, decoded, "callback", "values JSON cannot hold are dropped")
}
Loaded 100 of 118 files, more files were not shown because too many files have changed in this diff. Show more