mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2026-11-08 10:10:18 -10:00
Compare commits
36
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7a64f6913b | ||
|
|
2dbdb6e111 | ||
|
|
4cde02dd2e | ||
|
|
f6abdcf2f2 | ||
|
|
f5d0ef81a5 | ||
|
|
911c2303fe | ||
|
|
dd2374604b | ||
|
|
4fb9d22b68 | ||
|
|
38dd02f13b | ||
|
|
fe7b1df128 | ||
|
|
33d71f9156 | ||
|
|
c126e2abcc | ||
|
|
d00461a2cc | ||
|
|
09c1bbf51b | ||
|
|
d7d7cf2b2c | ||
|
|
689026b756 | ||
|
|
d0e6a9d869 | ||
|
|
80f1839aa3 | ||
|
|
fcbaa8f4b2 | ||
|
|
4138b68134 | ||
|
|
f66a832d8e | ||
|
|
dc5f464282 | ||
|
|
2e22d4f612 | ||
|
|
7ebe19d60b | ||
|
|
2ba82f722f | ||
|
|
82071fe0c9 | ||
|
|
550b92da42 | ||
|
|
a34f810ffa | ||
|
|
6d4d1be80c | ||
|
|
46ea619bc6 | ||
|
|
87fbc2d045 | ||
|
|
6ec0e9e52c | ||
|
|
91ffec51a1 | ||
|
|
24a101fbdf | ||
|
|
eb10c9f160 | ||
|
|
e908de14fa |
No files matched your search
@@ -16,5 +16,4 @@ jobs:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: projectdiscovery/actions/setup/go/compat-checks@v1
|
||||
with:
|
||||
go-version: "stable"
|
||||
release-test: true
|
||||
@@ -4,6 +4,8 @@ on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "17 4 * * 1"
|
||||
push:
|
||||
paths: [.github/workflows/security-context.yml]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -18,8 +20,9 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
# The API analyzes the repository default branch, so use that branch for
|
||||
# both scheduled and manually dispatched updates.
|
||||
# A workflow can initially run from the branch that adds this file. The
|
||||
# API analyzes the repository default branch, so check out that same
|
||||
# branch before calculating the HEAD we wait for.
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
@@ -71,25 +74,19 @@ jobs:
|
||||
fi
|
||||
echo "changed=$changed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Point every supported root agent-instructions file at the context.
|
||||
# Create AGENTS.md as the cross-agent default when the repo has none.
|
||||
# Point the repository's agent instructions at the file, when it has any.
|
||||
# Never create one: adding an agent instructions file uninvited is a
|
||||
# bigger footprint than this warrants.
|
||||
- name: Reference the context from agent instructions
|
||||
id: agents
|
||||
env:
|
||||
REPORT_CHANGED: ${{ steps.report.outputs.changed }}
|
||||
run: |
|
||||
line="Before writing or reviewing code, read SECURITY_CONTEXT.md. Check new and modified code against its historical fixes and recurring vulnerability patterns. Do not reintroduce a previously fixed issue or a similar variant."
|
||||
line="See SECURITY_CONTEXT.md for this repo's known vulnerabilities and recurring weak spots. Check it before writing or reviewing security-sensitive code."
|
||||
touched=""
|
||||
paths="SECURITY_CONTEXT.md"
|
||||
instruction_files=""
|
||||
for f in AGENTS.md CLAUDE.md; do
|
||||
if [ -f "$f" ]; then instruction_files="${instruction_files} $f"; fi
|
||||
done
|
||||
if [ -z "$instruction_files" ]; then
|
||||
printf '%s\n' '# Agent instructions' > AGENTS.md
|
||||
instruction_files="AGENTS.md"
|
||||
fi
|
||||
for f in $instruction_files; do
|
||||
for f in CLAUDE.md AGENTS.md; do
|
||||
if [ ! -f "$f" ]; then continue; fi
|
||||
paths="$(printf '%s\n%s' "$paths" "$f")"
|
||||
if grep -q "SECURITY_CONTEXT.md" "$f"; then continue; fi
|
||||
printf '\n%s\n' "$line" >> "$f"
|
||||
@@ -120,7 +117,7 @@ jobs:
|
||||
|
||||
${{ steps.context.outputs.visibility == 'private' && format('[Read the updated file]({0}/{1}/blob/security-context/refresh/SECURITY_CONTEXT.md)', github.server_url, github.repository) || format('[Full report for this repository](https://securitycontext.dev/r/{0})', github.repository) }}
|
||||
|
||||
${{ steps.agents.outputs.touched != '' && format('Also adds a one-line reference to {0} so your coding agent picks this up automatically.', steps.agents.outputs.touched) || 'The repository agent instructions already reference `SECURITY_CONTEXT.md`.' }}
|
||||
${{ steps.agents.outputs.touched != '' && format('Also adds a one-line reference to {0} so your coding agent picks this up automatically.', steps.agents.outputs.touched) || 'Tip: reference `SECURITY_CONTEXT.md` from your agent instructions file (`CLAUDE.md` or `AGENTS.md`) so your coding agent reads it automatically.' }}
|
||||
|
||||
---
|
||||
_Your agent doesn't know this repo's security history. [Security Context](https://securitycontext.dev) keeps it where your agent can see it, so old vulnerabilities and their variants don't come back in new code._
|
||||
|
||||
@@ -206,8 +206,6 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: projectdiscovery/actions/setup/go@v1
|
||||
with:
|
||||
go-version: "stable"
|
||||
- uses: projectdiscovery/actions/goreleaser@v1
|
||||
|
||||
flamegraph:
|
||||
|
||||
@@ -33,6 +33,8 @@
|
||||
|
||||
# Templates
|
||||
/*.yaml
|
||||
!/.sops.yaml
|
||||
!/.golangci.yaml
|
||||
|
||||
# Generated docs
|
||||
/dsl.md
|
||||
|
||||
@@ -12,6 +12,7 @@ builds:
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
- GOEXPERIMENT=greenteagc
|
||||
- GODEBUG=tlssha1=1
|
||||
goos: [windows, linux, darwin]
|
||||
goarch: [amd64, '386', arm, arm64]
|
||||
ignore:
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
# SOPS Creation Rules for Secret File (ref: https://docs.projectdiscovery.io/tools/nuclei/authenticated-scans)
|
||||
|
||||
creation_rules:
|
||||
- encrypted_regex: ^(password|username|token|value|key|raw)$
|
||||
@@ -33,8 +33,9 @@ clean:
|
||||
|
||||
go-build: clean
|
||||
go-build:
|
||||
CGO_ENABLED=0 $(GOBUILD) -trimpath $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \
|
||||
-o '${GOBUILD_OUTPUT}${GOBUILD_OUTPUT_EXT}' $(GOBUILD_PACKAGES)
|
||||
CGO_ENABLED=0 GOEXPERIMENT=greenteagc GODEBUG=tlssha1=1 $(GOBUILD) \
|
||||
-trimpath $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \
|
||||
-o '${GOBUILD_OUTPUT}${GOBUILD_OUTPUT_EXT}' $(GOBUILD_PACKAGES)
|
||||
|
||||
build: GOFLAGS = -pgo=auto
|
||||
build: GOBUILD_OUTPUT = ./bin/nuclei
|
||||
@@ -297,4 +298,4 @@ template-validate:
|
||||
./bin/nuclei -validate \
|
||||
-w workflows \
|
||||
-et http/technologies \
|
||||
-ept code
|
||||
-ept code
|
||||
+168
-215
@@ -1,302 +1,255 @@
|
||||
<!-- Generated by securitycontext.dev. Source HEAD: b04fc59ad056c3d818aa103a85164c633e856277. -->
|
||||
<!-- Generated by securitycontext.dev. Source HEAD: 3e1e0e81f091c7817bbf7af64bb8eec6e8c2bac1. -->
|
||||
# Security Context: projectdiscovery/nuclei @ dev
|
||||
|
||||
> Reference context for editing projectdiscovery/nuclei. Before changing any component below, **preserve the listed guards and do not reintroduce these fixed issues**. Each fingerprint is a regression check, and each shared attack surface is a guard that must keep holding on every path.
|
||||
|
||||
_Generated 2026-09-03 00:44 UTC by secc_
|
||||
_Generated 2026-09-01 16:22 UTC by secc_
|
||||
|
||||
_Analyzed commit: b04fc59ad056c3d818aa103a85164c633e856277_
|
||||
_Analyzed commit: 3e1e0e81f091c7817bbf7af64bb8eec6e8c2bac1_
|
||||
|
||||
**Remediation coverage: 26.5%**
|
||||
_9 of 34 historical fixes have no remaining unguarded variant; 522 open remediation leads to verify. Measures fix consistency, not a vulnerability verdict._
|
||||
**Remediation coverage: 56.7%**
|
||||
_17 of 30 historical fixes have no remaining unguarded variant; 368 open remediation leads to verify. Measures fix consistency, not a vulnerability verdict._
|
||||
|
||||
- Commits analyzed: **4820**
|
||||
- Commits flagged for review: **415**
|
||||
- Confirmed historical security fixes: **38**
|
||||
- Known published CVEs: **6**
|
||||
- Project type: **client**, excluding 34 availability-only (DoS/crash) finding(s) as out-of-scope for a non-server project
|
||||
- Commits analyzed: **4814**
|
||||
- Commits flagged for review: **235**
|
||||
- Confirmed historical security fixes: **32**
|
||||
- Known published CVEs: **0**
|
||||
|
||||
## Summary
|
||||
|
||||
Nuclei's historical security posture is dominated by sandbox and trust-boundary enforcement failures: template-driven code execution, local file access, and network egress are repeatedly gated inconsistently across loaders, protocol runtimes, and JS helper libraries. Signature verification bypasses (CVE-2024-43405, CVE-2024-27920) and expression/DSL injection (CVE-2026-41645, CVE-2026-41282) show that untrusted template content and response data are persistently re-evaluated or executed. Path traversal and arbitrary file write/read issues cluster in file-loading and output-writing paths, often due to weak prefix checks or missing sandbox flags. The most critical theme is asymmetric enforcement: a guard applied on one entry point (e.g., direct template load) is frequently missing on sibling paths (workflows, JS runtime, protocol sinks, helper libraries).
|
||||
Nuclei's historical security posture is dominated by sandbox escapes and code execution via unsigned or unverified templates, with recurring SSRF, path traversal, and auth bypass issues across JS libraries, loaders, and workflow paths. The most critical theme is inconsistent enforcement of signature verification and local-file-access (-lfa) guards across direct, workflow, DAST, and JS-sandbox entry points, enabling variant attacks. A second theme is network policy (host allowlist) gaps in LDAP dialing and proxy handling. The security context also includes dependency upgrades (Alpine, go-git) and output file permission hardening, but the highest-value focus is on template trust boundaries and sandbox enforcement parity.
|
||||
|
||||
## Recurring Security Patterns
|
||||
|
||||
**18.4% of classified fixes repeat a previously fixed pattern** (7 recurring, 31 first-seen). A recurring pattern means this repository fixed a similar issue before — not that the code is currently vulnerable.
|
||||
**15.6% of classified fixes repeat a previously fixed pattern** (5 recurring, 27 first-seen). A recurring pattern means this repository fixed a similar issue before — not that the code is currently vulnerable.
|
||||
|
||||
- **Code Execution / Signature Bypass:** [fac0b7196d](https://github.com/projectdiscovery/nuclei/commit/fac0b7196d2fbca56c8af8ff8e1899c111041a37) → [9de96e4dda](https://github.com/projectdiscovery/nuclei/commit/9de96e4dda5a03da963b9ae6582f03ea55791a76) — matched on `CWE-347`, `component:pkg/catalog/loader/loader.go`
|
||||
- **Auth Bypass:** [1c440e755a](https://github.com/projectdiscovery/nuclei/commit/1c440e755a97471c56fb0276ee8a8c4132974645) → [fac0b7196d](https://github.com/projectdiscovery/nuclei/commit/fac0b7196d2fbca56c8af8ff8e1899c111041a37) — matched on `CWE-347`, `component:pkg/catalog/loader/loader.go`
|
||||
- **Code Execution:** [1c440e755a](https://github.com/projectdiscovery/nuclei/commit/1c440e755a97471c56fb0276ee8a8c4132974645) → [9de96e4dda](https://github.com/projectdiscovery/nuclei/commit/9de96e4dda5a03da963b9ae6582f03ea55791a76) — matched on `class:code execution`, `component:pkg/catalog/loader/loader.go`
|
||||
- **Code Execution via Unsigned Template:** [fa56800fcc](https://github.com/projectdiscovery/nuclei/commit/fa56800fcca3c8c8fe8e8dec28584210ca1527de) → [fac0b7196d](https://github.com/projectdiscovery/nuclei/commit/fac0b7196d2fbca56c8af8ff8e1899c111041a37) — matched on `class:code execution via unsigned template`, `CWE-347`, `sink:loadtemplates`
|
||||
- **SSRF:** [a167e6c57b](https://github.com/projectdiscovery/nuclei/commit/a167e6c57b80f5dc7be231231b62d7c594ee2770) → [9def9af7e3](https://github.com/projectdiscovery/nuclei/commit/9def9af7e3f7ebcf5cd561c37b623b9999ee7572) — matched on `class:ssrf`, `CWE-918`, `component:pkg/js/libs/ldap/ldap.go`
|
||||
- **Path Traversal:** [e510bc207f](https://github.com/projectdiscovery/nuclei/commit/e510bc207f4336058f7b1536ddbb3942b394b05c) → [f663d1c9cf](https://github.com/projectdiscovery/nuclei/commit/f663d1c9cf77da53533bf328750621b4a9fa996d) — matched on `class:path traversal`, `CWE-22`, `sink:os openfile`
|
||||
- **Path Traversal:** [e7fb40a413](https://github.com/projectdiscovery/nuclei/commit/e7fb40a41308f6c41267e858d1781eb67f7e0572) → [e5154d362a](https://github.com/projectdiscovery/nuclei/commit/e5154d362af900e83b30fb79028bfab1a7f0c648) — matched on `class:path traversal`, `CWE-22`, `component:v2/pkg/protocols/common/generators`
|
||||
- **Insecure File Permissions:** [4bccb6cf8a](https://github.com/projectdiscovery/nuclei/commit/4bccb6cf8a830724b3bda624cb3bfe43eff31683) → [af4854f90d](https://github.com/projectdiscovery/nuclei/commit/af4854f90de728bb08689d5cae3dec4acfdb6b9a) — matched on `CWE-732`, `sink:os openfile`
|
||||
- **Path Traversal:** [f9c214a66f](https://github.com/projectdiscovery/nuclei/commit/f9c214a66f37a00fc29fe64232b690e6285ec59c) → [e510bc207f](https://github.com/projectdiscovery/nuclei/commit/e510bc207f4336058f7b1536ddbb3942b394b05c) — matched on `class:path traversal`, `CWE-22`, `sink:os openfile`
|
||||
- **Dependency Vulnerability:** [d93b4a01df](https://github.com/projectdiscovery/nuclei/commit/d93b4a01df9c218a36dd2fccfb1c2c1f96b181fe) → [fcd5c6b111](https://github.com/projectdiscovery/nuclei/commit/fcd5c6b111ddcb746525f88b7d0bf4b4abbc2529) — matched on `class:dependency vulnerability`, `CWE-1104`, `component:dockerfile`
|
||||
- **Sandbox Escape:** [96049a32fa](https://github.com/projectdiscovery/nuclei/commit/96049a32fa70aef0483d4180d386a57b3c2d973a) → [cf2a98b03d](https://github.com/projectdiscovery/nuclei/commit/cf2a98b03d76327c20fb0183b7f1c849eb692c91) — matched on `class:sandbox escape`, `CWE-284`, `component:v2/internal/runner/processor.go`, `sink:stdlib allmodulenames`, `sink:stdlib getmodulemap`
|
||||
|
||||
## Top Risks
|
||||
|
||||
### 1. Code Execution / Signature Bypass: `pkg/catalog/loader/loader.go, pkg/templates/compile.go, pkg/protocols/javascript/js.go` (critical)
|
||||
### 1. Sandbox Escape: `pkg/js/compiler/pool.go, v2/pkg/protocols/javascript/js.go, v2/pkg/protocols/headless/engine/engine.go, v2/internal/runner/processor.go` (high)
|
||||
|
||||
Unsigned or unverified templates with JavaScript init blocks were compiled and executed; verification results were not bound to content digests, allowing cache reuse of stale verification. This is the root of multiple RCE and signature bypass CVEs.
|
||||
Multiple sandbox escapes across JS require(), eval/ReadFile, headless browser flags, and tengo imports, indicating weak default sandbox boundaries that require consistent -lfa and safe-module enforcement.
|
||||
|
||||
Evidence: 9de96e4dda5a03da963b9ae6582f03ea55791a76, CVE-2024-43405, CVE-2024-27920
|
||||
Evidence: 6f2ade6a9b427c284c15a43445f9c7f055e60e5d, c35162c8ef061a96dd74b54c8f022c1a41cf5833, df55f7a2eb591a7949b5c66a9c54db54aa28f450, cf2a98b03d76327c20fb0183b7f1c849eb692c91, 96049a32fa70aef0483d4180d386a57b3c2d973a
|
||||
|
||||
### 2. Remote Code Execution: `pkg/protocols/code, pkg/protocols, pkg/templates/workflows.go` (high)
|
||||
### 2. Code Execution via Unsigned Template: `pkg/catalog/loader/loader.go, pkg/templates/workflows.go, pkg/templates/templates.go` (critical)
|
||||
|
||||
Code protocol execution sink lacked gating for signature verification and the -code flag; workflows could load unsigned code templates; dynamic runtime values were recursively evaluated as DSL expressions. Multiple independent paths to local code execution.
|
||||
Unsigned templates can execute code through direct loading, workflows, and DAST branches; signature verification was bypassed via cache poisoning and missing guards on specific paths.
|
||||
|
||||
Evidence: 1427b35cdbcb11a63c338700fe8e228ada8ec59d, ccbfb12bd01447ba25f6e755dfb2bee677736da6, e86f38299765b82ad724fdb701557e0eaff3884d
|
||||
Evidence: 9de96e4dda5a03da963b9ae6582f03ea55791a76, 1c440e755a97471c56fb0276ee8a8c4132974645, fac0b7196d2fbca56c8af8ff8e1899c111041a37, e86f38299765b82ad724fdb701557e0eaff3884d, fa56800fcca3c8c8fe8e8dec28584210ca1527de
|
||||
|
||||
### 3. Path Traversal: `pkg/catalog/config, pkg/js/compiler, v2/pkg/protocols/common/generators, v2/internal/runner/update.go` (high)
|
||||
### 3. Auth Bypass: `pkg/templates/compile.go, pkg/templates/signer/tmpl_signer.go, pkg/catalog/loader/loader.go, pkg/protocols/http/request.go` (high)
|
||||
|
||||
Weak lexical prefix checks allowed sibling paths to bypass containment; JS runtime allowed local file reads via require(); payload loading and template updater lacked path canonicalization, enabling arbitrary file read/write and Zip Slip.
|
||||
Signature verification can be bypassed via cached metadata, improper signature extraction, or missing checks on specific load paths, and cookie jar bypass via disabled cookie handling.
|
||||
|
||||
Evidence: 20cbf0bd73893c1270582c9778b6b26a5cb11206, 6f2ade6a9b427c284c15a43445f9c7f055e60e5d, e5154d362af900e83b30fb79028bfab1a7f0c648, f9c214a66f37a00fc29fe64232b690e6285ec59c
|
||||
Evidence: 4534e9cb3034ebe5cc7b0cecabe399b6fb9ce0e4, 0da993afe6d41b4b1b814e8fad23a2acba13c60a, 9bd4db3f74aedd067af568df4f9198a8a090f68b, a976ca8d3f6a18c9ca746c8ce73f28fdcd518642
|
||||
|
||||
### 4. Expression Language Injection: `pkg/protocols/common/expressions` (high)
|
||||
### 4. SSRF: `pkg/js/libs/ldap/ldap.go, v2/internal/runner/proxy.go` (high)
|
||||
|
||||
Template-authored expressions were parsed after placeholder substitution, allowing untrusted response-derived data to be re-evaluated as DSL expressions. Related CVEs confirm multi-step template injection with -env-vars.
|
||||
SSRF via LDAP dialing to internal hosts and via proxy configuration; host allowlist checks were missing or applied to the wrong target (domain vs controller).
|
||||
|
||||
Evidence: d2217320162d5782ca7cb95bef9dda17063818f3, CVE-2026-41645, CVE-2026-41282
|
||||
Evidence: 9def9af7e3f7ebcf5cd561c37b623b9999ee7572, a167e6c57b80f5dc7be231231b62d7c594ee2770, 0e46d3e0410f89e4af41099f5cd38fa7f54ae54f
|
||||
|
||||
### 5. SSRF: `pkg/js/libs/ldap/ldap.go, pkg/js/libs/smb/smb.go` (high)
|
||||
### 5. Path Traversal: `pkg/catalog/config/nucleiconfig.go, v2/internal/runner/update.go, v2/pkg/protocols/common/protocolstate/headless.go` (high)
|
||||
|
||||
LDAP host validation checked the domain instead of the actual target address; SMB dialing lacked host allowlist checks. JS helper libraries bypassed the global network policy.
|
||||
Path traversal via lexical prefix checks, zip-slip in template extraction, and file:// scheme in headless mode; canonicalization and containment checks are required.
|
||||
|
||||
Evidence: a167e6c57b80f5dc7be231231b62d7c594ee2770, a677fca192ee732a7b0a278f725517fa998b7897
|
||||
Evidence: 20cbf0bd73893c1270582c9778b6b26a5cb11206, f9c214a66f37a00fc29fe64232b690e6285ec59c, d3928e080d819f25306bfbaecc399f03516db5aa
|
||||
|
||||
### 6. Sandbox Escape: `v2/pkg/protocols/common/generators/validate.go, v2/internal/runner/processor.go` (high)
|
||||
### 6. Code Execution: `pkg/catalog/loader/loader.go, pkg/templates/workflows.go` (high)
|
||||
|
||||
Go-template payload generators and Tengo scripting in workflows allowed unsafe functions and arbitrary imports, escaping the intended sandbox and enabling system command execution.
|
||||
Code execution via cache poisoning and unverified code templates in workflows; requires -code flag and verified templates.
|
||||
|
||||
Evidence: aefa2717f7d6d1702e3008ac18ebdcc3660dc256, 96049a32fa70aef0483d4180d386a57b3c2d973a
|
||||
Evidence: 9de96e4dda5a03da963b9ae6582f03ea55791a76, e86f38299765b82ad724fdb701557e0eaff3884d
|
||||
|
||||
## Shared Attack Surfaces
|
||||
|
||||
Capabilities guarded by more than one past fix on different code paths. A guard enforced on one entry point is often missing on a sibling; verify each alternate path before assuming it's covered.
|
||||
|
||||
### 1. Privileged-type opt-in for template execution (code, JavaScript, self-contained, file-protocol, DAST)
|
||||
### 1. Template signature verification and unsigned-template opt-in enforcement across all load paths
|
||||
|
||||
- Guard that should hold everywhere: signature verification (Verify, parsed.IsUnsignedJavascriptTemplate), -code flag, parsed.SelfContained flag, parsed.HasFileProtocol() flag, DASTServerToken length >= 16
|
||||
- Entry points: `pkg/catalog/loader/loader.go`, `pkg/templates/compile.go`, `pkg/protocols/code`, `pkg/protocols/javascript/js.go`, `pkg/templates/workflows.go`
|
||||
- Check: Verify the workflow sub-template loader enforces signature verification and the -code flag for code templates, AND the self-contained and file-protocol opt-ins, not just the code-template one. Also check the DAST-mode loader path for the same full set.
|
||||
- Evidence: 9de96e4dda5a03da963b9ae6582f03ea55791a76, fac0b7196d2fbca56c8af8ff8e1899c111041a37, 1427b35cdbcb11a63c338700fe8e228ada8ec59d, 1c440e755a97471c56fb0276ee8a8c4132974645, 63687c2ce07e464f6b38f797d81a8b92597f6ef9, e86f38299765b82ad724fdb701557e0eaff3884d
|
||||
- Guard that should hold everywhere: signature verification (template.Verified, Verify), DisableUnsignedTemplates flag, -code flag for code templates, cache invalidation on verification status/fingerprint/digest
|
||||
- Entry points: `pkg/catalog/loader/loader.go (LoadTemplatesWithTags, DAST branch)`, `pkg/templates/workflows.go (loadTemplates, workflowExecutor)`, `pkg/templates/templates.go (loadTemplates)`, `pkg/templates/compile.go (template.Verified)`, `pkg/templates/signer/tmpl_signer.go (Verify)`
|
||||
- Check: Inspect the DAST and workflow sub-loaders: they must enforce the same DisableUnsignedTemplates and -code checks as the direct loader, and must not rely on cached verification results without re-checking signatures.
|
||||
- Evidence: 9de96e4dda5a03da963b9ae6582f03ea55791a76, 1c440e755a97471c56fb0276ee8a8c4132974645, fac0b7196d2fbca56c8af8ff8e1899c111041a37, e86f38299765b82ad724fdb701557e0eaff3884d, fa56800fcca3c8c8fe8e8dec28584210ca1527de, 4534e9cb3034ebe5cc7b0cecabe399b6fb9ce0e4, 0da993afe6d41b4b1b814e8fad23a2acba13c60a, 9bd4db3f74aedd067af568df4f9198a8a090f68b
|
||||
|
||||
### 2. Local file access sandbox (LFA) across JS runtime, payloads, extractors, headless
|
||||
### 2. Local file access (read/write) outside the sandbox, gated by -lfa flag
|
||||
|
||||
- Guard that should hold everywhere: path canonicalization + prefix check, LfaAllowed flag, template-directory confinement, file:// opt-in, FilesInput opt-in
|
||||
- Entry points: `pkg/js/compiler`, `pkg/js/global/scripts.go`, `v2/pkg/protocols/common/generators`, `pkg/operators/extractors`, `v2/pkg/protocols/headless/engine/page_actions.go`, `pkg/protocols/common/protocolstate/file.go`
|
||||
- Check: Inspect the JS runtime's require() source loader and the headless page-actions path: they must enforce the same canonical path containment and LFA flag as the payload generator and extractor paths.
|
||||
- Evidence: 6f2ade6a9b427c284c15a43445f9c7f055e60e5d, e5154d362af900e83b30fb79028bfab1a7f0c648, e7fb40a41308f6c41267e858d1781eb67f7e0572, f663d1c9cf77da53533bf328750621b4a9fa996d, d3928e080d819f25306bfbaecc399f03516db5aa, 5b89811b906f086e47fdf5357a5ee367f8f1daef
|
||||
- Guard that should hold everywhere: -lfa flag enforcement, path canonicalization and containment (sandbox directory), file:// scheme blocking, allowAllFiles stripping, require() registry rebuild
|
||||
- Entry points: `pkg/js/libs/krbforge/krbforge.go (CreateTicket)`, `pkg/js/libs/mysql/mysql_private.go (sql.Open)`, `pkg/js/compiler/pool.go (require)`, `v2/pkg/protocols/javascript/js.go (ReadFile, eval)`, `v2/pkg/protocols/common/protocolstate/headless.go (file:)`, `v2/pkg/protocols/common/generators/load.go (loadPayloadsFromFile)`
|
||||
- Check: Verify that every JS library and protocol helper (krbforge, mysql, require, headless file://, payload loaders) enforces the -lfa flag and path containment; a sibling path that skips the -lfa check is a sandbox escape.
|
||||
- Evidence: 3a8b65cb45f121fcdf418f60e5177ed9f250c122, 9611926a9a1ca99e8ed554d1b26fbeff60e06368, 6f2ade6a9b427c284c15a43445f9c7f055e60e5d, c35162c8ef061a96dd74b54c8f022c1a41cf5833, d3928e080d819f25306bfbaecc399f03516db5aa, e7fb40a41308f6c41267e858d1781eb67f7e0572
|
||||
|
||||
### 3. Network egress policy (host allowlist) for JS helper protocols and remote template loading
|
||||
### 3. Network policy (host allowlist) enforcement before outbound connections
|
||||
|
||||
- Guard that should hold everywhere: protocolstate.IsHostAllowed, dialers.Fastdialer.Dial/DialTLSWithConfig, remote-template-domain allowlist
|
||||
- Entry points: `pkg/js/libs/ldap/ldap.go`, `pkg/js/libs/smb/smb.go`, `pkg/js/libs/mysql/mysql.go`, `v2/pkg/catalog/loader`
|
||||
- Check: Verify the MySQL DSN and LDAP dial paths check the actual target host (not just a derived domain) and that the remote-template loader applies the domain allowlist before http.Get, matching the SMB dial check.
|
||||
- Evidence: 9def9af7e3f7ebcf5cd561c37b623b9999ee7572, a167e6c57b80f5dc7be231231b62d7c594ee2770, a677fca192ee732a7b0a278f725517fa998b7897, 9611926a9a1ca99e8ed554d1b26fbeff60e06368, d1303797c0e5f88343a1f2ed781955dd086df03d
|
||||
- Guard that should hold everywhere: IsHostAllowed check on the actual target host (controller, not domain), proxy validation and connectivity check, ProxyInternal gate for HTTP_PROXY env
|
||||
- Entry points: `pkg/js/libs/ldap/ldap.go (Dial, newLdapSession)`, `v2/internal/runner/proxy.go (loadProxyServers, os.Setenv HTTP_PROXY)`
|
||||
- Check: Inspect other dialing paths (e.g., HTTP, headless) to ensure they apply the same host allowlist check on the resolved IP/host, not just a user-supplied domain, and that proxy settings cannot redirect traffic to internal hosts.
|
||||
- Evidence: 9def9af7e3f7ebcf5cd561c37b623b9999ee7572, a167e6c57b80f5dc7be231231b62d7c594ee2770, 0e46d3e0410f89e4af41099f5cd38fa7f54ae54f, 3f5c055aca1499642caf4f40e8cf55ad32bd4da9
|
||||
|
||||
### 4. Template expression evaluation boundary (response data vs. template DSL)
|
||||
### 4. Path traversal in file extraction and template path trust boundaries
|
||||
|
||||
- Guard that should hold everywhere: collect expressions before placeholder substitution (FindExpressions), treat runtime values as literal data (expressions.Evaluate)
|
||||
- Entry points: `pkg/protocols/common/expressions`, `pkg/protocols`
|
||||
- Check: Inspect multi-step template processing where HTTP response data flows into expression evaluation: ensure expressions are collected before substitution on every path, including headless and JavaScript protocol steps.
|
||||
- Evidence: d2217320162d5782ca7cb95bef9dda17063818f3, ccbfb12bd01447ba25f6e755dfb2bee677736da6
|
||||
- Guard that should hold everywhere: canonical path containment check (filepath.Rel, prefix), zip-slip validation against template directory, sandbox path prefix validation
|
||||
- Entry points: `pkg/catalog/config/nucleiconfig.go (IsCustomTemplate)`, `v2/internal/runner/update.go (os.OpenFile)`, `v2/pkg/protocols/common/generators/load.go (loadPayloadsFromFile)`
|
||||
- Check: Audit all file-write and file-read paths that accept template-controlled paths (e.g., template extraction, payload loading) to ensure they use canonical containment, not lexical prefix checks, and apply the same sandbox directory restriction.
|
||||
- Evidence: 20cbf0bd73893c1270582c9778b6b26a5cb11206, f9c214a66f37a00fc29fe64232b690e6285ec59c, e7fb40a41308f6c41267e858d1781eb67f7e0572
|
||||
|
||||
## Historical Vulnerability Classes
|
||||
|
||||
| Class | Count |
|
||||
|---|---|
|
||||
| Path Traversal | 10 |
|
||||
| Auth Bypass | 6 |
|
||||
| Remote Code Execution | 3 |
|
||||
| Sandbox Escape | 3 |
|
||||
| SSRF | 2 |
|
||||
| Arbitrary Code Execution | 1 |
|
||||
| Sandbox Escape | 7 |
|
||||
| Auth Bypass | 4 |
|
||||
| Code Execution | 3 |
|
||||
| Dependency Vulnerability | 3 |
|
||||
| Path Traversal | 3 |
|
||||
| SSRF | 3 |
|
||||
| Code Execution via Unsigned Template | 2 |
|
||||
| Arbitrary File Read | 1 |
|
||||
| Code Execution | 1 |
|
||||
| Code Execution / Signature Bypass | 1 |
|
||||
| Cryptographically Weak PRNG | 1 |
|
||||
| Expression Language Injection | 1 |
|
||||
| Cryptographic Weakness | 1 |
|
||||
| Deserialization of Untrusted Data | 1 |
|
||||
| Information Disclosure | 1 |
|
||||
| Insecure File Permissions | 1 |
|
||||
| Insecure Permissions | 1 |
|
||||
| Race Condition | 1 |
|
||||
| SQL Injection | 1 |
|
||||
| Sandbox Bypass | 1 |
|
||||
| Server-Side Request Forgery | 1 |
|
||||
| Signature Bypass | 1 |
|
||||
| Security Misconfiguration | 1 |
|
||||
|
||||
## Fixed-Issue Fingerprints
|
||||
|
||||
These are real security fixes mined from git history. Use them as regression checks.
|
||||
|
||||
- **Code Execution / Signature Bypass** (critical) [CWE-347]: This commit fixes a signature verification bypass that allowed unsigned templates with JavaScript init blocks to be compiled and executed, by binding verification results to content digests and enforcing verification before compilation.
|
||||
- sink: `Template compilation and JavaScript execution of unsigned or unverified templates`
|
||||
- fix: Signature verification enforced before protocol compilation and at request execution; verification result bound to content digest and verifier fingerprint to prevent cache reuse of stale verification.
|
||||
- **Auth Bypass** (high) [CWE-287]: Honors disable-cookie by preventing the shared cookie jar from being applied when cookie handling is disabled, closing a path where cookies could be reused across requests.
|
||||
- sink: `SetCookieJar(input.CookieJar)`
|
||||
- fix: Adds !request.DisableCookie check before applying shared cookie jar
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/a976ca8d3f6a18c9ca746c8ce73f28fdcd518642 (2026-08-25)
|
||||
- **Code Execution** (high) [CWE-347]: This commit fixes a cache poisoning vulnerability where unsigned templates could bypass signature verification and execute malicious code during compilation or request execution.
|
||||
- sink: `template compilation and execution of JavaScript`
|
||||
- fix: cache invalidation based on verification status, verifier fingerprint, and content digest
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/9de96e4dda5a03da963b9ae6582f03ea55791a76 (2026-08-04)
|
||||
- **Auth Bypass** (high) [CWE-347]: Enforce template signature verification for javascript-protocol templates before loading and executing them.
|
||||
- sink: `execution path`
|
||||
- fix: signature check for javascript templates
|
||||
- **Code Execution via Unsigned Template** (high) [CWE-347]: Enforces signature verification on JavaScript templates, preventing execution of unsigned templates that could lead to arbitrary code execution.
|
||||
- sink: `loading unsigned javascript templates into execution paths`
|
||||
- fix: Require signature verification for javascript templates in direct loading and workflows when signing is enforced
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/fac0b7196d2fbca56c8af8ff8e1899c111041a37 (2026-07-03)
|
||||
- **Remote Code Execution** (high) [CWE-94]: Enforce signature verification and the code execution flag (-code) directly within the code protocol execution sink to prevent execution of untrusted templates via indirect SDK/API paths.
|
||||
- sink: `ExecuteWithResults`
|
||||
- fix: gated check for enabled and signature-verified templates
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/1427b35cdbcb11a63c338700fe8e228ada8ec59d (2026-07-03)
|
||||
- **Remote Code Execution** (high) [CWE-94]: Fixes a template-injection risk where dynamic runtime values/extracted payloads were recursively evaluated as DSL expressions instead of being treated as literal data.
|
||||
- sink: `expressions.Evaluate`
|
||||
- fix: template rendering boundary
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/ccbfb12bd01447ba25f6e755dfb2bee677736da6 (2026-06-30)
|
||||
- **Path Traversal** (medium) [CWE-22]: Enforce Nuclei's local file sandbox restrictions on the Kerberos credential cache (ccache) output paths in krbforge helper to prevent arbitrary file writes.
|
||||
- sink: `github.com/Mzack9999/goimpacket/pkg/kerberos.CreateTicket`
|
||||
- fix: path canonicalization + prefix check
|
||||
- **Sandbox Escape** (high) [CWE-22]: Enforces nuclei's local file access policy on krbforge output paths, preventing arbitrary file writes outside the sandbox.
|
||||
- sink: `gpkrb.CreateTicket(cfg) with cfg.OutputFile`
|
||||
- fix: Path normalization into sandbox and denial of outside paths unless -lfa
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/3a8b65cb45f121fcdf418f60e5177ed9f250c122 (2026-06-24)
|
||||
- **SSRF** (medium) [CWE-918]: Enforce the configured network policy and local network restrictions on LDAP/LDAPS/LDAPI protocols prior to dialing hosts.
|
||||
- **SSRF** (high) [CWE-918]: Adds network policy host allowlist checks before LDAP dialing, preventing SSRF to restricted internal hosts.
|
||||
- sink: `dialers.Fastdialer.Dial`
|
||||
- fix: host allowlist
|
||||
- fix: host allowlist check via IsHostAllowed before dialing
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/9def9af7e3f7ebcf5cd561c37b623b9999ee7572 (2026-06-24)
|
||||
- **Arbitrary File Read** (high) [CWE-200]: Enforces local file access sandbox restrictions on MySQL connections by disabling the allowAllFiles option in the DSN unless the local file access (-lfa) flag is set.
|
||||
- sink: `sql.Open`
|
||||
- fix: host allowlist
|
||||
- **Arbitrary File Read** (high) [CWE-22]: Gates MySQL's allowAllFiles option behind the -lfa flag, preventing arbitrary local file reads via LOAD DATA LOCAL INFILE.
|
||||
- sink: `sql.Open("mysql", dsn) with AllowAllFiles=true`
|
||||
- fix: DSN sandbox that strips allowAllFiles unless -lfa is set
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/9611926a9a1ca99e8ed554d1b26fbeff60e06368 (2026-06-19)
|
||||
- **Code Execution** (high) [CWE-347]: Fixes a signature validation bypass that allowed unsigned Code templates to execute arbitrary local commands when run under DAST mode.
|
||||
- sink: `LoadTemplatesWithTags`
|
||||
- fix: signature verification before execution
|
||||
- **Code Execution** (critical) [CWE-94]: Moves the unsigned code-template guard ahead of the DAST branch, preventing unsigned code templates from being loaded and executed under DAST, which previously bypassed the signature check.
|
||||
- sink: `Template loading and execution`
|
||||
- fix: Hoisting unsigned code-template guard before DAST branch to ensure it applies on all load paths
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/1c440e755a97471c56fb0276ee8a8c4132974645 (2026-06-19)
|
||||
- **Path Traversal** (medium) [CWE-22]: Fixes path containment validation vulnerabilities across several filesystem boundaries by replacing weak lexical prefix checks (which allowed sibling paths of trusted directories to bypass security restrictions) with path-aware canonical containment checks.
|
||||
- sink: `filepath.Rel`
|
||||
- fix: path canonicalization + prefix check
|
||||
- **Path Traversal** (high) [CWE-22]: This commit replaces lexical prefix checks with canonical path containment checks across multiple filesystem trust boundaries, preventing path traversal where sibling paths were incorrectly treated as trusted.
|
||||
- sink: `IsCustomTemplate`
|
||||
- fix: path canonicalization and containment check
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/20cbf0bd73893c1270582c9778b6b26a5cb11206 (2026-06-04)
|
||||
- **Path Traversal** (high) [CWE-22]: Fixes a sandbox bypass in the JavaScript runtime compiler and template runner that allowed local file access and module importing regardless of restrictions by canonicalizing symlinks and clearing cached modules between runtime executions.
|
||||
- sink: `require.DefaultSourceLoader`
|
||||
- fix: path canonicalization + prefix check
|
||||
- **Information Disclosure** (medium) [CWE-200]: This commit fixes a data leakage issue in workflows where sibling state could leak across concurrent executions, and race requests ignored dynamic values, by syncing context and cloning inputs.
|
||||
- sink: `syncWorkflowInputToTemplateCtx`
|
||||
- fix: context synchronization and input cloning
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/805bcfe76f3629dbe71021795d4ce865406c1555 (2026-05-08)
|
||||
- **Sandbox Escape** (high) [CWE-284]: Fixes sandbox escape where require() bypassed local file access restrictions and reused privileged module cache across executions.
|
||||
- sink: `require() using default host filesystem loader`
|
||||
- fix: Rebuild require registry per execution and enforce allow-local-file-access sandbox rules
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/6f2ade6a9b427c284c15a43445f9c7f055e60e5d (2026-04-10)
|
||||
- **Expression Language Injection** (high) [CWE-917]: Fixes an expression injection vulnerability by parsing and collecting template-authored expressions before placeholder substitution, preventing untrusted response-derived data from being re-evaluated as DSL expressions.
|
||||
- sink: `pkg/protocols/common/expressions/expressions.go`
|
||||
- fix: collect expressions before placeholder substitution
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/d2217320162d5782ca7cb95bef9dda17063818f3 (2026-03-16)
|
||||
- **Cryptographically Weak PRNG** (medium) [CWE-338]: Replaces the cryptographically insecure math/rand PRNG with crypto/rand in JavaScript helper functions to prevent predictable token generation.
|
||||
- sink: `math/rand.Intn`
|
||||
- fix: crypto/rand replacement
|
||||
- **Cryptographic Weakness** (medium) [CWE-338]: Replaces predictable math/rand PRNG with cryptographically secure crypto/rand in JS helper functions.
|
||||
- sink: `rand.Intn(255) / rand.Int63()`
|
||||
- fix: Replace predictable math/rand with crypto/rand
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/44ef9ce34ffcfdfca3a96626abc73d99dd0e83ab (2026-03-15)
|
||||
- **SQL Injection** (medium) [CWE-89]: Fixes a path-based fuzzing component encoding logic issue that prevented successful detection and isolated fuzz testing of SQL Injection paths.
|
||||
- sink: `pkg/testutils/fuzzplayground/sqli_test.go`
|
||||
- fix: path component fuzzing fix
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/19247ae74b99c5871b62d7c465a63edb446ceba1 (2025-08-25)
|
||||
- **Auth Bypass** (high) [CWE-341]: Fix a race condition in concurrent Nuclei engine executions where the global `LfaAllowed` state could be shared across different execution contexts, allowing unauthorized local file access.
|
||||
- sink: `protocolstate.NormalizePath`
|
||||
- fix: path canonicalization + prefix check
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/5b89811b906f086e47fdf5357a5ee367f8f1daef (2025-07-18)
|
||||
- **Auth Bypass** (medium) [CWE-307]: Enforce a minimum length restriction of 16 characters on the DAST server token option to prevent weak or easily guessable authentication credentials.
|
||||
- sink: `options.DASTServerToken`
|
||||
- fix: minimum token length constraint
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/5f0b7eb19b6cf160c8b277c9774f8e624d2069cf (2025-02-13)
|
||||
- **Auth Bypass** (medium) [CWE-862]: Disables self-contained and file-protocol templates by default to enforce safer execution boundaries and prevent unauthorized local file access or unexpected outbound connections unless explicitly enabled.
|
||||
- sink: `store.config.ExecutorOptions.Options`
|
||||
- fix: host allowlist
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/63687c2ce07e464f6b38f797d81a8b92597f6ef9 (2024-11-19)
|
||||
- **Race Condition** (medium) [CWE-362]: Fixes a race condition in the HTTP request execution where shared connection configurations (including cookie jars and timeouts) were concurrently modified, which could lead to cookie leakage across concurrent sessions.
|
||||
- sink: `connConfiguration.Connection.SetCookieJar`
|
||||
- fix: struct cloning
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/3064788d35a8ec8623eb3a54eca920ea289f1259 (2024-08-19)
|
||||
- **Signature Bypass** (high) [CWE-347]: Fixes a template signature verification bypass by ensuring the digest/signature is strictly parsed from the end of the template rather than extracted via an arbitrary regex match anywhere in the file.
|
||||
- **Auth Bypass** (high) [CWE-347]: Caching signature verification results without cryptographic re-check allows a metadata index compromise to bypass template signature verification.
|
||||
- sink: `template.Verified, template.TemplateVerifier`
|
||||
- fix: cached verification data reuse
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/4534e9cb3034ebe5cc7b0cecabe399b6fb9ce0e4 (2026-01-21)
|
||||
- **Race Condition** (medium) [CWE-362]: Fixes a race condition in workflow execution caused by shared context callbacks, which could lead to data corruption or crashes; the primary impact is reliability, not a direct security breach.
|
||||
- sink: `shared context callbacks in workflow execution`
|
||||
- fix: fix race condition by making workflow execution sequential
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/24d1f58682027e23cb21a299a170abd2bdf20dc2 (2025-12-05)
|
||||
- **Auth Bypass** (high) [CWE-347]: Fixes template signature verification by centralizing signature extraction and requiring the signature to be at the end of the template, preventing bypass.
|
||||
- sink: `Verify`
|
||||
- fix: signature extraction logic refinement
|
||||
- fix: signature extraction and verification
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/0da993afe6d41b4b1b814e8fad23a2acba13c60a (2024-08-19)
|
||||
- **Auth Bypass** (high) [CWE-566]: Fixes a Go loop variable reference bug in FileAuthProvider that causes all registered domains to use the credentials of the last secret entry in the configuration list.
|
||||
- sink: `f.compiled[domain]`
|
||||
- fix: loop variable copy
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/350fa4c10c43c0888561dfe1ee5fa473d5023f73 (2024-08-05)
|
||||
- **Arbitrary Code Execution** (high) [CWE-94]: Enforces validation checks for code-execution templates referenced within workflows, preventing unauthorized local code execution via untrusted workflows.
|
||||
- sink: `workflowTemplates = append(workflowTemplates, template)`
|
||||
- fix: verification and flag checks before loading code templates in workflows
|
||||
- **Dependency Vulnerability** (medium) [CWE-1104]: Upgrades Alpine base image from 3.18.5 to 3.18.6 to fix known security vulnerabilities in bundled packages.
|
||||
- sink: `FROM alpine:3.18.5`
|
||||
- fix: Upgrade base image to patched version
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/fcd5c6b111ddcb746525f88b7d0bf4b4abbc2529 (2024-03-15)
|
||||
- **Dependency Vulnerability** (medium) [CWE-1104]: Upgrades Alpine base image from 3.18.2 to 3.18.5 to fix known security vulnerabilities in bundled packages.
|
||||
- sink: `FROM alpine:3.18.2`
|
||||
- fix: Upgrade base image to patched version
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/d93b4a01df9c218a36dd2fccfb1c2c1f96b181fe (2024-03-15)
|
||||
- **Dependency Vulnerability** (high) [CWE-1035]: Updates go-git to fix known vulnerabilities (CVE-2023-49569, CVE-2023-49568), which could be exploited when processing untrusted git data.
|
||||
- sink: `go-git library usage`
|
||||
- fix: dependency upgrade to patched version
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/1700cdf4e23128a358066364062246b58c1c013f (2024-03-14)
|
||||
- **Code Execution via Unsigned Template** (high) [CWE-347]: Introduces a flag to disable unsigned templates, preventing execution of unverified templates that could lead to arbitrary code execution.
|
||||
- sink: `loading unsigned templates when -dut flag is set`
|
||||
- fix: Adds -disable-unsigned-templates flag to skip unsigned templates and report them separately
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/fa56800fcca3c8c8fe8e8dec28584210ca1527de (2024-03-14)
|
||||
- **Auth Bypass** (medium) [CWE-347]: This commit introduces a flag to skip unsigned templates, closing a path where an attacker could execute untrusted templates by bypassing signature verification.
|
||||
- sink: `parsed.Verified`
|
||||
- fix: skip unverified templates when DisableUnsignedTemplates is set
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/9bd4db3f74aedd067af568df4f9198a8a090f68b (2024-03-10)
|
||||
- **Code Execution** (high) [CWE-94]: Prevents execution of unverified code templates in workflows, mitigating code execution risk.
|
||||
- sink: `template.RequestsCode`
|
||||
- fix: require -code flag and verified templates for code templates in workflows
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/e86f38299765b82ad724fdb701557e0eaff3884d (2024-03-01)
|
||||
- **SSRF** (high) [CWE-918]: Fixes a security bypass in the LDAP client where host validation checked the domain instead of the actual target domain controller address, allowing SSRF to restricted networks.
|
||||
- sink: `c.newLdapSession`
|
||||
- fix: validate target host instead of domain name
|
||||
- **SSRF** (high) [CWE-918]: Fixes SSRF by validating the actual LDAP controller host against the network policy instead of the domain name.
|
||||
- sink: `newLdapSession`
|
||||
- fix: host allowlist check on controller instead of domain
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/a167e6c57b80f5dc7be231231b62d7c594ee2770 (2024-01-20)
|
||||
- **Server-Side Request Forgery** (high) [CWE-918]: Fixes an SSRF/network policy bypass in SMB helper functions by introducing `protocolstate.IsHostAllowed` checks prior to dialing.
|
||||
- sink: `protocolstate.Dialer.Dial`
|
||||
- fix: host allowlist
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/a677fca192ee732a7b0a278f725517fa998b7897 (2024-01-18)
|
||||
- **Path Traversal** (high) [CWE-22]: Removes the insecure file extraction `to` feature from extractors and enforces local file access (LFA) boundary checks on headless screenshot file writes to prevent arbitrary file write vulnerabilities via malicious templates.
|
||||
- sink: `os.OpenFile`
|
||||
- fix: path canonicalization + prefix check
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/f663d1c9cf77da53533bf328750621b4a9fa996d (2024-01-05)
|
||||
- **Sandbox Bypass** (high) [CWE-693]: Disables the 'eval' function and strengthens file validation within the JavaScript sandbox environment to prevent sandbox escape and unauthorized code execution.
|
||||
- sink: `eval`
|
||||
- fix: disable eval function
|
||||
- **Sandbox Escape** (high) [CWE-94]: Disables eval and tightens file read validation in the JavaScript sandbox, preventing template-driven code execution and file access outside the sandbox.
|
||||
- sink: `eval function and file read in sandbox mode`
|
||||
- fix: Disable eval function and rewrite file validation in sandbox mode
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/c35162c8ef061a96dd74b54c8f022c1a41cf5833 (2023-10-13)
|
||||
- **Path Traversal** (high) [CWE-22]: Restricts local file input actions and file:// protocol access in headless browser protocols unless explicitly enabled via command-line flags.
|
||||
- sink: `p.FilesInput`
|
||||
- fix: host allowlist
|
||||
- **Path Traversal** (high) [CWE-22]: Adds network policy and blocks file:// protocol in headless mode unless -lfa is set, preventing local file disclosure.
|
||||
- sink: `file:// protocol access in headless browser`
|
||||
- fix: Network policy validation and file:// scheme blocking when local file access is disabled
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/d3928e080d819f25306bfbaecc399f03516db5aa (2023-08-25)
|
||||
- **Path Traversal** (medium) [CWE-22]: Implements a file-access sandbox to prevent malicious templates from reading arbitrary local files via payloads by restricting access to the template directory unless explicitly allowed.
|
||||
- sink: `os.Open`
|
||||
- fix: path canonicalization + prefix check
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/e5154d362af900e83b30fb79028bfab1a7f0c648 (2023-07-14)
|
||||
- **Auth Bypass** (high) [CWE-347]: Introduces a code protocol for template execution alongside a cryptographic signature verification mechanism (RSA/ECDSA) to ensure that only signed, trusted templates can execute local commands or scripts.
|
||||
- sink: `v2/pkg/protocols/code/code.go`
|
||||
- fix: introducing command + trivial trust store mechanism
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/a7fb15d0bd548257fe70af3cdddd469f21a75e6a (2023-06-09)
|
||||
- **Path Traversal** (medium) [CWE-22]: Introduces a sandbox restriction that validates and confines template payload file paths to the template directory, preventing arbitrary local file reads when safe template execution is enabled.
|
||||
- sink: `generator.loadPayloadsFromFile(pt)`
|
||||
- fix: path canonicalization + prefix check
|
||||
- **Deserialization of Untrusted Data** (medium) [CWE-502]: Fixes JSON deserialization by adding explicit UnmarshalJSON methods and correcting tags, preventing type confusion and potential logic bypasses.
|
||||
- sink: `UnmarshalJSON on user-controlled fields like Type and Name`
|
||||
- fix: Adds proper UnmarshalJSON implementations and corrects json tags to prevent type confusion
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/85090b75317977bb9301444c9dfd8e4c2dba667a (2023-02-07)
|
||||
- **Sandbox Escape** (medium) [CWE-22]: The commit adds sandbox mode that restricts payload file paths to the template directory, preventing arbitrary file reads when sandbox is enabled.
|
||||
- sink: `loadPayloadsFromFile`
|
||||
- fix: path prefix validation against template directory and template path directory
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/e7fb40a41308f6c41267e858d1781eb67f7e0572 (2022-11-16)
|
||||
- **Insecure File Permissions** (medium) [CWE-732]: Restricts the file permissions of written debug data files from world-writable/readable (os.ModePerm) to owner-readable/writable (0644).
|
||||
- **Insecure File Permissions** (low) [CWE-732]: Fixes overly permissive file permissions on debug output files by changing from os.ModePerm (0777) to 0644.
|
||||
- sink: `os.OpenFile`
|
||||
- fix: restrict file permissions to 0644
|
||||
- fix: restrict file mode to 0644
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/af4854f90de728bb08689d5cae3dec4acfdb6b9a (2022-06-20)
|
||||
- **Path Traversal** (medium) [CWE-22]: Sanitizes special and directory traversal characters from generated markdown filenames to prevent arbitrary file write/path traversal.
|
||||
- sink: `os.OpenFile`
|
||||
- fix: character replacement/sanitization
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/e510bc207f4336058f7b1536ddbb3942b394b05c (2022-04-06)
|
||||
- **Remote Code Execution** (high) [CWE-829]: Implements a domain restriction mechanism (remote-template-domain allowlist) to prevent loading and executing arbitrary untrusted remote templates from arbitrary hostnames.
|
||||
- sink: `http.Get`
|
||||
- fix: host allowlist
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/d1303797c0e5f88343a1f2ed781955dd086df03d (2022-01-12)
|
||||
- **Information Disclosure** (medium) [CWE-532]: Prevents disclosure of sensitive AWS credentials by pruning internal-only signing variables (aws-id and aws-secret) from the event metadata payload.
|
||||
- sink: `eventcreator.CreateEventWithAdditionalOptions`
|
||||
- fix: output pruning
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/c26a1ac21c32f716635738387d5e7d369d7538d1 (2022-01-09)
|
||||
- **Path Traversal** (medium) [CWE-22]: Replaces filepath.Join with path.Join when processing HTTP request paths to prevent platform-specific path manipulation issues on Windows.
|
||||
- sink: `filepath.Join`
|
||||
- fix: replace filepath with path
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/d15298648527f4edd1015e077201551b3a6eab06 (2021-12-16)
|
||||
- **Sandbox Escape** (low) [CWE-250]: Enables Chromium's security sandbox by default in headless mode, disabling it only when running as root on Linux where it is strictly required.
|
||||
- sink: `chromeLauncher`
|
||||
- fix: conditional sandbox disablement based on OS and user privileges
|
||||
- **Security Misconfiguration** (medium) [CWE-16]: Prevents setting HTTP_PROXY environment variable unless explicitly enabled, avoiding potential proxy hijacking.
|
||||
- sink: `os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String())`
|
||||
- fix: Added check for options.ProxyInternal before setting HTTP_PROXY environment variable
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/3f5c055aca1499642caf4f40e8cf55ad32bd4da9 (2022-03-09)
|
||||
- **Sandbox Escape** (high) [CWE-250]: The commit restricts the use of the --no-sandbox flag to Linux root users, preventing sandbox escape vulnerabilities in other environments.
|
||||
- sink: `NoSandbox(true)`
|
||||
- fix: Conditionally disable sandbox only on Linux as root
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/df55f7a2eb591a7949b5c66a9c54db54aa28f450 (2021-12-05)
|
||||
- **Path Traversal** (high) [CWE-22]: Fixes a Zip Slip (Path Traversal) vulnerability in the template updater by validating and sanitizing paths before writing unzipped files.
|
||||
- **Path Traversal** (high) [CWE-22]: This commit fixes a zip-slip path traversal vulnerability by validating the extracted file path against the configured template directory.
|
||||
- sink: `os.OpenFile`
|
||||
- fix: path validation in calculateTemplateAbsolutePath
|
||||
- fix: path canonicalization and prefix check
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/f9c214a66f37a00fc29fe64232b690e6285ec59c (2021-11-25)
|
||||
- **Insecure Permissions** (medium) [CWE-732]: Fixes an insecure file and directory creation permissions vulnerability by replacing overly permissive permissions (0777/ModePerm) with restrictive defaults (0644 for files, 0755 for directories).
|
||||
- sink: `os.OpenFile,ioutil.WriteFile,os.MkdirAll`
|
||||
- fix: restrictive file/directory permissions (0644/0755)
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/4bccb6cf8a830724b3bda624cb3bfe43eff31683 (2021-11-24)
|
||||
- **Sandbox Escape** (high) [CWE-116]: Introduces payload validation for template generators to restrict execution of unsafe functions within Nuclei templates.
|
||||
- sink: `Go-template compiler/executor`
|
||||
- fix: input validation / payload sandbox checks
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/aefa2717f7d6d1702e3008ac18ebdcc3660dc256 (2020-12-29)
|
||||
- **Sandbox Escape** (high) [CWE-265]: Restricts the Tengo scripting engine imports in workflows to a safe subset of standard library modules, preventing arbitrary system command execution.
|
||||
- sink: `tengo.NewScript`
|
||||
- fix: import allowlist restriction
|
||||
- **SSRF** (high) [CWE-918]: Adds validation and connectivity checks for SOCKS5 proxies before launching a scan, preventing SSRF through proxy configuration.
|
||||
- sink: `Proxy connection validation and loading`
|
||||
- fix: Proxy server validation and connectivity check before scan launch
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/0e46d3e0410f89e4af41099f5cd38fa7f54ae54f (2021-11-10)
|
||||
- **Sandbox Escape** (high) [CWE-284]: This commit adds a sandbox flag that, when enabled, restricts the tengo scripting engine to safe modules, preventing sandbox escape and arbitrary command execution.
|
||||
- sink: `stdlib.GetModuleMap(stdlib.AllModuleNames()...)`
|
||||
- fix: Added sandbox flag to restrict imports to safe modules when sandbox is enabled
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/cf2a98b03d76327c20fb0183b7f1c849eb692c91 (2020-12-13)
|
||||
- **Sandbox Escape** (high) [CWE-284]: This commit restricts the tengo scripting engine to a safe module list, preventing sandbox escape and arbitrary command execution via workflow templates.
|
||||
- sink: `stdlib.GetModuleMap(stdlib.AllModuleNames()...)`
|
||||
- fix: Restricted tengo script imports to a safe module list
|
||||
- commit: https://github.com/projectdiscovery/nuclei/commit/96049a32fa70aef0483d4180d386a57b3c2d973a (2020-12-13)
|
||||
|
||||
## Known CVEs
|
||||
|
||||
- **CVE-2026-41645** (medium 5.3) CWE-94
|
||||
- Nuclei 3.0.0 to < 3.8.0 contains an expression injection vulnerability caused by reuse of HTTP response data with helper/function syntax in multi-step templates, letting malicious target servers execu...
|
||||
- **CVE-2026-41646** (medium 5.5) CWE-284
|
||||
- Nuclei 3.0.0 to < 3.8.0 contains a local file read vulnerability caused by JavaScript protocol runtime allowing require() to access local .js and .json files, letting attackers read restricted files, ...
|
||||
- **CVE-2026-41282** (medium 4.0) CWE-94
|
||||
- ProjectDiscovery Nuclei < 3.8.0 contains a template injection caused by unsafe use of -env-vars in multi-step templates against untrusted targets, letting attackers inject DSL expressions, exploit req...
|
||||
- **CVE-2024-43405** (high 7.4) CWE-78
|
||||
- Nuclei versions prior to 3.3.2 contain a signature verification bypass caused by discrepancies in newline handling between the verification process and YAML parser, letting attackers inject malicious ...
|
||||
- **CVE-2024-27920** (high 7.4) CWE-78
|
||||
- projectdiscovery/nuclei v3 contains a vulnerability caused by execution of unsigned code templates through workflows, letting malicious code run on the user's system, exploit requires using custom wor...
|
||||
- **CVE-2023-37896** (high 7.5) CWE-22
|
||||
- Nuclei prior to version 2.9.9 contains a sandbox sanitization issue caused by relative path handling in payload loading, letting arbitrary files be read on the filesystem when used as Go SDK, exploit ...
|
||||
|
||||
---
|
||||
_Fixed-issue fingerprints and CVE matches are derived from real data; LLM-written prose is advisory and should be verified against the cited commits._
|
||||
@@ -1477,6 +1477,19 @@ DisableCookie is an optional setting that disables cookie reuse
|
||||
|
||||
<div class="dd">
|
||||
|
||||
<code>disable-http-cache</code> <i>bool</i>
|
||||
|
||||
</div>
|
||||
<div class="dt">
|
||||
|
||||
DisableHTTPCache turns off HTTP caching for this request. It cannot turn caching on when -http-cache is unset.
|
||||
|
||||
</div>
|
||||
|
||||
<hr />
|
||||
|
||||
<div class="dd">
|
||||
|
||||
<code>read-all</code> <i>bool</i>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -450,6 +450,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.BoolVarP(&options.DisableHTTPProbe, "no-httpx", "nh", false, "disable httpx probing for non-url input"),
|
||||
flagSet.BoolVar(&options.PreflightPortScan, "preflight-portscan", false, "run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)"),
|
||||
flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "disable stdin processing"),
|
||||
flagSet.BoolVar(&options.EnableHTTPCache, "http-cache", false, "enable HTTP cache (RFC 9111) for HTTP requests"),
|
||||
)
|
||||
|
||||
flagSet.CreateGroup("headless", "Headless",
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
# Nuclei Runtime Reuse Milestone 1 Implementation Plan
|
||||
|
||||
> Execute this plan test-first. Do not begin scheduler rechunking until the baseline and worker-runtime gates at the end of this plan pass.
|
||||
|
||||
**Goal:** Remove confirmed repeated parsing/compilation and unbounded per-chunk compiled-cache retention from Aurora's Nuclei execution path, while producing trustworthy before/after timing, allocation, memory, and coverage evidence.
|
||||
|
||||
**Architecture:** A long-lived worker owns the parsed template cache and safe protocol infrastructure. Each concurrent scan execution receives a private parser with a private compiled cache, while sharing only immutable parsed template source. The execution cache is explicitly released at execution completion. Aurora's rate-limited batch loop uses the same ownership model and stops before starting another batch after cancellation.
|
||||
|
||||
**Tech stack:** Go, Nuclei SDK/core, Aurora worker, Prometheus, Go benchmarks, race detector, deterministic local HTTP/TLS fixtures.
|
||||
|
||||
**Design reference:** `docs/superpowers/specs/2026-09-08-aurora-nuclei-performance-design.md` in the Aurora `codex/aurora-nuclei-performance` worktree.
|
||||
|
||||
## Task 1: Add an explicit Nuclei execution-parser lifecycle
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `pkg/templates/parser.go`
|
||||
- Modify: `pkg/templates/parser_purge_test.go`
|
||||
- Modify: `pkg/templates/compile_bench_test.go`
|
||||
|
||||
### Step 1: Write failing lifecycle tests
|
||||
|
||||
Add tests proving that an execution parser:
|
||||
|
||||
- shares the parent's parsed cache;
|
||||
- starts with an empty, private compiled cache;
|
||||
- preserves parser validation/strictness settings;
|
||||
- can purge its compiled cache without purging the parent's parsed cache;
|
||||
- does not expose its compiled entries to a sibling execution parser.
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./pkg/templates -run 'TestParser.*Execution|TestParserPurgeCompiled' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL because the explicit execution-parser and compiled-only purge APIs do not exist.
|
||||
|
||||
### Step 2: Implement the minimum lifecycle API
|
||||
|
||||
Add:
|
||||
|
||||
```go
|
||||
func NewExecutionParser(parent *Parser) *Parser
|
||||
func (p *Parser) PurgeCompiled()
|
||||
```
|
||||
|
||||
`NewExecutionParser` must share only `parsedTemplatesCache`, copy parser behaviour flags, and allocate a new compiled cache. Reject or safely handle a nil parent. `PurgeCompiled` must never clear the shared parsed cache.
|
||||
|
||||
### Step 3: Run focused tests and race tests
|
||||
|
||||
```bash
|
||||
go test ./pkg/templates -run 'TestParser.*Execution|TestParserPurge' -count=1
|
||||
go test -race ./pkg/templates -run 'TestParser.*Execution|TestParserPurge' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
### Step 4: Extend the existing cache benchmark
|
||||
|
||||
Update `BenchmarkParseAcrossEngineLocalCaches` to use the public execution-parser lifecycle. Report allocation and timing results before and after the change without claiming scan-level improvement from this microbenchmark.
|
||||
|
||||
```bash
|
||||
go test ./pkg/templates -run '^$' -bench BenchmarkParseAcrossEngineLocalCaches -benchmem -count=5
|
||||
```
|
||||
|
||||
### Step 5: Commit
|
||||
|
||||
```bash
|
||||
git add pkg/templates/parser.go pkg/templates/parser_purge_test.go pkg/templates/compile_bench_test.go
|
||||
git commit -m "feat(templates): add execution-local parser lifecycle"
|
||||
```
|
||||
|
||||
## Task 2: Make the thread-safe Nuclei SDK reuse parsed templates safely
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `lib/multi.go`
|
||||
- Modify: `lib/multi_internal_test.go`
|
||||
- Modify: `lib/multi_bench_test.go`
|
||||
- Verify: `lib/result_callback_test.go`
|
||||
|
||||
### Step 1: Write failing SDK ownership tests
|
||||
|
||||
Test `createEphemeralObjects` directly and prove:
|
||||
|
||||
- each call receives a distinct execution parser;
|
||||
- both parsers share the base parsed cache but not a compiled cache;
|
||||
- caching is enabled within that isolated execution;
|
||||
- closing ephemeral objects clears that execution's compiled cache;
|
||||
- base and sibling caches remain intact;
|
||||
- concurrent per-call output callbacks remain isolated.
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./lib -run 'TestEphemeral.*Parser|TestThreadSafe.*Callback' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL because the current code reuses the base parser and sets `DoNotCache=true`.
|
||||
|
||||
### Step 2: Implement execution-local parser ownership
|
||||
|
||||
Create an execution parser from the long-lived engine parser in `createEphemeralObjects`, store its ownership in `unsafeOptions`, set it on `ExecutorOptions`, and allow caching inside the private execution cache. In `closeEphemeralObjects`, purge only that private compiled cache before removing inherited references.
|
||||
|
||||
Do not share compiled request objects across concurrent executions. Do not move output writers, credentials, variables, global matchers, workflow state, or rate limiters into the shared parser.
|
||||
|
||||
### Step 3: Run correctness and race tests
|
||||
|
||||
```bash
|
||||
go test ./lib -run 'TestEphemeral.*Parser|TestThreadSafe|TestResult' -count=1
|
||||
go test -race ./lib -run 'TestEphemeral.*Parser|TestThreadSafe|TestResult' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS with no race reports.
|
||||
|
||||
### Step 4: Capture SDK benchmark evidence
|
||||
|
||||
Run the existing repeated-execution benchmark on the parent revision and candidate revision with identical fixture and machine settings:
|
||||
|
||||
```bash
|
||||
go test ./lib -run '^$' -bench BenchmarkExecuteNucleiWithOptsCtx -benchmem -count=10
|
||||
```
|
||||
|
||||
Compare with `benchstat`. Record sec/op, B/op, and allocs/op. This is an SDK setup/execute benchmark, not yet an Aurora end-to-end result.
|
||||
|
||||
### Step 5: Commit
|
||||
|
||||
```bash
|
||||
git add lib/multi.go lib/multi_internal_test.go lib/multi_bench_test.go
|
||||
git commit -m "perf(lib): reuse parsed templates across isolated executions"
|
||||
```
|
||||
|
||||
## Task 3: Add Aurora phase and terminal-reason observability
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `internal/agent/metrics/metrics.go`
|
||||
- Modify: `internal/agent/nuclei/nuclei.go`
|
||||
- Modify: `internal/agent/tasks.go`
|
||||
- Modify: `internal/agent/tasks_test.go` or the nearest existing scan-settlement test
|
||||
- Add: `internal/agent/nuclei/execution_metrics_test.go`
|
||||
|
||||
### Step 1: Write failing metric/terminal-reason tests
|
||||
|
||||
Tests must distinguish:
|
||||
|
||||
- Aurora chunk execution deadline;
|
||||
- outer worker lifecycle timeout/cancellation;
|
||||
- Nuclei core/batch deadline;
|
||||
- target network timeout counted as an executed scan attempt rather than an orchestration timeout;
|
||||
- normal completion and non-deadline execution failure.
|
||||
|
||||
Avoid new high-cardinality labels such as scan ID, target, or template ID.
|
||||
|
||||
### Step 2: Add bounded-cardinality metrics
|
||||
|
||||
Add counters/histograms for terminal reason and phase duration using fixed labels such as stream, phase, and reason. Time at least template discovery/load, compile/load-store, execute, output flush, and cleanup. Emit the same reason in structured logs.
|
||||
|
||||
### Step 3: Verify
|
||||
|
||||
```bash
|
||||
GOWORK=off go test ./internal/agent/nuclei ./internal/agent -run 'Test.*ExecutionMetric|Test.*Settlement|Test.*Deadline' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
### Step 4: Commit
|
||||
|
||||
```bash
|
||||
git add internal/agent/metrics/metrics.go internal/agent/nuclei internal/agent/tasks.go internal/agent/*test.go
|
||||
git commit -m "feat(agent): expose nuclei execution phases and terminal reasons"
|
||||
```
|
||||
|
||||
## Task 4: Bound Aurora rate-limited batch memory and honor cancellation
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `internal/agent/nuclei/nuclei.go`
|
||||
- Add: `internal/agent/nuclei/rate_limited_batches_test.go`
|
||||
|
||||
### Step 1: Extract a testable sequential batch executor
|
||||
|
||||
Extract only the rate-limited batch loop behind a small internal function. Pass the loader/engine execution seam as a function so unit tests do not need live network services or a full Nuclei corpus.
|
||||
|
||||
### Step 2: Write failing behavioural tests
|
||||
|
||||
Cover only the confirmed boundaries:
|
||||
|
||||
- cancellation before a batch prevents that batch and every later batch from loading or executing;
|
||||
- cancellation during a batch prevents the next batch from starting;
|
||||
- each batch gets a private compiled cache backed by the process parsed cache;
|
||||
- the previous batch compiled cache is empty/released before the next batch starts;
|
||||
- every template path appears exactly once when no cancellation occurs;
|
||||
- private templates/workflows retain the existing execute-once semantics.
|
||||
|
||||
### Step 3: Implement the bounded lifecycle
|
||||
|
||||
Before each batch, check `context.Cause(ctx)`. Create an execution parser with a private compiled cache for the batch, execute it, and explicitly purge the compiled cache on every exit path. Remove forced `runtime.GC()` from the hot loop only after the lifecycle tests prove cached objects are no longer retained.
|
||||
|
||||
Keep the current batch size initially. Changing batch size and cache ownership simultaneously would make the performance result ambiguous.
|
||||
|
||||
### Step 4: Verify
|
||||
|
||||
```bash
|
||||
GOWORK=off go test ./internal/agent/nuclei -run 'TestRateLimitedBatch' -count=1
|
||||
GOWORK=off go test -race ./internal/agent/nuclei -run 'TestRateLimitedBatch' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
### Step 5: Commit
|
||||
|
||||
```bash
|
||||
git add internal/agent/nuclei/nuclei.go internal/agent/nuclei/rate_limited_batches_test.go
|
||||
git commit -m "fix(agent): bound nuclei batch cache lifetime"
|
||||
```
|
||||
|
||||
## Task 5: Build the deterministic milestone benchmark
|
||||
|
||||
**Files:**
|
||||
|
||||
- Add: `internal/agent/nuclei/tests/perfserver/server.go`
|
||||
- Add: `internal/agent/nuclei/runtime_benchmark_test.go`
|
||||
- Add: `scripts/scan-performance/run-milestone-1.sh`
|
||||
- Add: `docs/benchmarks/aurora-nuclei-milestone-1.md`
|
||||
|
||||
### Step 1: Implement deterministic fixtures
|
||||
|
||||
Provide fixed fast, 100-ms, 500-ms, timeout, connection-reset, TLS, redirect, and multi-request endpoints. Track accepted connections and request identities. Use fixed seeds and local-only addresses.
|
||||
|
||||
### Step 2: Define the milestone coverage oracle
|
||||
|
||||
For this milestone, generate a seeded template/target manifest and compare expected server-observed attempts with worker terminal accounting. A target network timeout counts as a terminal attempt; missing or duplicated attempts fail. Full workflow/payload canonical-ledger support remains Workstream 1's next milestone and is not claimed here.
|
||||
|
||||
### Step 3: Capture baseline and candidate
|
||||
|
||||
Run at least five cold and ten warm repetitions using the same rate limit, template set, and concurrency. Capture:
|
||||
|
||||
- complete/missing/duplicate attempts;
|
||||
- full-run wall time and attempts/second;
|
||||
- process CPU time;
|
||||
- peak RSS and RSS byte-seconds;
|
||||
- heap allocations and GC count/pause;
|
||||
- connections created and reused;
|
||||
- compiled/parsed cache counts after each batch and run;
|
||||
- Aurora versus target timeout reasons.
|
||||
|
||||
### Step 4: Apply the milestone gate
|
||||
|
||||
Proceed to deadline-fit scheduler work only if:
|
||||
|
||||
- the candidate has zero missing or duplicate attempts;
|
||||
- no Aurora/core deadline expires in the deterministic matrix;
|
||||
- repeated-run compiled-cache size is bounded by the active batch/execution;
|
||||
- wall time and attempts/second do not regress;
|
||||
- CPU-seconds and RSS byte-seconds improve.
|
||||
|
||||
### Step 5: Commit evidence
|
||||
|
||||
```bash
|
||||
git add internal/agent/nuclei/tests/perfserver internal/agent/nuclei/runtime_benchmark_test.go scripts/scan-performance docs/benchmarks/aurora-nuclei-milestone-1.md
|
||||
git commit -m "test(agent): add nuclei runtime performance benchmark"
|
||||
```
|
||||
|
||||
## Task 6: Milestone verification and handoff
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./pkg/templates ./lib -count=1
|
||||
go test -race ./pkg/templates ./lib -count=1
|
||||
GOWORK=off go test ./internal/agent/nuclei ./internal/agent -count=1
|
||||
GOWORK=off go test -race ./internal/agent/nuclei -count=1
|
||||
```
|
||||
|
||||
Then run the deterministic benchmark and record the exact commands, revisions, machine limits, and raw summary in the benchmark document.
|
||||
|
||||
Do not claim the overall zero-deadline goal at this milestone. The milestone claim is limited to safe runtime reuse, bounded batch memory, cancellation correctness, and demonstrated local performance. The next implementation plan covers canonical attempt accounting, deadline-fit chunking, capacity-aware admission, and work-based fleet sizing.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module github.com/projectdiscovery/nuclei/v3
|
||||
|
||||
go 1.26
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/andygrunwald/go-jira v1.16.1
|
||||
@@ -17,13 +17,13 @@ require (
|
||||
github.com/olekukonko/tablewriter v1.0.8
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/projectdiscovery/clistats v0.1.6
|
||||
github.com/projectdiscovery/fastdialer v0.5.18
|
||||
github.com/projectdiscovery/fastdialer v0.5.19
|
||||
github.com/projectdiscovery/hmap v0.0.102
|
||||
github.com/projectdiscovery/interactsh v1.3.1
|
||||
github.com/projectdiscovery/rawhttp v0.1.91
|
||||
github.com/projectdiscovery/rawhttp v0.1.92
|
||||
github.com/projectdiscovery/retryabledns v1.0.116
|
||||
github.com/projectdiscovery/retryablehttp-go v1.3.24
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.6
|
||||
github.com/projectdiscovery/retryablehttp-go v1.3.25
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.7
|
||||
github.com/remeh/sizedwaitgroup v1.0.0
|
||||
github.com/rs/xid v1.6.0
|
||||
github.com/segmentio/ksuid v1.0.4
|
||||
@@ -32,14 +32,15 @@ require (
|
||||
github.com/syndtr/goleveldb v1.0.0
|
||||
github.com/weppos/publicsuffix-go v0.50.3 // indirect
|
||||
go.uber.org/multierr v1.11.0
|
||||
golang.org/x/net v0.57.0
|
||||
golang.org/x/net v0.58.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/text v0.40.0
|
||||
golang.org/x/text v0.41.0
|
||||
)
|
||||
|
||||
require (
|
||||
carvel.dev/ytt v0.52.0
|
||||
code.gitea.io/sdk/gitea v0.17.0
|
||||
filippo.io/age v1.2.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0
|
||||
github.com/Azure/go-ntlmssp v0.1.1
|
||||
@@ -55,9 +56,9 @@ require (
|
||||
github.com/antchfx/xmlquery v1.4.4
|
||||
github.com/antchfx/xpath v1.3.6
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.17
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.70
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.16
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0
|
||||
github.com/bytedance/sonic v1.15.0
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
@@ -68,6 +69,7 @@ require (
|
||||
github.com/fatih/structs v1.1.0
|
||||
github.com/fullstorydev/grpcurl v1.9.3
|
||||
github.com/getkin/kin-openapi v0.144.0
|
||||
github.com/getsops/sops/v3 v3.9.2
|
||||
github.com/go-git/go-git/v5 v5.19.2
|
||||
github.com/go-ldap/ldap/v3 v3.4.12
|
||||
github.com/go-pdf/fpdf v0.9.0
|
||||
@@ -102,20 +104,21 @@ require (
|
||||
github.com/projectdiscovery/gostruct v0.0.2
|
||||
github.com/projectdiscovery/govaluate v0.0.0-20260615100919-5ee2581bbf7e
|
||||
github.com/projectdiscovery/gozero v0.1.1-0.20260530071156-fa1dad563d76
|
||||
github.com/projectdiscovery/httpx v1.10.0
|
||||
github.com/projectdiscovery/httpx v1.11.0
|
||||
github.com/projectdiscovery/mapcidr v1.1.97
|
||||
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
|
||||
github.com/projectdiscovery/networkpolicy v0.1.48
|
||||
github.com/projectdiscovery/networkpolicy v0.1.49
|
||||
github.com/projectdiscovery/ratelimit v0.0.88
|
||||
github.com/projectdiscovery/rdap v0.9.0
|
||||
github.com/projectdiscovery/sarif v0.1.0
|
||||
github.com/projectdiscovery/tlsx v1.3.2
|
||||
github.com/projectdiscovery/uncover v1.2.1
|
||||
github.com/projectdiscovery/useragent v0.0.109
|
||||
github.com/projectdiscovery/utils v0.11.2
|
||||
github.com/projectdiscovery/wappalyzergo v0.2.95
|
||||
github.com/projectdiscovery/utils v0.11.3
|
||||
github.com/projectdiscovery/wappalyzergo v0.2.96
|
||||
github.com/redis/go-redis/v9 v9.11.0
|
||||
github.com/rs/zerolog v1.34.0
|
||||
github.com/sandrolain/httpcache v1.4.0
|
||||
github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466
|
||||
github.com/sijms/go-ora/v2 v2.9.0
|
||||
github.com/stretchr/testify v1.12.1
|
||||
@@ -126,21 +129,35 @@ require (
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/term v0.45.0
|
||||
google.golang.org/grpc v1.82.1
|
||||
google.golang.org/grpc v1.83.1
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
moul.io/http2curl v1.0.0
|
||||
)
|
||||
|
||||
require (
|
||||
aead.dev/minisign v0.3.0 // indirect
|
||||
cel.dev/expr v0.25.2 // indirect
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
cloud.google.com/go/auth v0.18.2 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.5.3 // indirect
|
||||
cloud.google.com/go/kms v1.25.0 // indirect
|
||||
cloud.google.com/go/longrunning v0.8.0 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.3 // indirect
|
||||
cloud.google.com/go/storage v1.56.0 // indirect
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.1 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
|
||||
github.com/BurntSushi/toml v1.4.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect
|
||||
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
|
||||
@@ -158,24 +175,26 @@ require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/kms v1.37.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect
|
||||
github.com/aws/smithy-go v1.24.2 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bits-and-blooms/bitset v1.22.0 // indirect
|
||||
github.com/bits-and-blooms/bloom/v3 v3.5.0 // indirect
|
||||
github.com/blang/semver v3.5.1+incompatible // indirect
|
||||
github.com/bodgit/plumbing v1.3.0 // indirect
|
||||
github.com/bodgit/sevenzip v1.6.1 // indirect
|
||||
github.com/bodgit/windows v1.0.1 // indirect
|
||||
@@ -204,6 +223,7 @@ require (
|
||||
github.com/containerd/continuity v0.4.5 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.6.1 // indirect
|
||||
github.com/davidmz/go-pageant v1.0.2 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
@@ -213,6 +233,7 @@ require (
|
||||
github.com/dlclark/regexp2/v2 v2.2.1 // indirect
|
||||
github.com/docker/cli v29.2.0+incompatible // indirect
|
||||
github.com/docker/go-connections v0.7.0 // indirect
|
||||
github.com/dop251/goja v0.0.0-20260311135729-065cd970411c // indirect
|
||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.10.0 // indirect
|
||||
@@ -227,6 +248,7 @@ require (
|
||||
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
|
||||
github.com/gaissmai/bart v0.29.0 // indirect
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/getsops/gopgagent v0.0.0-20240527072608-0c14999532fe // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/gin-gonic/gin v1.9.1 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
|
||||
@@ -251,14 +273,27 @@ require (
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/golang/snappy v1.0.0 // indirect
|
||||
github.com/google/certificate-transparency-go v1.3.2 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 // indirect
|
||||
github.com/gorilla/css v1.0.1 // indirect
|
||||
github.com/gosimple/slug v1.15.0 // indirect
|
||||
github.com/gosimple/unidecode v1.0.1 // indirect
|
||||
github.com/goware/prefixer v0.0.0-20160118172347-395022866408 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/hashicorp/go-version v1.8.0 // indirect
|
||||
github.com/hashicorp/go-version v1.9.0 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/hashicorp/vault/api v1.15.0 // indirect
|
||||
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect
|
||||
github.com/hdm/jarm-go v0.0.8 // indirect
|
||||
github.com/iangcarroll/cookiemonster v1.6.0 // indirect
|
||||
@@ -284,7 +319,7 @@ require (
|
||||
github.com/libdns/libdns v1.1.1 // indirect
|
||||
github.com/lor00x/goldap v0.0.0-20240304151906-8d785c64d1c8 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20250827001030-24949be3fa54 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
|
||||
github.com/mackerelio/go-osstat v0.2.6 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
@@ -296,6 +331,8 @@ require (
|
||||
github.com/minio/minlz v1.0.1 // indirect
|
||||
github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/moby/api v1.54.2 // indirect
|
||||
github.com/moby/moby/client v0.4.1 // indirect
|
||||
@@ -325,11 +362,13 @@ require (
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/projectdiscovery/asnmap v1.1.1 // indirect
|
||||
github.com/projectdiscovery/blackrock v0.0.2 // indirect
|
||||
github.com/projectdiscovery/cdncheck v1.2.50 // indirect
|
||||
github.com/projectdiscovery/cdncheck v1.2.51 // indirect
|
||||
github.com/projectdiscovery/freeport v0.0.7 // indirect
|
||||
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
|
||||
github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 // indirect
|
||||
github.com/refraction-networking/utls v1.8.2 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||
github.com/sashabaranov/go-openai v1.37.0 // indirect
|
||||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
|
||||
@@ -340,7 +379,7 @@ require (
|
||||
github.com/sorairolake/lzip-go v0.3.8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.7.0 // indirect
|
||||
github.com/tidwall/btree v1.8.1 // indirect
|
||||
github.com/tidwall/buntdb v1.3.2 // indirect
|
||||
github.com/tidwall/gjson v1.18.0 // indirect
|
||||
@@ -356,6 +395,7 @@ require (
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.11 // indirect
|
||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||
github.com/urfave/cli v1.22.16 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/vmihailenco/bufpool v0.1.11 // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect
|
||||
@@ -379,15 +419,23 @@ require (
|
||||
github.com/zcalusic/sysinfo v1.1.3 // indirect
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
|
||||
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
go.uber.org/zap/exp v0.3.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
||||
golang.org/x/arch v0.3.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect
|
||||
google.golang.org/api v0.264.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
mellium.im/sasl v0.3.2 // indirect
|
||||
software.sslmate.com/src/go-pkcs12 v0.7.2 // indirect
|
||||
)
|
||||
@@ -398,7 +446,7 @@ require (
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/go-querystring v1.2.0 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
|
||||
github.com/trivago/tgo v1.0.7
|
||||
@@ -411,12 +459,12 @@ require (
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
goftp.io/server/v2 v2.0.1 // indirect
|
||||
golang.org/x/crypto v0.54.0 // indirect
|
||||
golang.org/x/crypto v0.55.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3
|
||||
golang.org/x/mod v0.37.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
golang.org/x/tools v0.47.0
|
||||
golang.org/x/mod v0.39.0 // indirect
|
||||
golang.org/x/sys v0.47.0
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
golang.org/x/tools v0.49.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
|
||||
gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ=
|
||||
aead.dev/minisign v0.3.0 h1:8Xafzy5PEVZqYDNP60yJHARlW1eOQtsKNp/Ph2c0vRA=
|
||||
aead.dev/minisign v0.3.0/go.mod h1:NLvG3Uoq3skkRMDuc3YHpWUTMTrSExqm+Ij73W13F6Y=
|
||||
c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805 h1:u2qwJeEvnypw+OCPUHmoZE3IqwfuN5kgDfo5MLzpNM0=
|
||||
c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805/go.mod h1:FomMrUJ2Lxt5jCLmZkG3FHa72zUprnhd3v/Z18Snm4w=
|
||||
carvel.dev/ytt v0.52.0 h1:tkJPL8Gun5snVfypNXbmMKwnbwMyspcTi3Ypyso3nRY=
|
||||
carvel.dev/ytt v0.52.0/go.mod h1:QgmuU7E15EXW1r2wxTt7zExVz14IHwEG4WNMmaFBkJo=
|
||||
cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs=
|
||||
cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
@@ -18,6 +22,12 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
|
||||
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
|
||||
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
|
||||
cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
|
||||
cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
|
||||
cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
|
||||
@@ -25,8 +35,20 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM
|
||||
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
|
||||
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
|
||||
cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
|
||||
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
|
||||
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
|
||||
cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc=
|
||||
cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU=
|
||||
cloud.google.com/go/kms v1.25.0 h1:gVqvGGUmz0nYCmtoxWmdc1wli2L1apgP8U4fghPGSbQ=
|
||||
cloud.google.com/go/kms v1.25.0/go.mod h1:XIdHkzfj0bUO3E+LvwPg+oc7s58/Ns8Nd8Sdtljihbk=
|
||||
cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
|
||||
cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
|
||||
cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8=
|
||||
cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk=
|
||||
cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE=
|
||||
cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
|
||||
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
|
||||
@@ -36,11 +58,17 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
|
||||
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
|
||||
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
|
||||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
cloud.google.com/go/storage v1.56.0 h1:iixmq2Fse2tqxMbWhLWC9HfBj1qdxqAmiK8/eqtsLxI=
|
||||
cloud.google.com/go/storage v1.56.0/go.mod h1:Tpuj6t4NweCLzlNbw9Z9iwxEkrSem20AetIeH/shgVU=
|
||||
cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U=
|
||||
cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s=
|
||||
code.gitea.io/sdk/gitea v0.17.0 h1:8JPBss4+Jf7AE1YcfyiGrngTXE8dFSG3si/bypsTH34=
|
||||
code.gitea.io/sdk/gitea v0.17.0/go.mod h1:ndkDk99BnfiUCCYEUhpNzi0lpmApXlwRFqClBlOlEBg=
|
||||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
filippo.io/age v1.2.0 h1:vRDp7pUMaAJzXNIWJVAZnEf/Dyi4Vu4wI8S1LBzufhE=
|
||||
filippo.io/age v1.2.0/go.mod h1:JL9ew2lTN+Pyft4RiNGguFfOpewKwSHm5ayKD/A4004=
|
||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8=
|
||||
@@ -77,6 +105,14 @@ github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/
|
||||
github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM=
|
||||
github.com/FalconOpsLLC/goexec v0.3.0 h1:ryLMkrGT6asnkqdc5rFMNOSTYdMH/iCfyEuwu0D6ZhA=
|
||||
github.com/FalconOpsLLC/goexec v0.3.0/go.mod h1:kiyxVbmFCGbbwXRyZmOSKlOy7PiK+JH2gq07Ztag/k8=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 h1:l7+6kwRMJNwdCvYdDl7Eax+wzEYHSnNY7zrrfbhDdTA=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo=
|
||||
github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE=
|
||||
github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
|
||||
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
||||
@@ -159,20 +195,20 @@ github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.17 h1:jSuiQ5jEe4SAMH6lLRMY9OVC+TqJLP5655pBGjmnjr0=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.17/go.mod h1:9P4wwACpbeXs9Pm9w1QTh6BwWwJjwYvJ1iCt5QbCXh8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.70 h1:ONnH5CM16RTXRkS8Z1qg7/s2eDOhHhaXVd72mmyv4/0=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.70/go.mod h1:M+lWhhmomVGgtuPOhO85u4pEa3SmssPTdcYpP/5J/xc=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 h1:KAXP9JSHO1vKGCr5f4O6WmlVKLFFXgWYAGoJosorxzU=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32/go.mod h1:h4Sg6FQdexC1yYG9RDnOvLbW1a/P986++/Y/a+GyEM8=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82 h1:EO13QJTCD1Ig2IrQnoHTRrn981H9mB7afXsZ89WptI4=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82/go.mod h1:AGh1NCg0SH+uyJamiJA5tTQcql4MMRDXGRdMmCxCXzY=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.16 h1:E4Tz+tJiPc7kGnXwIfCyUj6xHJNpENlY11oKpRTgsjc=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.16/go.mod h1:2S9hBElpCyGMifv14WxQ7EfPumgoeCPZUpuPX8VtW34=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2 h1:9/HxDeIgA7DcKK6e6ZaP5PQiXugYbNERx3Z5u30mN+k=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2/go.mod h1:3N1RoxKNcVHmbOKVMMw8pvMs5TUhGYPQP/aq1zmAWqo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
|
||||
@@ -183,14 +219,16 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3x
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/kms v1.37.7 h1:dZmNIRtPUvtvUIIDVNpvtnJQ8N8Iqm7SQAxf18htZYw=
|
||||
github.com/aws/aws-sdk-go-v2/service/kms v1.37.7/go.mod h1:vj8PlfJH9mnGeIzd6uMLPi5VgiqzGG7AZoe1kf1uTXM=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 h1:AIRJ3lfb2w/1/8wOOSqYb9fUKGwQbtysJ2H1MofRUPg=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.25.5/go.mod h1:b7SiVprpU+iGazDUqvRSLf5XmCdn+JtT1on7uNL6Ipc=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 h1:BpOxT3yhLwSJ77qIY3DoHAQjZsc4HEGfMCE4NGy3uFg=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3/go.mod h1:vq/GQR1gOFLquZMSrxUK/cpvKCNVYibNyJ1m7JrU88E=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 h1:NFOJ/NXEGV4Rq//71Hs1jC/NvPs1ezajK+yQmkwnPV0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.34.0/go.mod h1:7ph2tGpfQvwzgistp2+zga9f+bCjlQJPkPUmMgDSD7w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
@@ -209,6 +247,8 @@ github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCk
|
||||
github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
|
||||
github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY=
|
||||
github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs=
|
||||
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
|
||||
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
|
||||
github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU=
|
||||
github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs=
|
||||
github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4=
|
||||
@@ -297,6 +337,8 @@ github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ
|
||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
|
||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
|
||||
github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE=
|
||||
@@ -328,6 +370,8 @@ github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf
|
||||
github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/dop251/goja v0.0.0-20260311135729-065cd970411c h1:OcLmPfx1T1RmZVHHFwWMPaZDdRf0DBMZOFMVWJa7Pdk=
|
||||
github.com/dop251/goja v0.0.0-20260311135729-065cd970411c/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4=
|
||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4=
|
||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
|
||||
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
|
||||
@@ -343,8 +387,12 @@ github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FM
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA=
|
||||
github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A=
|
||||
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
|
||||
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds=
|
||||
github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0=
|
||||
@@ -377,6 +425,10 @@ github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm6
|
||||
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/getkin/kin-openapi v0.144.0 h1:hIRcTH+KjLfkLpYU6bSSfdFpi0fZi1fp+hSPi4aQu9Y=
|
||||
github.com/getkin/kin-openapi v0.144.0/go.mod h1:3BH9M9XDe/y9M5DSvEocVYAYq1w0qrhJHjC/vZi0AaY=
|
||||
github.com/getsops/gopgagent v0.0.0-20240527072608-0c14999532fe h1:QKe/kmAYbndxwu91TcjHERsnMh5SgOB1x/qicvOdUJ8=
|
||||
github.com/getsops/gopgagent v0.0.0-20240527072608-0c14999532fe/go.mod h1:awFzISqLJoZLm+i9QQ4SgMNHDqljH6jWV0B36V5MrUM=
|
||||
github.com/getsops/sops/v3 v3.9.2 h1:Cuuahc/UGu7W3+NXd+mrl+2oXGxxj/jIPsw6Ah2CbxA=
|
||||
github.com/getsops/sops/v3 v3.9.2/go.mod h1:J8CMEdmPWT8gxFaNmPBSrc5NFd4uNndf8aFQwhD/ZR8=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
|
||||
@@ -448,6 +500,8 @@ github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5Nq
|
||||
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
|
||||
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw=
|
||||
github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw=
|
||||
@@ -535,13 +589,17 @@ github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG
|
||||
github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8=
|
||||
github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0=
|
||||
github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
|
||||
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
|
||||
github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
@@ -553,12 +611,18 @@ github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0Z
|
||||
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k=
|
||||
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
|
||||
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
|
||||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
|
||||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
|
||||
github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
|
||||
@@ -573,24 +637,43 @@ github.com/gosimple/slug v1.15.0 h1:wRZHsRrRcs6b0XnxMUBM6WK1U1Vg5B0R7VkIf1Xzobo=
|
||||
github.com/gosimple/slug v1.15.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ=
|
||||
github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o=
|
||||
github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc=
|
||||
github.com/goware/prefixer v0.0.0-20160118172347-395022866408 h1:Y9iQJfEqnN3/Nce9cOegemcy/9Ai5k3huT6E80F3zaw=
|
||||
github.com/goware/prefixer v0.0.0-20160118172347-395022866408/go.mod h1:PE1ycukgRPJ7bJ9a1fdfQ9j8i/cEcRAoLZzbxYpNB/s=
|
||||
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
|
||||
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
|
||||
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
|
||||
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw=
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
|
||||
github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
|
||||
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8 h1:iBt4Ew4XEGLfh6/bPk4rSYmuZJGizr6/x/AEizP0CQc=
|
||||
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8/go.mod h1:aiJI+PIApBRQG7FZTEBx5GiiX+HbOHilUdNxUZi4eV0=
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
|
||||
github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw=
|
||||
github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw=
|
||||
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
|
||||
github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4=
|
||||
github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA=
|
||||
github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/hashicorp/vault/api v1.15.0 h1:O24FYQCWwhwKnF7CuSqP30S51rTV7vz1iACXE/pj5DA=
|
||||
github.com/hashicorp/vault/api v1.15.0/go.mod h1:+5YTO09JGn0u+b6ySD/LLVf8WkJCPLAL2Vkmrn2+CM8=
|
||||
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk=
|
||||
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA=
|
||||
github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDjJSQ=
|
||||
@@ -703,8 +786,8 @@ github.com/lor00x/goldap v0.0.0-20240304151906-8d785c64d1c8 h1:z9RDOBcFcf3f2hSfK
|
||||
github.com/lor00x/goldap v0.0.0-20240304151906-8d785c64d1c8/go.mod h1:37YR9jabpiIxsb8X9VCIx8qFOjTDIIrIHHODa8C4gz0=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/lufia/plan9stats v0.0.0-20250827001030-24949be3fa54 h1:mFWunSatvkQQDhpdyuFAYwyAan3hzCuma+Pz8sqvOfg=
|
||||
github.com/lufia/plan9stats v0.0.0-20250827001030-24949be3fa54/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k=
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/mackerelio/go-osstat v0.2.6 h1:gs4U8BZeS1tjrL08tt5VUliVvSWP26Ai2Ob8Lr7f2i0=
|
||||
github.com/mackerelio/go-osstat v0.2.6/go.mod h1:lRy8V9ZuHpuRVZh+vyTkODeDPl3/d5MgXHtLSaqG8bA=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
@@ -746,6 +829,10 @@ github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02G
|
||||
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
|
||||
github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/moby/api v1.54.2 h1:wiat9QAhnDQjA7wk1kh/TqHz2I1uUA7M7t9SAl/JNXg=
|
||||
@@ -835,14 +922,14 @@ github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kI
|
||||
github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60=
|
||||
github.com/projectdiscovery/blackrock v0.0.2 h1:mxXdu0uM8P2L2Qi210COlU8QiICPFW/Rxk5QUhlPO2k=
|
||||
github.com/projectdiscovery/blackrock v0.0.2/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
|
||||
github.com/projectdiscovery/cdncheck v1.2.50 h1:KZBYT4jKNvlmlDKJqGMJ7r6XtwAim82oEZLaI+NAWWk=
|
||||
github.com/projectdiscovery/cdncheck v1.2.50/go.mod h1:9oE9KKxCSHNvUf0UaMeqqUwWpC38FkNaTll0ScIBT3w=
|
||||
github.com/projectdiscovery/cdncheck v1.2.51 h1:uB+/JdG39HaP3GQ3ncBsWmPh/8cbYmJ3p/UxstHZl0o=
|
||||
github.com/projectdiscovery/cdncheck v1.2.51/go.mod h1:GIAA0SqjDfxzDQjHR/X+bK05BGmUGfsVplP5Zdv/kHA=
|
||||
github.com/projectdiscovery/clistats v0.1.6 h1:e2Hc7cVWKJ4sCbyQ7th5zuBvgqsZcLwRq1qY0vmQXY4=
|
||||
github.com/projectdiscovery/clistats v0.1.6/go.mod h1:RxRGxhMzkT344VTTvZWIrojurUXdQQRN1yfwgg/88I8=
|
||||
github.com/projectdiscovery/dsl v0.8.22 h1:ZfPunLaaYUJTBffE1ShuAAV9hkSH2thvlszK48FlZS0=
|
||||
github.com/projectdiscovery/dsl v0.8.22/go.mod h1:fr8WMU381As89e4Z+SQ8tYQ1NudniWAS6NLPHrhe+Vs=
|
||||
github.com/projectdiscovery/fastdialer v0.5.18 h1:bBGJdni/xruoUISBug86KwRpuewLg1lEUfQLTg3p0G0=
|
||||
github.com/projectdiscovery/fastdialer v0.5.18/go.mod h1:fH4CDmC7Dk7Shm4v3Rsql7ql3OhlKeZfOdAlc/muP1I=
|
||||
github.com/projectdiscovery/fastdialer v0.5.19 h1:FQOoGGaCyreq53yz3QI+KztvOKn2M9OXJqNg7/4e03w=
|
||||
github.com/projectdiscovery/fastdialer v0.5.19/go.mod h1:6I4/kPu9s/LTUgmPqIc0ZOEh8TfZGY0Nlr5M84cFAj8=
|
||||
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
|
||||
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
|
||||
github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk=
|
||||
@@ -865,8 +952,8 @@ github.com/projectdiscovery/gozero v0.1.1-0.20260530071156-fa1dad563d76 h1:AN70b
|
||||
github.com/projectdiscovery/gozero v0.1.1-0.20260530071156-fa1dad563d76/go.mod h1:cWHYnRXoYWHtTpOYyAp5laGYX8GH8ITUhgQaP8G/8FA=
|
||||
github.com/projectdiscovery/hmap v0.0.102 h1:ybPuf7UQvuOLiHLOUbVMvd+PSlItQU5iQpP1Erl8kaU=
|
||||
github.com/projectdiscovery/hmap v0.0.102/go.mod h1:IZ92wXFmxYpdn0euZ7VFmypzOBNDwq8c3yRyIIol6XE=
|
||||
github.com/projectdiscovery/httpx v1.10.0 h1:sZvfeJDeX4eJfB5Oh29My4gOHmLfvpCt2wCqhargJrE=
|
||||
github.com/projectdiscovery/httpx v1.10.0/go.mod h1:gfdNnFznLLlAgAgSQlYjUeEVvYzKrNV0jO5Qyo5w+FI=
|
||||
github.com/projectdiscovery/httpx v1.11.0 h1:0jPnhqiCMlkcEGv5qFVkBYUX7BRbe3gStvDkBZdkbYs=
|
||||
github.com/projectdiscovery/httpx v1.11.0/go.mod h1:parFzVgFWVb7qDgyPgztD2I2OaVGJl8CCKGLPkWaObA=
|
||||
github.com/projectdiscovery/interactsh v1.3.1 h1:5HzeVGVCAX/cjTguJ+7ClOmML5r97Ty7op9s+/F7BiM=
|
||||
github.com/projectdiscovery/interactsh v1.3.1/go.mod h1:MXQ11EoBPROb4bEw+WP9e4DX4fMhrpS6EwfMfZomBsw=
|
||||
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8=
|
||||
@@ -877,18 +964,18 @@ github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8Qu
|
||||
github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA=
|
||||
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
|
||||
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.48 h1:s/d0nEyfkHYVBeZI/bgRiOCS66KDxv3+s7Hb+AvepHY=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.48/go.mod h1:bb2ZkDnOeVuR9Ok8BdCdoBEz8S9Y9EFvFFcn5FTUV2U=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.49 h1:4F2lIy/iZUfLltpAwudXF/93CCICipr+pBshz5Ozts4=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.49/go.mod h1:wm/KWiKQ9I0LLowandFnLiSype7HEvilXVjEwNLk2PE=
|
||||
github.com/projectdiscovery/ratelimit v0.0.88 h1:AcurW9aLRzlEyPe9kSjnOpr3XzLMWTpiWAlW/w73ALU=
|
||||
github.com/projectdiscovery/ratelimit v0.0.88/go.mod h1:CU1s+68UUG2mctSl2wi32/DHLJA6TMg+4rxgP59LfVk=
|
||||
github.com/projectdiscovery/rawhttp v0.1.91 h1:6EIhZxCkBn27kJAl47FieCFCe4H2ZdbhoBVj0J6ZCVY=
|
||||
github.com/projectdiscovery/rawhttp v0.1.91/go.mod h1:VZYAM25UI/wVB3URZ95ZaftgOnsbphxyAw/XnQRRz4Y=
|
||||
github.com/projectdiscovery/rawhttp v0.1.92 h1:38GRsVLNllES/dZcEcA6s6P+rYBp34H9mDphMJMDvvg=
|
||||
github.com/projectdiscovery/rawhttp v0.1.92/go.mod h1:spfXTPjxCj+bfWLynMVB7uOb+LS3PDNAI9HexMEV1Zg=
|
||||
github.com/projectdiscovery/rdap v0.9.0 h1:wPhHx5pQ2QI+WGhyNb2PjhTl0NtB39Nk7YFZ9cp8ZGA=
|
||||
github.com/projectdiscovery/rdap v0.9.0/go.mod h1:zk4yrJFQ2Hy36Aqk+DvotYQxYAeALaCJ5ORySkff36Q=
|
||||
github.com/projectdiscovery/retryabledns v1.0.116 h1:tK28PJ4tLO7bgM89RpIFjkfTPMSbRegwvg9mN/q6z78=
|
||||
github.com/projectdiscovery/retryabledns v1.0.116/go.mod h1:r3vH0AKYtJ9fculcqRFrdVYe2ABcHwZeR4C6j3rqCbY=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.3.24 h1:bODdeUEka/U5SS9wfnhoAjBRg1wq/AJdwb9CUNIOR88=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.3.24/go.mod h1:OL5zZEBd59GXEMaDjEHW/bfGlFsV441ADz64hWajgxc=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.3.25 h1:caWjhagt49zkSbaOZnsD+qi8UwfgbofezHOtYPHvhb0=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.3.25/go.mod h1:+lyetyXnmFCGzjfebzVCsszts9JHkKncal+nNwYeqCs=
|
||||
github.com/projectdiscovery/sarif v0.1.0 h1:O541T+a448nSJsmIMnXXSOeDQEzpnCAYvRfe0eG5h74=
|
||||
github.com/projectdiscovery/sarif v0.1.0/go.mod h1:LBC+reM3bkI3qIIhE0rZaINaYX6VG+En6u2hHa5mA7E=
|
||||
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
|
||||
@@ -899,12 +986,12 @@ github.com/projectdiscovery/uncover v1.2.1 h1:8U46T/96CLT7BPoXBgkTvWqB06lOyeTSLv
|
||||
github.com/projectdiscovery/uncover v1.2.1/go.mod h1:0p8onrWxfpXQEYs90ZDzTSpu1107gWmodX1NWqu/+z4=
|
||||
github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE=
|
||||
github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk=
|
||||
github.com/projectdiscovery/utils v0.11.2 h1:gfvXFBEHN4DgX8HBRzsoO1s2wH/2R+cH17cLpC865x4=
|
||||
github.com/projectdiscovery/utils v0.11.2/go.mod h1:HMxhxLigsAr+M9Oa8n9Z0ROZcs8wAJHGsgR7UGb+oUE=
|
||||
github.com/projectdiscovery/wappalyzergo v0.2.95 h1:XHjbQVrrxusezFq7ciWmtznPKWhpr/srWp7Yk2noxqg=
|
||||
github.com/projectdiscovery/wappalyzergo v0.2.95/go.mod h1:E2p8L90ysTTUkU5FUOgGoCQ7ucEUgqJ/tvYgiGGAlEM=
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.6 h1:GCEdIRlQjDux28xTXKszM7n3jlMf152d5nqVpVoetas=
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.6/go.mod h1:R5lWrNzP+7Oyn77NDVPnBsxx2/FyQZBBkIAaSaCQFxw=
|
||||
github.com/projectdiscovery/utils v0.11.3 h1:TNBhSNJ8uFw7DWASUHBf1vfCx7mwMi+jtGizrizMM5s=
|
||||
github.com/projectdiscovery/utils v0.11.3/go.mod h1:HMxhxLigsAr+M9Oa8n9Z0ROZcs8wAJHGsgR7UGb+oUE=
|
||||
github.com/projectdiscovery/wappalyzergo v0.2.96 h1:/YHMiUA2kL2bap2vHDuZs1cJK3RgHZjxdVam5GxJ12k=
|
||||
github.com/projectdiscovery/wappalyzergo v0.2.96/go.mod h1:5ZuunxbKPGvnJeEOMIqlfj5xg/XSEAoakZr/s9l2pNI=
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.7 h1:Odmjd1o4EkMff2F6QHi1dIYPIQPZUFy24YqmQCprmTk=
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.7/go.mod h1:nqD1iPMcofpRA1DbXavPByJhWURrshT+l8WN9XjyQag=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
|
||||
@@ -946,9 +1033,15 @@ github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
|
||||
github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
|
||||
github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
|
||||
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
|
||||
github.com/sandrolain/httpcache v1.4.0 h1:Jf4Vx62X2ybvNPSpPvI1kT3xvMdDG1AsApQjOQKO9E0=
|
||||
github.com/sandrolain/httpcache v1.4.0/go.mod h1:kHBuXveitSn39SNPBhdf/ybG272X706HJ2RJqOQ+Em0=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/sashabaranov/go-openai v1.37.0 h1:hQQowgYm4OXJ1Z/wTrE+XZaO20BYsL0R3uRPSpfNZkY=
|
||||
@@ -992,8 +1085,8 @@ github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
|
||||
github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4=
|
||||
github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
@@ -1012,6 +1105,7 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
@@ -1061,6 +1155,8 @@ github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ
|
||||
github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
|
||||
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/urfave/cli v1.22.16 h1:MH0k6uJxdwdeWQTwhSO42Pwr4YLrNLwBtg1MRgTqPdQ=
|
||||
github.com/urfave/cli v1.22.16/go.mod h1:EeJR6BKodywf4zciqrdw6hpCPk68JO9z5LazXZMn5Po=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94=
|
||||
@@ -1170,16 +1266,24 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo=
|
||||
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
|
||||
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw=
|
||||
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
|
||||
go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA=
|
||||
go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
|
||||
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
|
||||
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
@@ -1222,8 +1326,8 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -1261,8 +1365,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
|
||||
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
|
||||
golang.org/x/mod v0.39.0 h1:UF5zwQdCRRUpHfyPwr7d4UrGiVeldIsogtzWVnczL74=
|
||||
golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -1312,8 +1416,8 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -1443,13 +1547,13 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
@@ -1496,8 +1600,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
|
||||
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
|
||||
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
||||
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -1520,6 +1624,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
|
||||
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
|
||||
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||
google.golang.org/api v0.264.0 h1:+Fo3DQXBK8gLdf8rFZ3uLu39JpOnhvzJrLMQSoSYZJM=
|
||||
google.golang.org/api v0.264.0/go.mod h1:fAU1xtNNisHgOF5JooAs8rRaTkl2rT3uaoNGo9NS3R8=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -1556,8 +1662,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
|
||||
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
@@ -1570,8 +1680,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
|
||||
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
|
||||
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE=
|
||||
google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
|
||||
google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y=
|
||||
google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
@@ -1602,6 +1712,8 @@ gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DS
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/gologger/formatter"
|
||||
"github.com/projectdiscovery/gologger/levels"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type ignoreFileLogRecord struct {
|
||||
message string
|
||||
level levels.Level
|
||||
}
|
||||
|
||||
type ignoreFileLogWriter struct {
|
||||
records []ignoreFileLogRecord
|
||||
}
|
||||
|
||||
func (w *ignoreFileLogWriter) Write(data []byte, level levels.Level) {
|
||||
w.records = append(w.records, ignoreFileLogRecord{message: string(data), level: level})
|
||||
}
|
||||
|
||||
func newIgnoreFileTestRunner(t *testing.T, root string, writer *ignoreFileLogWriter) *Runner {
|
||||
t.Helper()
|
||||
|
||||
cfg := config.DefaultConfig
|
||||
oldRoot := cfg.TemplatesDirectory
|
||||
t.Cleanup(func() { cfg.SetTemplatesDir(oldRoot) })
|
||||
cfg.SetTemplatesDir(root)
|
||||
|
||||
logger := &gologger.Logger{}
|
||||
logger.SetFormatter(formatter.NewCLI(false))
|
||||
logger.SetWriter(writer)
|
||||
logger.SetMaxLevel(levels.LevelWarning)
|
||||
|
||||
options := types.DefaultOptions()
|
||||
options.Logger = logger
|
||||
return &Runner{options: options, Logger: logger}
|
||||
}
|
||||
|
||||
func TestLoadIgnoreFileWarnsAndContinuesWhenActiveFileIsMissing(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writer := &ignoreFileLogWriter{}
|
||||
runner := newIgnoreFileTestRunner(t, root, writer)
|
||||
|
||||
require.NoError(t, runner.loadIgnoreFile())
|
||||
require.Empty(t, runner.options.ExcludeTags)
|
||||
require.Empty(t, runner.options.ExcludedTemplates)
|
||||
require.Len(t, writer.records, 1)
|
||||
require.Equal(t, levels.LevelWarning, writer.records[0].level)
|
||||
require.Contains(t, writer.records[0].message, strconv.Quote(filepath.Join(root, config.NucleiIgnoreFileName)))
|
||||
require.Contains(t, writer.records[0].message, "continuing without ignore exclusions")
|
||||
}
|
||||
|
||||
func TestLoadIgnoreFileRejectsCorruptActiveFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, config.NucleiIgnoreFileName)
|
||||
require.NoError(t, os.WriteFile(path, []byte("tags: ["), 0o600))
|
||||
writer := &ignoreFileLogWriter{}
|
||||
runner := newIgnoreFileTestRunner(t, root, writer)
|
||||
|
||||
err := runner.loadIgnoreFile()
|
||||
require.ErrorContains(t, err, "error parsing")
|
||||
require.ErrorContains(t, err, strconv.Quote(path))
|
||||
require.Empty(t, writer.records)
|
||||
}
|
||||
@@ -15,7 +15,9 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
|
||||
httpprotocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/utils/env"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
@@ -79,6 +81,8 @@ func GetLazyAuthFetchCallback(opts *AuthLazyFetchOptions) authx.LazyFetchSecret
|
||||
}
|
||||
data := map[string]interface{}{}
|
||||
tmpl := tmpls[0]
|
||||
restore := skipSecretFileOnAuthTemplate(tmpl)
|
||||
defer restore()
|
||||
// add args to tmpl here
|
||||
vars := map[string]interface{}{}
|
||||
mainCtx := context.Background()
|
||||
@@ -149,3 +153,46 @@ func GetLazyAuthFetchCallback(opts *AuthLazyFetchOptions) authx.LazyFetchSecret
|
||||
return finalErr
|
||||
}
|
||||
}
|
||||
|
||||
// skipSecretFileOnAuthTemplate disables secret-file auth on the login template for
|
||||
// the duration of the fetch. Login requests that hit the same domain as the
|
||||
// secret would otherwise ApplyAuth → Fetch and deadlock on sync.Once (#7639).
|
||||
// Previous SkipSecretFile values are restored so a cached template is unchanged.
|
||||
func skipSecretFileOnAuthTemplate(tmpl *templates.Template) func() {
|
||||
if tmpl == nil {
|
||||
return func() {}
|
||||
}
|
||||
var reqs []*httpprotocol.Request
|
||||
seen := make(map[*httpprotocol.Request]struct{})
|
||||
add := func(r *httpprotocol.Request) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
if _, ok := seen[r]; ok {
|
||||
return
|
||||
}
|
||||
seen[r] = struct{}{}
|
||||
reqs = append(reqs, r)
|
||||
}
|
||||
for _, r := range tmpl.RequestsHTTP {
|
||||
add(r)
|
||||
}
|
||||
for _, r := range tmpl.RequestsWithHTTP {
|
||||
add(r)
|
||||
}
|
||||
for _, req := range tmpl.RequestsQueue {
|
||||
if r, ok := req.(*httpprotocol.Request); ok {
|
||||
add(r)
|
||||
}
|
||||
}
|
||||
prev := make([]bool, len(reqs))
|
||||
for i, r := range reqs {
|
||||
prev[i] = r.SkipSecretFile
|
||||
r.SkipSecretFile = true
|
||||
}
|
||||
return func() {
|
||||
for i, r := range reqs {
|
||||
r.SkipSecretFile = prev[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,9 @@ import (
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
httpprotocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -137,3 +140,27 @@ func TestCallbackDynamicValues(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "sess-xyz-123", data["session"], "should pick the longest value")
|
||||
}
|
||||
|
||||
func TestSkipSecretFileOnAuthTemplate(t *testing.T) {
|
||||
t.Run("nil template is a no-op", func(t *testing.T) {
|
||||
restore := skipSecretFileOnAuthTemplate(nil)
|
||||
require.NotPanics(t, restore)
|
||||
})
|
||||
|
||||
t.Run("sets skip during fetch and restores previous values", func(t *testing.T) {
|
||||
alreadySkipped := &httpprotocol.Request{SkipSecretFile: true}
|
||||
needsSkip := &httpprotocol.Request{}
|
||||
tmpl := &templates.Template{
|
||||
RequestsHTTP: []*httpprotocol.Request{alreadySkipped, needsSkip},
|
||||
}
|
||||
tmpl.RequestsQueue = []protocols.Request{alreadySkipped, needsSkip}
|
||||
|
||||
restore := skipSecretFileOnAuthTemplate(tmpl)
|
||||
require.True(t, alreadySkipped.SkipSecretFile)
|
||||
require.True(t, needsSkip.SkipSecretFile)
|
||||
|
||||
restore()
|
||||
require.True(t, alreadySkipped.SkipSecretFile, "templates that already skipped secret-file must stay skipped")
|
||||
require.False(t, needsSkip.SkipSecretFile, "scan-time ApplyAuth must remain enabled after login fetch")
|
||||
})
|
||||
}
|
||||
@@ -604,10 +604,13 @@ func (r *Runner) RunEnumeration() error {
|
||||
OutScope: r.options.OutOfScope,
|
||||
NucleiExecutorOptions: execurOpts,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r.dastServer = dastServer
|
||||
|
||||
return dastServer.Start()
|
||||
}
|
||||
|
||||
@@ -617,16 +620,18 @@ func (r *Runner) RunEnumeration() error {
|
||||
r.options.Templates = append(r.options.Templates, arr...)
|
||||
}
|
||||
}
|
||||
|
||||
if len(r.options.NewTemplatesWithVersion) > 0 {
|
||||
if arr := installer.GetNewTemplatesInVersions(r.options.NewTemplatesWithVersion...); len(arr) > 0 {
|
||||
r.options.Templates = append(r.options.Templates, arr...)
|
||||
}
|
||||
}
|
||||
// Exclude ignored file for validation
|
||||
|
||||
// Apply the active ignore policy only when templates can execute.
|
||||
if !r.options.Validate {
|
||||
ignoreFile := config.ReadIgnoreFile()
|
||||
r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
|
||||
r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
|
||||
if err := r.loadIgnoreFile(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
fuzzFreqCache := frequency.New(frequency.DefaultMaxTrackCount, r.options.FuzzParamFrequency)
|
||||
@@ -886,6 +891,24 @@ func (r *Runner) RunEnumeration() error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *Runner) loadIgnoreFile() error {
|
||||
ignoreFile, err := config.ReadIgnoreFile()
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
r.Logger.Warning().Msgf("Could not read active .nuclei-ignore file: %s; continuing without ignore exclusions", err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
|
||||
r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func shortDur(d time.Duration) string {
|
||||
if d < time.Minute {
|
||||
return d.String()
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
@@ -96,6 +97,8 @@ var httpTestcases = []integrationCase{
|
||||
{Path: "protocols/http/response-data-literal-reuse.yaml", TestCase: &httpResponseDataLiteralReuse{}},
|
||||
{Path: "protocols/http/raw-path-single-slash.yaml", TestCase: &httpRawPathSingleSlash{}},
|
||||
{Path: "protocols/http/raw-unsafe-path-single-slash.yaml", TestCase: &httpRawUnsafePathSingleSlash{}},
|
||||
{Path: "protocols/http/disable-http-cache.yaml", TestCase: &httpDisableCache{}},
|
||||
{Path: "protocols/http/http-cache.yaml", TestCase: &httpCache{}},
|
||||
}
|
||||
|
||||
type httpMultiVarSharing struct{}
|
||||
@@ -1849,3 +1852,67 @@ func (h *httpRawUnsafePathSingleSlash) Execute(filepath string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type httpCache struct{}
|
||||
|
||||
func (h *httpCache) Execute(filePath string) error {
|
||||
router := httprouter.New()
|
||||
var requestCount int32
|
||||
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
atomic.AddInt32(&requestCount, 1)
|
||||
w.Header().Set("Cache-Control", "max-age=2")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = fmt.Fprint(w, requestCount)
|
||||
})
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-http-cache")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// We expect 2 results because we made 2 requests and both should match
|
||||
if err := expectResultsCount(results, 2); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// We expect only 1 actual request to the server because of caching
|
||||
if count := atomic.LoadInt32(&requestCount); count != 1 {
|
||||
return fmt.Errorf("expected 1 request to server, got %d", count)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type httpDisableCache struct{}
|
||||
|
||||
func (h *httpDisableCache) Execute(filePath string) error {
|
||||
var requestCount int32
|
||||
router := httprouter.New()
|
||||
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
atomic.AddInt32(&requestCount, 1)
|
||||
w.Header().Set("Cache-Control", "max-age=2")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = fmt.Fprint(w, requestCount)
|
||||
})
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-http-cache")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// We expect 2 results because we made 2 requests and both should match
|
||||
if err := expectResultsCount(results, 2); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// We expect 2 actual requests to the server because cache is disabled
|
||||
if count := atomic.LoadInt32(&requestCount); count != 2 {
|
||||
return fmt.Errorf("expected 2 requests to server, got %d", count)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -92,7 +93,14 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error)
|
||||
actualTemplatePath = fixturePath(actualTemplatePath)
|
||||
}
|
||||
defaultOpts.Templates = goflags.StringSlice{actualTemplatePath}
|
||||
defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags
|
||||
ignoreFile, err := config.ReadIgnoreFile()
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
defaultOpts.Logger.Warning().Msgf("%s; continuing without ignore exclusions\n", err)
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
defaultOpts.ExcludeTags = ignoreFile.Tags
|
||||
}
|
||||
|
||||
outputWriter := testutils.NewMockOutputWriter(defaultOpts.OmitTemplate)
|
||||
var results []string
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
id: disable-http-cache
|
||||
|
||||
info:
|
||||
name: Disable HTTP Cache Test
|
||||
author: dwisiswant0
|
||||
severity: info
|
||||
description: Tests if disable-http-cache works as expected
|
||||
tags: test
|
||||
|
||||
requests:
|
||||
- raw:
|
||||
- |
|
||||
GET / HTTP/1.1
|
||||
Host: {{Hostname}}
|
||||
- |
|
||||
GET / HTTP/1.1
|
||||
Host: {{Hostname}}
|
||||
disable-http-cache: true
|
||||
matchers:
|
||||
- type: dsl
|
||||
dsl:
|
||||
- body_1 == "1"
|
||||
- body_2 == "2"
|
||||
@@ -0,0 +1,23 @@
|
||||
id: http-cache
|
||||
|
||||
info:
|
||||
name: HTTP Cache Test
|
||||
author: dwisiswant0
|
||||
severity: info
|
||||
description: Tests if HTTP cache (RFC 9111) works as expected
|
||||
tags: test
|
||||
|
||||
requests:
|
||||
- raw:
|
||||
- |
|
||||
GET / HTTP/1.1
|
||||
Host: {{Hostname}}
|
||||
- |
|
||||
GET / HTTP/1.1
|
||||
Host: {{Hostname}}
|
||||
disable-http-cache: false
|
||||
matchers:
|
||||
- type: dsl
|
||||
dsl:
|
||||
- body_1 == "1"
|
||||
- body_2 == "1"
|
||||
@@ -0,0 +1,28 @@
|
||||
package nuclei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNewNucleiEngineRejectsCorruptActiveIgnoreFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, config.NucleiIgnoreFileName)
|
||||
require.NoError(t, os.WriteFile(path, []byte("tags: ["), 0o600))
|
||||
|
||||
cfg := config.DefaultConfig
|
||||
oldRoot := cfg.TemplatesDirectory
|
||||
t.Cleanup(func() { cfg.SetTemplatesDir(oldRoot) })
|
||||
cfg.SetTemplatesDir(root)
|
||||
|
||||
engine, err := NewNucleiEngineCtx(context.Background())
|
||||
require.Nil(t, engine)
|
||||
require.ErrorContains(t, err, "error parsing")
|
||||
require.ErrorContains(t, err, strconv.Quote(path))
|
||||
}
|
||||
+53
-8
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
@@ -22,7 +23,8 @@ import (
|
||||
// hence they are ephemeral and are created on every ExecuteNucleiWithOpts invocation
|
||||
// in ThreadSafeNucleiEngine
|
||||
type unsafeOptions struct {
|
||||
executerOpts *protocols.ExecutorOptions
|
||||
executerOpts *protocols.ExecutorOptions
|
||||
executionParser *templates.Parser
|
||||
}
|
||||
|
||||
// createEphemeralObjects creates ephemeral nuclei objects/instances/types.
|
||||
@@ -31,7 +33,9 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
|
||||
if outputWriter == nil {
|
||||
outputWriter = base.customWriter
|
||||
}
|
||||
u := &unsafeOptions{}
|
||||
u := &unsafeOptions{
|
||||
executionParser: templates.NewExecutionParser(base.parser),
|
||||
}
|
||||
u.executerOpts = &protocols.ExecutorOptions{
|
||||
Output: outputWriter,
|
||||
Options: opts,
|
||||
@@ -42,13 +46,12 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
|
||||
Interactsh: base.interactshClient,
|
||||
Colorizer: aurora.New(aurora.WithColors(true)),
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
Parser: base.parser,
|
||||
Parser: u.executionParser,
|
||||
Browser: base.browserInstance,
|
||||
// Thread-safe executes can run concurrently with different Output writers.
|
||||
// The compiled-template cache shallow-copies requests and mutates shared
|
||||
// ExecutorOptions.Output via UpdateOptions/ApplyNewEngineOptions, which
|
||||
// would otherwise route all findings to whichever call last won the race.
|
||||
DoNotCache: true,
|
||||
// Each execution has a private compiled-template cache because compiled
|
||||
// requests retain mutable ExecutorOptions. Only clean parsed templates are
|
||||
// shared with the long-lived engine parser.
|
||||
DoNotCache: false,
|
||||
}
|
||||
if opts.ShouldUseHostError() && base.hostErrCache != nil {
|
||||
u.executerOpts.HostErrorsCache = base.hostErrCache
|
||||
@@ -64,6 +67,32 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// restoreBaseExcludeTags re-adds any of the engine's baseline excluded tags that
|
||||
// a per-execution option dropped, appending only what is missing so the result
|
||||
// is order-stable and repeated application is a no-op.
|
||||
//
|
||||
// The current ExcludeTags slice is cloned before merging so a caller-owned
|
||||
// backing array (e.g. a reused WithTemplateFilters ExcludeTags value with spare
|
||||
// capacity) is never mutated or shared across concurrent executions.
|
||||
func restoreBaseExcludeTags(base []string, opts *types.Options) {
|
||||
if len(base) == 0 {
|
||||
return
|
||||
}
|
||||
merged := append([]string(nil), opts.ExcludeTags...)
|
||||
present := make(map[string]struct{}, len(merged)+len(base))
|
||||
for _, tag := range merged {
|
||||
present[tag] = struct{}{}
|
||||
}
|
||||
for _, tag := range base {
|
||||
if _, ok := present[tag]; ok {
|
||||
continue
|
||||
}
|
||||
merged = append(merged, tag)
|
||||
present[tag] = struct{}{}
|
||||
}
|
||||
opts.ExcludeTags = merged
|
||||
}
|
||||
|
||||
// resolveEphemeralOutput combines the base/global writer with any per-call result callbacks.
|
||||
// Global callbacks keep working; per-call callbacks are isolated to this execution.
|
||||
func resolveEphemeralOutput(base, call *NucleiEngine) output.Writer {
|
||||
@@ -90,6 +119,10 @@ func resolveEphemeralOutput(base, call *NucleiEngine) output.Writer {
|
||||
|
||||
// closeEphemeralObjects closes all resources used by ephemeral nuclei objects/instances/types
|
||||
func closeEphemeralObjects(u *unsafeOptions) {
|
||||
if u.executionParser != nil {
|
||||
u.executionParser.PurgeCompiled()
|
||||
u.executionParser = nil
|
||||
}
|
||||
if u.executerOpts.RateLimiter != nil {
|
||||
u.executerOpts.RateLimiter.Stop()
|
||||
}
|
||||
@@ -161,6 +194,18 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, t
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Per-execution options land on a COPY of the engine's options, so an option
|
||||
// that assigns a whole filter set clears every field it does not name —
|
||||
// WithTemplateFilters does exactly that. ExcludeTags is one of those fields,
|
||||
// and it carries the .nuclei-ignore deny-list that applyRequiredDefaults
|
||||
// installed at construction; nothing re-reads the ignore file afterwards, so
|
||||
// the exclusions would be lost for this execution and templates tagged dos,
|
||||
// bruteforce, fuzz, local or txt-service would run.
|
||||
//
|
||||
// Restore the engine's baseline exclusions so a per-execution filter can add
|
||||
// to them but never silently discard them. IncludeTags remains the explicit
|
||||
// per-tag override for callers who do want an ignored template to run.
|
||||
restoreBaseExcludeTags(e.eng.opts.ExcludeTags, tmpEngine.opts)
|
||||
|
||||
// create ephemeral nuclei objects/instances/types using base nuclei engine
|
||||
unsafeOpts, err := createEphemeralObjects(ctx, e.eng, tmpEngine.opts, resolveEphemeralOutput(e.eng, tmpEngine))
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
package nuclei_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
nuclei "github.com/projectdiscovery/nuclei/v3/lib"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestExecuteNucleiWithOptsCtxKeepsBaseExcludeTags covers the case where a
|
||||
// per-execution filter that names only Tags used to clear the engine's
|
||||
// ExcludeTags — which carry the .nuclei-ignore deny-list installed at
|
||||
// construction — letting templates tagged dos (and local, fuzz, bruteforce,
|
||||
// txt-service) execute against the target.
|
||||
//
|
||||
// The engine's construction-time ExcludeTags stand in for that deny-list, and
|
||||
// the per-execution filter deliberately asks for the excluded tag: exclusion
|
||||
// must still win, because IncludeTags is the documented way to override it.
|
||||
func TestExecuteNucleiWithOptsCtxKeepsBaseExcludeTags(t *testing.T) {
|
||||
templatePath := writeSDKDosTaggedTemplate(t)
|
||||
|
||||
var reached atomic.Bool
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
reached.Store(true)
|
||||
_, _ = w.Write([]byte("token-dos"))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
ne, err := nuclei.NewThreadSafeNucleiEngineCtx(context.Background(),
|
||||
nuclei.WithTemplateFilters(nuclei.TemplateFilters{ExcludeTags: []string{"dos"}}),
|
||||
)
|
||||
require.NoError(t, err, "could not create thread-safe nuclei engine")
|
||||
t.Cleanup(ne.Close)
|
||||
|
||||
var matches atomic.Int64
|
||||
ne.GlobalResultCallback(func(*output.ResultEvent) { matches.Add(1) })
|
||||
|
||||
err = ne.ExecuteNucleiWithOptsCtx(context.Background(), []string{srv.URL},
|
||||
nuclei.WithTemplatesOrWorkflows(nuclei.TemplateSources{Templates: []string{templatePath}}),
|
||||
nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"dos"}}),
|
||||
)
|
||||
|
||||
require.ErrorIs(t, err, nuclei.ErrNoTemplatesAvailable,
|
||||
"the dos-tagged template must stay excluded, leaving nothing to run")
|
||||
assert.Zero(t, matches.Load(), "an excluded template must not report results")
|
||||
assert.False(t, reached.Load(), "an excluded template must not reach the target")
|
||||
}
|
||||
|
||||
// TestExecuteNucleiWithOptsCtxHonoursIncludeTags is the other half of the
|
||||
// contract: restoring the baseline exclusions must not break the documented
|
||||
// per-tag override, or callers lose the only way to run an ignored template.
|
||||
func TestExecuteNucleiWithOptsCtxHonoursIncludeTags(t *testing.T) {
|
||||
templatePath := writeSDKDosTaggedTemplate(t)
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte("token-dos"))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
ne, err := nuclei.NewThreadSafeNucleiEngineCtx(context.Background(),
|
||||
nuclei.WithTemplateFilters(nuclei.TemplateFilters{ExcludeTags: []string{"dos"}}),
|
||||
)
|
||||
require.NoError(t, err, "could not create thread-safe nuclei engine")
|
||||
t.Cleanup(ne.Close)
|
||||
|
||||
var matches atomic.Int64
|
||||
ne.GlobalResultCallback(func(*output.ResultEvent) { matches.Add(1) })
|
||||
|
||||
err = ne.ExecuteNucleiWithOptsCtx(context.Background(), []string{srv.URL},
|
||||
nuclei.WithTemplatesOrWorkflows(nuclei.TemplateSources{Templates: []string{templatePath}}),
|
||||
nuclei.WithTemplateFilters(nuclei.TemplateFilters{IncludeTags: []string{"dos"}}),
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.NotZero(t, matches.Load(), "IncludeTags should still override the exclusion")
|
||||
}
|
||||
|
||||
func writeSDKDosTaggedTemplate(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "sdk-dos-tagged.yaml")
|
||||
content := `id: sdk-dos-tagged
|
||||
info:
|
||||
name: SDK dos tagged
|
||||
author: nuclei-sdk-test
|
||||
severity: info
|
||||
tags: dos
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}/"
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "token-"
|
||||
`
|
||||
require.NoError(t, os.WriteFile(path, []byte(content), 0o600))
|
||||
return path
|
||||
}
|
||||
|
||||
// TestExecuteNucleiWithOptsCtxKeepsIgnoreFileExcludeTags is the end-to-end path
|
||||
// for #7695 once #7697 is on the tree: tags from a real .nuclei-ignore must
|
||||
// survive a per-execution WithTemplateFilters that only sets Tags.
|
||||
func TestExecuteNucleiWithOptsCtxKeepsIgnoreFileExcludeTags(t *testing.T) {
|
||||
previousConfigDir := config.DefaultConfig.GetConfigDir()
|
||||
previousTemplatesDir := config.DefaultConfig.TemplatesDirectory
|
||||
previousStateDir := config.DefaultConfig.GetStateDir()
|
||||
t.Cleanup(func() {
|
||||
config.DefaultConfig.SetConfigDir(previousConfigDir)
|
||||
config.DefaultConfig.SetStateDir(previousStateDir)
|
||||
config.DefaultConfig.SetTemplatesDir(previousTemplatesDir)
|
||||
})
|
||||
|
||||
dir := t.TempDir()
|
||||
config.DefaultConfig.SetConfigDir(dir)
|
||||
config.DefaultConfig.SetTemplatesDir(dir)
|
||||
|
||||
require.NoError(t, os.WriteFile(
|
||||
config.DefaultConfig.GetActiveIgnoreFilePath(),
|
||||
[]byte("tags:\n - dos\n"),
|
||||
0o600,
|
||||
))
|
||||
|
||||
templatePath := writeSDKDosTaggedTemplate(t)
|
||||
|
||||
var reached atomic.Bool
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
reached.Store(true)
|
||||
_, _ = w.Write([]byte("token-dos"))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
ne, err := nuclei.NewThreadSafeNucleiEngineCtx(context.Background(),
|
||||
nuclei.WithOptions(types.DefaultOptions()),
|
||||
nuclei.DisableUpdateCheck(),
|
||||
)
|
||||
require.NoError(t, err, "could not create thread-safe nuclei engine")
|
||||
t.Cleanup(ne.Close)
|
||||
var matches atomic.Int64
|
||||
ne.GlobalResultCallback(func(*output.ResultEvent) { matches.Add(1) })
|
||||
|
||||
err = ne.ExecuteNucleiWithOptsCtx(context.Background(), []string{srv.URL},
|
||||
nuclei.WithTemplatesOrWorkflows(nuclei.TemplateSources{Templates: []string{templatePath}}),
|
||||
nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"dos"}}),
|
||||
)
|
||||
|
||||
require.ErrorIs(t, err, nuclei.ErrNoTemplatesAvailable,
|
||||
"ignore-file dos exclusion must survive a Tags-only per-execution filter")
|
||||
assert.Zero(t, matches.Load())
|
||||
assert.False(t, reached.Load())
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package nuclei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestEphemeralObjectsUseExecutionLocalParser(t *testing.T) {
|
||||
baseParser := templates.NewParser()
|
||||
baseParser.Cache().Store("parsed", &templates.Template{}, []byte("raw"), nil)
|
||||
base := &NucleiEngine{parser: baseParser}
|
||||
|
||||
first, err := createEphemeralObjects(context.Background(), base, types.DefaultOptions(), nil)
|
||||
require.NoError(t, err)
|
||||
second, err := createEphemeralObjects(context.Background(), base, types.DefaultOptions(), nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { closeEphemeralObjects(second) })
|
||||
|
||||
firstParser, ok := first.executerOpts.Parser.(*templates.Parser)
|
||||
require.True(t, ok)
|
||||
secondParser, ok := second.executerOpts.Parser.(*templates.Parser)
|
||||
require.True(t, ok)
|
||||
|
||||
require.NotSame(t, baseParser, firstParser)
|
||||
require.NotSame(t, firstParser, secondParser)
|
||||
require.Same(t, baseParser.Cache(), firstParser.Cache())
|
||||
require.Same(t, baseParser.Cache(), secondParser.Cache())
|
||||
require.NotSame(t, firstParser.CompiledCache(), secondParser.CompiledCache())
|
||||
require.False(t, first.executerOpts.DoNotCache)
|
||||
|
||||
firstParser.CompiledCache().StoreWithoutRaw("compiled", &templates.Template{}, nil)
|
||||
require.Equal(t, 1, firstParser.CompiledCount())
|
||||
require.Zero(t, secondParser.CompiledCount())
|
||||
|
||||
closeEphemeralObjects(first)
|
||||
require.Zero(t, firstParser.CompiledCount())
|
||||
require.Equal(t, 1, baseParser.ParsedCount())
|
||||
}
|
||||
|
||||
func TestRestoreBaseExcludeTags(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
base []string
|
||||
current []string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "per-execution filter cleared the deny-list",
|
||||
base: []string{"dos", "fuzz"},
|
||||
current: nil,
|
||||
expected: []string{"dos", "fuzz"},
|
||||
},
|
||||
{
|
||||
name: "per-execution exclusions are kept alongside the baseline",
|
||||
base: []string{"dos"},
|
||||
current: []string{"custom"},
|
||||
expected: []string{"custom", "dos"},
|
||||
},
|
||||
{
|
||||
name: "entries already present are not duplicated",
|
||||
base: []string{"dos", "fuzz"},
|
||||
current: []string{"fuzz"},
|
||||
expected: []string{"fuzz", "dos"},
|
||||
},
|
||||
{
|
||||
name: "no baseline is a no-op",
|
||||
base: nil,
|
||||
current: []string{"custom"},
|
||||
expected: []string{"custom"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
opts := &types.Options{ExcludeTags: tt.current}
|
||||
restoreBaseExcludeTags(tt.base, opts)
|
||||
assert.Equal(t, tt.expected, []string(opts.ExcludeTags))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRestoreBaseExcludeTagsIsIdempotent matters because the helper runs on
|
||||
// every execution against a long-lived engine.
|
||||
func TestRestoreBaseExcludeTagsIsIdempotent(t *testing.T) {
|
||||
opts := &types.Options{}
|
||||
restoreBaseExcludeTags([]string{"dos", "fuzz"}, opts)
|
||||
restoreBaseExcludeTags([]string{"dos", "fuzz"}, opts)
|
||||
|
||||
assert.Equal(t, []string{"dos", "fuzz"}, []string(opts.ExcludeTags))
|
||||
}
|
||||
|
||||
// TestRestoreBaseExcludeTagsDoesNotMutateCallerSlice guards the concurrent
|
||||
// ThreadSafeNucleiEngine case where WithTemplateFilters assigns a caller-owned
|
||||
// ExcludeTags slice with spare capacity: a naive append would race and mutate
|
||||
// the caller's backing array.
|
||||
func TestRestoreBaseExcludeTagsDoesNotMutateCallerSlice(t *testing.T) {
|
||||
shared := make([]string, 1, 8)
|
||||
shared[0] = "custom"
|
||||
base := []string{"dos", "fuzz"}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 32; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
opts := &types.Options{ExcludeTags: shared}
|
||||
restoreBaseExcludeTags(base, opts)
|
||||
if got := []string(opts.ExcludeTags); len(got) != 3 || got[0] != "custom" || got[1] != "dos" || got[2] != "fuzz" {
|
||||
panic(got)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
assert.Equal(t, []string{"custom"}, shared[:1])
|
||||
assert.Equal(t, make([]string, 7), shared[1:cap(shared)])
|
||||
}
|
||||
+27
-4
@@ -135,13 +135,32 @@ func (e *NucleiEngine) applyRequiredDefaults(ctx context.Context) {
|
||||
e.opts.ExcludeTags = []string{}
|
||||
}
|
||||
|
||||
// these templates are known to have weak matchers
|
||||
// and idea is to disable them to avoid false positives
|
||||
e.opts.ExcludeTags = append(e.opts.ExcludeTags, config.ReadIgnoreFile().Tags...)
|
||||
|
||||
e.inputProvider = provider.NewSimpleInputProvider()
|
||||
}
|
||||
|
||||
func (e *NucleiEngine) loadIgnoreFile() error {
|
||||
ignoreFile, err := config.ReadIgnoreFile()
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
e.Logger.Warning().Msgf("Could not read active .nuclei-ignore file: %s; continuing without ignore exclusions", err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// .nuclei-ignore blocks templates that should not run by default, by tag
|
||||
// (dos, fuzz, ...) and by path (templates known to have weak matchers, which
|
||||
// would otherwise generate false positives). Both sections are applied, as
|
||||
// the CLI runner does — applying only the tags leaves the `files` section
|
||||
// inert for every SDK consumer.
|
||||
e.opts.ExcludeTags = append(e.opts.ExcludeTags, ignoreFile.Tags...)
|
||||
e.opts.ExcludedTemplates = append(e.opts.ExcludedTemplates, ignoreFile.Files...)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// init
|
||||
func (e *NucleiEngine) init(ctx context.Context) error {
|
||||
if err := config.DefaultConfig.InitializationError(); err != nil {
|
||||
@@ -165,6 +184,10 @@ func (e *NucleiEngine) init(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := e.loadIgnoreFile(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if e.opts.Parser != nil {
|
||||
if op, ok := e.opts.Parser.(*templates.Parser); ok {
|
||||
e.parser = op
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
nuclei "github.com/projectdiscovery/nuclei/v3/lib"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -120,3 +121,37 @@ func TestWithOptionsRateLimitSetsRuntimeLimiter(t *testing.T) {
|
||||
require.NotNil(t, execOpts.RateLimiter, "rate limiter should be initialized")
|
||||
require.Equal(t, opts.RateLimit, int(execOpts.RateLimiter.GetLimit()), "runtime limiter should match rate limit from WithOptions")
|
||||
}
|
||||
|
||||
// TestIgnoreFileFilesSectionIsApplied verifies the SDK applies BOTH sections of
|
||||
// .nuclei-ignore, matching the CLI runner. The files section suppresses
|
||||
// templates known to have weak matchers, so skipping it hands SDK consumers
|
||||
// false positives the CLI would never report.
|
||||
func TestIgnoreFileFilesSectionIsApplied(t *testing.T) {
|
||||
previousConfigDir := config.DefaultConfig.GetConfigDir()
|
||||
previousTemplatesDir := config.DefaultConfig.TemplatesDirectory
|
||||
previousStateDir := config.DefaultConfig.GetStateDir()
|
||||
t.Cleanup(func() {
|
||||
config.DefaultConfig.SetConfigDir(previousConfigDir)
|
||||
config.DefaultConfig.SetStateDir(previousStateDir)
|
||||
config.DefaultConfig.SetTemplatesDir(previousTemplatesDir)
|
||||
})
|
||||
|
||||
dir := t.TempDir()
|
||||
config.DefaultConfig.SetConfigDir(dir)
|
||||
config.DefaultConfig.SetTemplatesDir(dir)
|
||||
|
||||
ignoreFile := "tags:\n - dos\nfiles:\n - http/fuzzing/wordpress-themes-detect.yaml\n"
|
||||
require.NoError(t, os.WriteFile(config.DefaultConfig.GetActiveIgnoreFilePath(), []byte(ignoreFile), 0o600))
|
||||
|
||||
ne, err := nuclei.NewNucleiEngineCtx(context.Background(),
|
||||
nuclei.WithOptions(types.DefaultOptions()),
|
||||
nuclei.DisableUpdateCheck(),
|
||||
)
|
||||
require.NoError(t, err, "could not create nuclei engine")
|
||||
defer ne.Close()
|
||||
|
||||
require.Contains(t, ne.Options().ExcludeTags, "dos",
|
||||
"tags section should reach ExcludeTags")
|
||||
require.Contains(t, ne.Options().ExcludedTemplates, "http/fuzzing/wordpress-themes-detect.yaml",
|
||||
"files section should reach ExcludedTemplates")
|
||||
}
|
||||
@@ -28,6 +28,9 @@ var knownLeaks = []goleak.Option{
|
||||
goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/ratelimit"),
|
||||
goleak.IgnoreAnyFunction("github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate.StartActiveMemGuardian.func1"),
|
||||
// NOTE(dwisiswant0): This is an indirect call and not used anywhere, not
|
||||
// even within SOPS (#5841). See `go mod why go.opencensus.io/stats/view`.
|
||||
goleak.IgnoreAnyContainingPkg("go.opencensus.io/stats/view"),
|
||||
}
|
||||
|
||||
func TestSimpleNuclei(t *testing.T) {
|
||||
|
||||
@@ -880,6 +880,11 @@
|
||||
"title": "optional disable cookie reuse",
|
||||
"description": "Optional setting that disables cookie reuse"
|
||||
},
|
||||
"disable-http-cache": {
|
||||
"type": "boolean",
|
||||
"title": "disable HTTP cache",
|
||||
"description": "Turns off HTTP caching for this request; cannot enable cache when -http-cache is unset"
|
||||
},
|
||||
"read-all": {
|
||||
"type": "boolean",
|
||||
"title": "force read all body",
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
package authx
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
|
||||
@@ -19,6 +23,9 @@ type LazyFetchSecret func(d *Dynamic) error
|
||||
type fetchState struct {
|
||||
once sync.Once
|
||||
err error
|
||||
// owner is the goroutine running the fetch callback. Nested Fetch/GetStrategies
|
||||
// on that goroutine must not wait on once (sync.Once deadlocks on re-entry).
|
||||
owner atomic.Int64
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -191,6 +198,11 @@ func (d *Dynamic) applyValuesToSecret(secret *Secret) error {
|
||||
|
||||
// GetStrategies returns the auth strategies for the dynamic secret
|
||||
func (d *Dynamic) GetStrategies() []AuthStrategy {
|
||||
// Nested lookup from inside the fetch callback (login template ApplyAuth)
|
||||
// must not wait on Fetch and must not apply still-unresolved secrets.
|
||||
if d.fetchingOnThisGoroutine() {
|
||||
return nil
|
||||
}
|
||||
// Ensure fetch has completed before returning strategies.
|
||||
// Fetch errors are treated as non-fatal here so a failed dynamic auth fetch
|
||||
// does not terminate the entire scan process.
|
||||
@@ -219,7 +231,13 @@ func (d *Dynamic) Fetch(isFatal bool) error {
|
||||
return errkit.New("dynamic secret not validated: call Validate() before Fetch()")
|
||||
}
|
||||
|
||||
if d.fetchingOnThisGoroutine() {
|
||||
return d.fetchState.err
|
||||
}
|
||||
|
||||
d.fetchState.once.Do(func() {
|
||||
d.fetchState.owner.Store(goroutineID())
|
||||
defer d.fetchState.owner.Store(0)
|
||||
if d.fetchCallback == nil {
|
||||
d.fetchState.err = errkit.New("dynamic secret fetch callback not set: call SetLazyFetchCallback() before Fetch()")
|
||||
return
|
||||
@@ -240,3 +258,31 @@ func (d *Dynamic) Error() error {
|
||||
}
|
||||
return d.fetchState.err
|
||||
}
|
||||
|
||||
func (d *Dynamic) fetchingOnThisGoroutine() bool {
|
||||
if d == nil || d.fetchState == nil {
|
||||
return false
|
||||
}
|
||||
owner := d.fetchState.owner.Load()
|
||||
return owner != 0 && owner == goroutineID()
|
||||
}
|
||||
|
||||
func goroutineID() int64 {
|
||||
var buf [64]byte
|
||||
n := runtime.Stack(buf[:], false)
|
||||
s := buf[:n]
|
||||
const prefix = "goroutine "
|
||||
if !bytes.HasPrefix(s, []byte(prefix)) {
|
||||
return 0
|
||||
}
|
||||
s = s[len(prefix):]
|
||||
i := bytes.IndexByte(s, ' ')
|
||||
if i <= 0 {
|
||||
return 0
|
||||
}
|
||||
id, err := strconv.ParseInt(string(s[:i]), 10, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return id
|
||||
}
|
||||
@@ -226,6 +226,78 @@ func TestDynamicFetchConcurrent(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestDynamicFetchReentry(t *testing.T) {
|
||||
t.Run("nested Fetch from callback does not deadlock", func(t *testing.T) {
|
||||
d := &Dynamic{
|
||||
TemplatePath: "auth.yaml",
|
||||
Variables: []KV{{Key: "user", Value: "admin"}},
|
||||
}
|
||||
require.NoError(t, d.Validate())
|
||||
|
||||
var nestedErr error
|
||||
d.SetLazyFetchCallback(func(dynamic *Dynamic) error {
|
||||
nestedErr = dynamic.Fetch(false)
|
||||
dynamic.Extracted = map[string]interface{}{"token": "abc"}
|
||||
return nil
|
||||
})
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- d.Fetch(false)
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, nestedErr)
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Fetch deadlocked on re-entry")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nested GetStrategies does not apply unresolved secrets", func(t *testing.T) {
|
||||
d := &Dynamic{
|
||||
Secret: &Secret{
|
||||
Type: "Header",
|
||||
Domains: []string{"example.com"},
|
||||
Headers: []KV{{Key: "X-Token", Value: "{{token}}"}},
|
||||
},
|
||||
TemplatePath: "auth.yaml",
|
||||
Variables: []KV{{Key: "user", Value: "admin"}},
|
||||
}
|
||||
require.NoError(t, d.Validate())
|
||||
|
||||
d.SetLazyFetchCallback(func(dynamic *Dynamic) error {
|
||||
req, err := http.NewRequest(http.MethodGet, "https://example.com", nil)
|
||||
require.NoError(t, err)
|
||||
strategy := &DynamicAuthStrategy{Dynamic: *dynamic}
|
||||
strategy.Apply(req)
|
||||
require.Empty(t, req.Header.Get("X-Token"), "login request must not receive secret-file auth during fetch")
|
||||
require.Nil(t, dynamic.GetStrategies())
|
||||
dynamic.Extracted = map[string]interface{}{"token": "abc"}
|
||||
return nil
|
||||
})
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- d.Fetch(false)
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("GetStrategies deadlocked on re-entry")
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, "https://example.com", nil)
|
||||
require.NoError(t, err)
|
||||
strategy := &DynamicAuthStrategy{Dynamic: *d}
|
||||
strategy.Apply(req)
|
||||
require.Equal(t, "abc", req.Header.Get("X-Token"), "scan requests must still receive secrets after fetch")
|
||||
})
|
||||
}
|
||||
|
||||
func TestDynamicValidate(t *testing.T) {
|
||||
t.Run("missing template path", func(t *testing.T) {
|
||||
d := &Dynamic{
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
package authx
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/getsops/sops/v3"
|
||||
"github.com/getsops/sops/v3/decrypt"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
"github.com/projectdiscovery/utils/generic"
|
||||
@@ -205,18 +208,30 @@ func (c *Cookie) Parse() error {
|
||||
|
||||
// GetAuthDataFromFile reads the auth data from file
|
||||
func GetAuthDataFromFile(file string) (*Authx, error) {
|
||||
ext := filepath.Ext(file)
|
||||
if !generic.EqualsAny(ext, ".yml", ".yaml", ".json") {
|
||||
return nil, fmt.Errorf("invalid file extension: supported extensions are .yml,.yaml and .json got %s", ext)
|
||||
var data []byte
|
||||
|
||||
ext := strings.TrimPrefix(filepath.Ext(file), ".")
|
||||
if !generic.EqualsAny(ext, "yml", "yaml", "json") {
|
||||
return nil, fmt.Errorf(`unsupported extensions: valid extensions are "yml", "yaml", and "json"; got %q`, ext)
|
||||
}
|
||||
bin, err := os.ReadFile(file)
|
||||
|
||||
data, err := decrypt.File(file, ext)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if errors.Is(err, sops.MetadataNotFound) {
|
||||
data, err = os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not read secret file: %w", err)
|
||||
}
|
||||
} else {
|
||||
return nil, fmt.Errorf("could not decrypt file: %w", err)
|
||||
}
|
||||
}
|
||||
if ext == ".yml" || ext == ".yaml" {
|
||||
return GetAuthDataFromYAML(bin)
|
||||
|
||||
if ext == "yml" || ext == "yaml" {
|
||||
return GetAuthDataFromYAML(data)
|
||||
}
|
||||
return GetAuthDataFromJSON(bin)
|
||||
|
||||
return GetAuthDataFromJSON(data)
|
||||
}
|
||||
|
||||
// GetTemplatePathsFromSecretFile reads the template IDs from the secret file
|
||||
|
||||
@@ -1,20 +1,100 @@
|
||||
package authx
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"filippo.io/age"
|
||||
"github.com/getsops/sops/v3"
|
||||
"github.com/getsops/sops/v3/aes"
|
||||
sopsage "github.com/getsops/sops/v3/age"
|
||||
"github.com/getsops/sops/v3/config"
|
||||
"github.com/getsops/sops/v3/stores/yaml"
|
||||
"github.com/getsops/sops/v3/version"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSecretsUnmarshal(t *testing.T) {
|
||||
loc := "testData/example-auth.yaml"
|
||||
data, err := GetAuthDataFromFile(loc)
|
||||
require.Nil(t, err, "could not read secrets file")
|
||||
data, err := GetAuthDataFromFile("testData/example-auth.yaml")
|
||||
require.NoError(t, err, "could not read secrets file")
|
||||
require.NotNil(t, data, "could not read secrets file")
|
||||
|
||||
t.Run("encrypted", func(t *testing.T) {
|
||||
loc := encryptAuthFile(t, "testData/example-auth.yaml")
|
||||
decryptedData, err := GetAuthDataFromFile(loc)
|
||||
require.NoError(t, err, "could not read encrypted secrets file")
|
||||
require.NotNil(t, decryptedData, "could not read secrets file")
|
||||
require.Equal(t, data, decryptedData, "decrypted data should match original data")
|
||||
|
||||
for _, s := range decryptedData.Secrets {
|
||||
require.NoError(t, s.Validate(), "could not validate secret")
|
||||
}
|
||||
|
||||
for _, d := range decryptedData.Dynamic {
|
||||
require.NoError(t, d.Validate(), "could not validate dynamic")
|
||||
}
|
||||
})
|
||||
|
||||
for _, s := range data.Secrets {
|
||||
require.Nil(t, s.Validate(), "could not validate secret")
|
||||
require.NoError(t, s.Validate(), "could not validate secret")
|
||||
}
|
||||
|
||||
for _, d := range data.Dynamic {
|
||||
require.Nil(t, d.Validate(), "could not validate dynamic")
|
||||
require.NoError(t, d.Validate(), "could not validate dynamic")
|
||||
}
|
||||
}
|
||||
|
||||
func encryptAuthFile(t *testing.T, source string) string {
|
||||
t.Helper()
|
||||
|
||||
plaintext, err := os.ReadFile(source)
|
||||
require.NoError(t, err)
|
||||
|
||||
identity, err := age.GenerateX25519Identity()
|
||||
require.NoError(t, err)
|
||||
t.Setenv(sopsage.SopsAgeKeyEnv, identity.String())
|
||||
|
||||
masterKey, err := sopsage.MasterKeyFromRecipient(identity.Recipient().String())
|
||||
require.NoError(t, err)
|
||||
|
||||
dataKey := make([]byte, 32)
|
||||
_, err = rand.Read(dataKey)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, masterKey.Encrypt(dataKey))
|
||||
|
||||
store := yaml.NewStore(&config.YAMLStoreConfig{})
|
||||
branches, err := store.LoadPlainFile(plaintext)
|
||||
require.NoError(t, err)
|
||||
|
||||
tree := sops.Tree{
|
||||
Branches: branches,
|
||||
Metadata: sops.Metadata{
|
||||
EncryptedRegex: `^(password|username|token|value|key|raw)$`,
|
||||
KeyGroups: []sops.KeyGroup{{masterKey}},
|
||||
Version: version.Version,
|
||||
},
|
||||
}
|
||||
|
||||
cipher := aes.NewCipher()
|
||||
mac, err := tree.Encrypt(dataKey, cipher)
|
||||
require.NoError(t, err)
|
||||
tree.Metadata.LastModified = time.Now().UTC()
|
||||
tree.Metadata.MessageAuthenticationCode, err = cipher.Encrypt(
|
||||
mac,
|
||||
dataKey,
|
||||
tree.Metadata.LastModified.Format(time.RFC3339),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
encrypted, err := store.EmitEncryptedFile(tree)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, string(encrypted), "sops:")
|
||||
require.NotContains(t, string(encrypted), "1a2b3c4d5e6f7g8h9i0j")
|
||||
|
||||
path := filepath.Join(t.TempDir(), "example-auth.yaml")
|
||||
require.NoError(t, os.WriteFile(path, encrypted, 0600))
|
||||
return path
|
||||
}
|
||||
@@ -1,50 +1,141 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
|
||||
)
|
||||
|
||||
// maxIgnoreFileBytes caps untrusted ignore-file payloads (PDTM downloads,
|
||||
// template archives) before YAML decode.
|
||||
const maxIgnoreFileBytes = 1 << 20
|
||||
|
||||
var errMultipleIgnoreFileDocuments = errors.New("multiple YAML documents are not supported")
|
||||
|
||||
// IgnoreFile is an internal nuclei template blocking configuration file
|
||||
type IgnoreFile struct {
|
||||
Tags []string `yaml:"tags"`
|
||||
Files []string `yaml:"files"`
|
||||
}
|
||||
|
||||
// ReadIgnoreFile reads the .nuclei-ignore file returning blocked tags and paths.
|
||||
func ReadIgnoreFile() IgnoreFile {
|
||||
// ReadIgnoreFile reads the active .nuclei-ignore file and returns blocked tags
|
||||
// and paths. Callers must decide whether a read error is recoverable. Malformed
|
||||
// content is returned as an error to prevent a fail-open scan.
|
||||
func ReadIgnoreFile() (IgnoreFile, error) {
|
||||
path := DefaultConfig.GetActiveIgnoreFilePath()
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Could not read .nuclei-ignore file %q: %s\n", path, err)
|
||||
return IgnoreFile{}
|
||||
return IgnoreFile{}, fmt.Errorf("error opening %q: %w", path, err)
|
||||
}
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
}()
|
||||
|
||||
ignore := IgnoreFile{}
|
||||
if err := yaml.NewDecoder(file).Decode(&ignore); err != nil {
|
||||
gologger.Error().Msgf("Could not parse .nuclei-ignore file %q: %s\n", path, err)
|
||||
return IgnoreFile{}
|
||||
ignore, err := decodeIgnoreFile(file, false)
|
||||
if err != nil {
|
||||
return IgnoreFile{}, fmt.Errorf("error parsing %q: %w", path, err)
|
||||
}
|
||||
|
||||
return ignore
|
||||
return ignore, nil
|
||||
}
|
||||
|
||||
// WriteActiveIgnoreFile atomically replaces the active root ignore file.
|
||||
func (c *Config) WriteActiveIgnoreFile(data []byte) error {
|
||||
if err := atomicWriteFile(c.GetActiveIgnoreFilePath(), data, 0o600); err != nil {
|
||||
path := c.GetActiveIgnoreFilePath()
|
||||
if err := validateIgnoreFileContents(data); err != nil {
|
||||
return fmt.Errorf("invalid active .nuclei-ignore file %q: %w", path, err)
|
||||
}
|
||||
|
||||
if err := atomicWriteFile(path, data, 0o600); err != nil {
|
||||
return fmt.Errorf("write active .nuclei-ignore file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateIgnoreFileContents(data []byte) error {
|
||||
if len(data) > maxIgnoreFileBytes {
|
||||
return fmt.Errorf("payload exceeds %d bytes", maxIgnoreFileBytes)
|
||||
}
|
||||
if len(bytes.TrimSpace(data)) == 0 {
|
||||
return errors.New("empty document")
|
||||
}
|
||||
|
||||
if _, err := decodeIgnoreFile(bytes.NewReader(data), true); err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
return errors.New("empty document")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||
var raw map[string]any
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return err
|
||||
}
|
||||
if raw == nil {
|
||||
return errors.New("document must be a YAML mapping")
|
||||
}
|
||||
|
||||
allowed := ignoreFileFieldNames()
|
||||
found := false
|
||||
for key := range raw {
|
||||
if !slices.Contains(allowed, key) {
|
||||
return fmt.Errorf("unknown field %q", key)
|
||||
}
|
||||
found = true
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("document must define at least one of %s", strings.Join(allowed, ", "))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func decodeIgnoreFile(r io.Reader, knownFields bool) (IgnoreFile, error) {
|
||||
decoder := yaml.NewDecoder(r)
|
||||
decoder.KnownFields(knownFields)
|
||||
|
||||
ignore := IgnoreFile{}
|
||||
if err := decoder.Decode(&ignore); err != nil {
|
||||
return IgnoreFile{}, err
|
||||
}
|
||||
|
||||
var additionalDocument any
|
||||
if err := decoder.Decode(&additionalDocument); !errors.Is(err, io.EOF) {
|
||||
if err != nil {
|
||||
return IgnoreFile{}, err
|
||||
}
|
||||
|
||||
return IgnoreFile{}, errMultipleIgnoreFileDocuments
|
||||
}
|
||||
|
||||
return ignore, nil
|
||||
}
|
||||
|
||||
func ignoreFileFieldNames() []string {
|
||||
typ := reflect.TypeOf(IgnoreFile{})
|
||||
names := make([]string, 0, typ.NumField())
|
||||
for i := 0; i < typ.NumField(); i++ {
|
||||
name, _, _ := strings.Cut(typ.Field(i).Tag.Get("yaml"), ",")
|
||||
if name == "" || name == "-" {
|
||||
continue
|
||||
}
|
||||
names = append(names, name)
|
||||
}
|
||||
slices.Sort(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// IgnoreFileNeedsUpdate reports whether the active ignore file is
|
||||
// missing or differs from the latest known hash.
|
||||
func (c *Config) IgnoreFileNeedsUpdate(latestHash string) bool {
|
||||
|
||||
@@ -1,33 +1,106 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReadIgnoreFileAllowsMissingOrInvalidActiveFile(t *testing.T) {
|
||||
func setActiveIgnoreFileRoot(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
root := t.TempDir()
|
||||
cfg := DefaultConfig
|
||||
oldRoot := cfg.TemplatesDirectory
|
||||
t.Cleanup(func() { cfg.SetTemplatesDir(oldRoot) })
|
||||
cfg.SetTemplatesDir(root)
|
||||
return root
|
||||
}
|
||||
|
||||
if ignore := ReadIgnoreFile(); len(ignore.Tags) != 0 || len(ignore.Files) != 0 {
|
||||
func TestReadIgnoreFileReportsMissingActiveFile(t *testing.T) {
|
||||
root := setActiveIgnoreFileRoot(t)
|
||||
path := filepath.Join(root, NucleiIgnoreFileName)
|
||||
|
||||
ignore, err := ReadIgnoreFile()
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("read missing ignore file error = %v, want os.ErrNotExist", err)
|
||||
}
|
||||
if len(ignore.Tags) != 0 || len(ignore.Files) != 0 {
|
||||
t.Fatalf("missing ignore file = %+v, want empty", ignore)
|
||||
}
|
||||
if message := err.Error(); !strings.Contains(message, strconv.Quote(path)) || !strings.Contains(message, "error opening") {
|
||||
t.Fatalf("missing ignore file error = %q, want open failure and active path", message)
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.WriteFile(filepath.Join(root, NucleiIgnoreFileName), []byte("tags: ["), 0o600); err != nil {
|
||||
func TestReadIgnoreFileRejectsCorruptActiveFile(t *testing.T) {
|
||||
root := setActiveIgnoreFileRoot(t)
|
||||
path := filepath.Join(root, NucleiIgnoreFileName)
|
||||
|
||||
if err := os.WriteFile(path, []byte("tags: ["), 0o600); err != nil {
|
||||
t.Fatalf("write invalid ignore file: %v", err)
|
||||
}
|
||||
if ignore := ReadIgnoreFile(); len(ignore.Tags) != 0 || len(ignore.Files) != 0 {
|
||||
t.Fatalf("invalid ignore file = %+v, want empty", ignore)
|
||||
ignore, err := ReadIgnoreFile()
|
||||
if err == nil {
|
||||
t.Fatalf("read corrupt ignore file = %+v, want error", ignore)
|
||||
}
|
||||
if message := err.Error(); !strings.Contains(message, strconv.Quote(path)) || !strings.Contains(message, "error parsing") {
|
||||
t.Fatalf("corrupt ignore file error = %q, want parse failure and active path", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadIgnoreFileRejectsMalformedTrailingDocument(t *testing.T) {
|
||||
root := setActiveIgnoreFileRoot(t)
|
||||
path := filepath.Join(root, NucleiIgnoreFileName)
|
||||
|
||||
if err := os.WriteFile(path, []byte("tags: [weak]\n---\ntags: ["), 0o600); err != nil {
|
||||
t.Fatalf("write ignore file: %v", err)
|
||||
}
|
||||
ignore, err := ReadIgnoreFile()
|
||||
if err == nil {
|
||||
t.Fatalf("read ignore file = %+v, want trailing parse error", ignore)
|
||||
}
|
||||
if len(ignore.Tags) != 0 || len(ignore.Files) != 0 {
|
||||
t.Fatalf("ignore file after trailing parse error = %+v, want empty", ignore)
|
||||
}
|
||||
if message := err.Error(); !strings.Contains(message, strconv.Quote(path)) || !strings.Contains(message, "error parsing") {
|
||||
t.Fatalf("trailing parse error = %q, want parse failure and active path", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadIgnoreFileRejectsAdditionalDocument(t *testing.T) {
|
||||
root := setActiveIgnoreFileRoot(t)
|
||||
path := filepath.Join(root, NucleiIgnoreFileName)
|
||||
|
||||
if err := os.WriteFile(path, []byte("tags: [weak]\n---\nfiles: [blocked.yaml]\n"), 0o600); err != nil {
|
||||
t.Fatalf("write ignore file: %v", err)
|
||||
}
|
||||
ignore, err := ReadIgnoreFile()
|
||||
if err == nil {
|
||||
t.Fatalf("read ignore file = %+v, want multiple-document error", ignore)
|
||||
}
|
||||
if len(ignore.Tags) != 0 || len(ignore.Files) != 0 {
|
||||
t.Fatalf("ignore file with additional document = %+v, want empty", ignore)
|
||||
}
|
||||
if message := err.Error(); !strings.Contains(message, strconv.Quote(path)) || !strings.Contains(message, "multiple YAML documents") {
|
||||
t.Fatalf("multiple-document error = %q, want active path and document count failure", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadIgnoreFileParsesActiveFile(t *testing.T) {
|
||||
root := setActiveIgnoreFileRoot(t)
|
||||
|
||||
if err := os.WriteFile(filepath.Join(root, NucleiIgnoreFileName), []byte("tags: [weak]\nfiles: [blocked.yaml]\n"), 0o600); err != nil {
|
||||
t.Fatalf("write ignore file: %v", err)
|
||||
}
|
||||
ignore := ReadIgnoreFile()
|
||||
ignore, err := ReadIgnoreFile()
|
||||
if err != nil {
|
||||
t.Fatalf("read ignore file: %v", err)
|
||||
}
|
||||
if len(ignore.Tags) != 1 || ignore.Tags[0] != "weak" {
|
||||
t.Fatalf("ignore tags = %v, want [weak]", ignore.Tags)
|
||||
}
|
||||
@@ -36,6 +109,77 @@ func TestReadIgnoreFileAllowsMissingOrInvalidActiveFile(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteActiveIgnoreFileRejectsInvalidPayloads(t *testing.T) {
|
||||
root := setActiveIgnoreFileRoot(t)
|
||||
cfg := DefaultConfig
|
||||
path := filepath.Join(root, NucleiIgnoreFileName)
|
||||
valid := []byte("tags: [weak]\nfiles: [blocked.yaml]\n")
|
||||
if err := os.WriteFile(path, valid, 0o600); err != nil {
|
||||
t.Fatalf("seed ignore file: %v", err)
|
||||
}
|
||||
|
||||
cases := map[string][]byte{
|
||||
"proxy html": []byte("<!DOCTYPE HTML>\n<html>\n<head>\n<meta name=\"description\" content=\"Zscaler: blocked\">\n</head>\n</html>\n"),
|
||||
"unrelated yaml": []byte("error: access denied\nmessage: blocked by proxy\n"),
|
||||
"empty": []byte(" \n"),
|
||||
"empty mapping": []byte("{}\n"),
|
||||
"null document": []byte("null\n"),
|
||||
"template yaml": []byte("id: example\ninfo:\n name: example\n"),
|
||||
"additional document": []byte("tags: [weak]\n---\nfiles: [blocked.yaml]\n"),
|
||||
"malformed yaml": []byte("tags: ["),
|
||||
"oversized": bytes.Repeat([]byte("a"), maxIgnoreFileBytes+1),
|
||||
}
|
||||
|
||||
for name, payload := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
err := cfg.WriteActiveIgnoreFile(payload)
|
||||
if err == nil {
|
||||
t.Fatal("write invalid ignore file succeeded")
|
||||
}
|
||||
if message := err.Error(); !strings.Contains(message, strconv.Quote(path)) {
|
||||
t.Fatalf("invalid write error = %q, want active path", message)
|
||||
}
|
||||
got, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read ignore file: %v", readErr)
|
||||
}
|
||||
if !bytes.Equal(got, valid) {
|
||||
t.Fatalf("invalid payload overwrote ignore file: got %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteActiveIgnoreFileAcceptsKnownShapes(t *testing.T) {
|
||||
root := setActiveIgnoreFileRoot(t)
|
||||
cfg := DefaultConfig
|
||||
path := filepath.Join(root, NucleiIgnoreFileName)
|
||||
|
||||
cases := map[string][]byte{
|
||||
"tags and files": []byte("tags: [weak]\nfiles: [blocked.yaml]\n"),
|
||||
"tags only": []byte("tags: []\n"),
|
||||
"files only": []byte("files: [blocked.yaml]\n"),
|
||||
}
|
||||
|
||||
for name, payload := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if err := cfg.WriteActiveIgnoreFile(payload); err != nil {
|
||||
t.Fatalf("write ignore file: %v", err)
|
||||
}
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read ignore file: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, payload) {
|
||||
t.Fatalf("ignore file = %q, want %q", got, payload)
|
||||
}
|
||||
if _, err := ReadIgnoreFile(); err != nil {
|
||||
t.Fatalf("read back written ignore file: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetTemplatesDirDoesNotCopyLegacyIgnoreFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
configDir := filepath.Join(dir, "config")
|
||||
|
||||
@@ -315,6 +315,7 @@ func (c *Config) WriteTemplatesIndex(index map[string]string) error {
|
||||
indexFile := c.GetTemplateIndexFilePath()
|
||||
|
||||
var buff bytes.Buffer
|
||||
|
||||
for k, v := range index {
|
||||
_, _ = buff.WriteString(k + "," + v + "\n")
|
||||
}
|
||||
@@ -329,7 +330,6 @@ func (c *Config) createConfigDirIfNotExists() error {
|
||||
return errkit.Newf("could not create nuclei config directory at %s: %v", c.configDir, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
//go:build !windows
|
||||
|
||||
package config
|
||||
|
||||
import "os"
|
||||
|
||||
func replaceTemplatesConfigFile(temporaryPath, configFilePath string) error {
|
||||
return os.Rename(temporaryPath, configFilePath)
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"unsafe"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
var replaceFile = windows.NewLazySystemDLL("kernel32.dll").NewProc("ReplaceFileW")
|
||||
|
||||
func replaceTemplatesConfigFile(temporaryPath, configFilePath string) error {
|
||||
temporary, err := windows.UTF16PtrFromString(temporaryPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
target, err := windows.UTF16PtrFromString(configFilePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := os.Lstat(configFilePath); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
return windows.MoveFileEx(temporary, target, windows.MOVEFILE_WRITE_THROUGH)
|
||||
}
|
||||
|
||||
replaced, _, callErr := replaceFile.Call(
|
||||
uintptr(unsafe.Pointer(target)),
|
||||
uintptr(unsafe.Pointer(temporary)),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
if replaced != 0 {
|
||||
return nil
|
||||
}
|
||||
if errors.Is(callErr, windows.ERROR_FILE_NOT_FOUND) {
|
||||
return windows.MoveFileEx(temporary, target, windows.MOVEFILE_WRITE_THROUGH)
|
||||
}
|
||||
|
||||
return callErr
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReplaceTemplatesConfigFile(t *testing.T) {
|
||||
for _, targetExists := range []bool{false, true} {
|
||||
name := "creates"
|
||||
if targetExists {
|
||||
name = "replaces"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
target := filepath.Join(directory, "config.json")
|
||||
if targetExists {
|
||||
if err := os.WriteFile(target, []byte("previous"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
temporary := filepath.Join(directory, "config.tmp")
|
||||
if err := os.WriteFile(temporary, []byte("current"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := replaceTemplatesConfigFile(temporary, target); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
contents, err := os.ReadFile(target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(contents) != "current" {
|
||||
t.Fatalf("expected replacement contents, got %q", contents)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -134,7 +134,7 @@ func atomicWriteFile(path string, data []byte, mode os.FileMode) error {
|
||||
|
||||
temp = nil
|
||||
|
||||
if err := os.Rename(tempPath, path); err != nil {
|
||||
if err := replaceTemplatesConfigFile(tempPath, path); err != nil {
|
||||
return fmt.Errorf("replace %q: %w", path, err)
|
||||
}
|
||||
|
||||
|
||||
@@ -823,7 +823,14 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) ([]*temp
|
||||
// tags, so it must be matched against the ignore-file tags specifically:
|
||||
// otherwise user-requested -exclude-tags drops would be mislabeled as
|
||||
// .nuclei-ignore exclusions.
|
||||
ignoreFileTags := config.ReadIgnoreFile().Tags
|
||||
ignoreFile, err := config.ReadIgnoreFile()
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
store.logger.Warning().Msgf("Could not read active .nuclei-ignore file: %s; continuing without ignore exclusions", err)
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ignoreFileTags := ignoreFile.Tags
|
||||
|
||||
noteExcludedByTag := func(templatePath string, metadata *index.Metadata) {
|
||||
if len(ignoreFileTags) == 0 || !slices.ContainsFunc(ignoreFileTags, metadata.HasTag) {
|
||||
return
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -60,6 +61,37 @@ func TestLoadTemplates(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestLoadTemplatesWithTagsRejectsCorruptIgnoreFileInValidationMode(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
ignoreFilePath := filepath.Join(root, config.NucleiIgnoreFileName)
|
||||
require.NoError(t, os.WriteFile(ignoreFilePath, []byte("tags: ["), 0o600))
|
||||
|
||||
cfg := config.DefaultConfig
|
||||
oldRoot := cfg.TemplatesDirectory
|
||||
t.Cleanup(func() { cfg.SetTemplatesDir(oldRoot) })
|
||||
cfg.SetTemplatesDir(root)
|
||||
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.Validate = true
|
||||
metadataIndex, err := metadataindex.NewIndex(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
store, err := New(&Config{
|
||||
Catalog: disk.NewCatalog(""),
|
||||
ExecutorOptions: &protocols.ExecutorOptions{
|
||||
Options: options,
|
||||
Parser: templates.NewParser(),
|
||||
},
|
||||
Logger: options.Logger,
|
||||
MetadataIndex: metadataIndex,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = store.LoadTemplatesWithTags(nil, nil)
|
||||
require.ErrorContains(t, err, "error parsing")
|
||||
require.ErrorContains(t, err, strconv.Quote(ignoreFilePath))
|
||||
}
|
||||
|
||||
func TestNewUsesConfiguredMetadataIndex(t *testing.T) {
|
||||
indexDir := t.TempDir()
|
||||
metadataIndex, err := metadataindex.NewIndex(indexDir)
|
||||
|
||||
+431
-240
@@ -4,15 +4,19 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/charmbracelet/glamour"
|
||||
"github.com/google/go-github/v30/github"
|
||||
"github.com/olekukonko/tablewriter"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
@@ -20,13 +24,13 @@ import (
|
||||
filepathutil "github.com/projectdiscovery/nuclei/v3/pkg/utils/filepath"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
mapsutil "github.com/projectdiscovery/utils/maps"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
updateutils "github.com/projectdiscovery/utils/update"
|
||||
)
|
||||
|
||||
const (
|
||||
checkSumFilePerm = 0644
|
||||
checkSumFilePerm = 0644
|
||||
templateOutputTemporaryPrefix = ".nuclei-write-"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -104,6 +108,10 @@ func (t *TemplateManager) updateIfOutdatedLocked() error {
|
||||
return t.FreshInstallIfNotExists()
|
||||
}
|
||||
|
||||
if err := recoverTemplateOwnership(config.DefaultConfig.TemplatesDirectory); err != nil {
|
||||
return errkit.Wrapf(err, "failed to recover template ownership at %s", config.DefaultConfig.TemplatesDirectory)
|
||||
}
|
||||
|
||||
needsUpdate := config.DefaultConfig.NeedsTemplateUpdate()
|
||||
|
||||
// NOTE(dwisiswant0): if PDTM API data is not available
|
||||
@@ -139,15 +147,19 @@ func (t *TemplateManager) installTemplatesAt(dir string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := recoverTemplateOwnership(dir); err != nil {
|
||||
return errkit.Wrapf(err, "failed to recover template ownership at %s", dir)
|
||||
}
|
||||
|
||||
ghrd, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTemplatesRepoName)
|
||||
if err != nil {
|
||||
return errkit.Wrapf(err, "failed to install templates at %s", dir)
|
||||
}
|
||||
|
||||
// write templates to disk
|
||||
_, err = t.writeTemplatesToDisk(ghrd, dir)
|
||||
if err != nil {
|
||||
return errkit.Wrapf(err, "failed to write templates to disk at %s", dir)
|
||||
writtenOutputs, writeErr := t.writeTemplatesToDisk(ghrd, dir)
|
||||
if _, finalizeErr := t.finalizeTemplateWrite(config.DefaultConfig, writtenOutputs, ghrd.Latest.GetTagName(), writeErr); finalizeErr != nil {
|
||||
return errkit.Wrapf(finalizeErr, "failed to finalize template installation at %s", dir)
|
||||
}
|
||||
|
||||
gologger.Info().Msgf("Successfully installed nuclei-templates at %s", dir)
|
||||
@@ -155,13 +167,204 @@ func (t *TemplateManager) installTemplatesAt(dir string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type templateArchiveFetcher func(version string) (*bytes.Reader, error)
|
||||
|
||||
func commitTemplateVersion(cfg *config.Config, version string) error {
|
||||
previousVersion := cfg.TemplateVersion
|
||||
if err := cfg.SetTemplatesVersion(version); err != nil {
|
||||
cfg.TemplateVersion = previousVersion
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TemplateManager) finalizeTemplateRelease(cfg *config.Config, writtenOutputs map[string]string, version string) (map[string]string, error) {
|
||||
dir := cfg.TemplatesDirectory
|
||||
|
||||
var finalizationErr error
|
||||
|
||||
if err := reconcileTemplateOwnership(dir, writtenOutputs); err != nil {
|
||||
finalizationErr = errors.Join(finalizationErr, fmt.Errorf("reconcile template ownership: %w", err))
|
||||
}
|
||||
|
||||
checksums, err := t.regenerateTemplateMetadata(cfg)
|
||||
if err != nil {
|
||||
finalizationErr = errors.Join(finalizationErr, fmt.Errorf("regenerate template metadata: %w", err))
|
||||
}
|
||||
|
||||
if finalizationErr != nil {
|
||||
return nil, finalizationErr
|
||||
}
|
||||
|
||||
if err := commitTemplateVersion(cfg, version); err != nil {
|
||||
return nil, errkit.Wrap(err, "failed to update templates version")
|
||||
}
|
||||
|
||||
return checksums, nil
|
||||
}
|
||||
|
||||
func (t *TemplateManager) finalizeTemplateWrite(cfg *config.Config, writtenOutputs map[string]string, version string, writeErr error) (map[string]string, error) {
|
||||
if writeErr != nil {
|
||||
if ownershipErr := recordPartialTemplateOwnership(cfg.TemplatesDirectory, writtenOutputs); ownershipErr != nil {
|
||||
writeErr = errors.Join(writeErr, fmt.Errorf("record ownership for partially written templates: %w", ownershipErr))
|
||||
}
|
||||
|
||||
return nil, writeErr
|
||||
}
|
||||
|
||||
return t.finalizeTemplateRelease(cfg, writtenOutputs, version)
|
||||
}
|
||||
|
||||
func (t *TemplateManager) bootstrapTemplateOwnership(dir, version string, fetchArchive templateArchiveFetcher) error {
|
||||
if _, err := loadTemplateOwnership(dir); err == nil {
|
||||
return nil
|
||||
} else if errors.Is(err, errTemplateOwnershipInvalid) {
|
||||
quarantines, recoveryErr := listTemplateOwnershipQuarantines(dir)
|
||||
if recoveryErr != nil {
|
||||
return errors.Join(err, recoveryErr)
|
||||
}
|
||||
|
||||
restoreStates, recoveryErr := listTemplateOwnershipRestoreStates(dir)
|
||||
if recoveryErr != nil {
|
||||
return errors.Join(err, recoveryErr)
|
||||
}
|
||||
|
||||
if len(quarantines) > 0 || len(restoreStates) > 0 {
|
||||
return err
|
||||
}
|
||||
|
||||
gologger.Verbose().Msgf("Replacing invalid template ownership metadata without retiring prior templates: %s", err)
|
||||
|
||||
return nil
|
||||
} else if !errors.Is(err, errTemplateOwnershipMissing) {
|
||||
return err
|
||||
}
|
||||
|
||||
version = strings.TrimSpace(version)
|
||||
if version == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
archive, err := fetchArchive(version)
|
||||
if err != nil {
|
||||
gologger.Verbose().Msgf("Continuing template update without prior ownership for %q: %s", version, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := t.bootstrapTemplateOwnershipFromArchive(dir, archive); err != nil {
|
||||
if errors.Is(err, errTemplateOwnershipArchiveInvalid) {
|
||||
gologger.Verbose().Msgf("Continuing template update without prior ownership for %q: %s", version, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("build ownership from prior template release %q: %w", version, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchTemplateReleaseArchive(version string) (*bytes.Reader, error) {
|
||||
ctx := context.Background()
|
||||
httpClient := &http.Client{Timeout: updateutils.DownloadUpdateTimeout}
|
||||
client := github.NewClient(httpClient)
|
||||
archiveURL, _, err := client.Repositories.GetArchiveLink(
|
||||
ctx,
|
||||
updateutils.Organization,
|
||||
config.OfficialNucleiTemplatesRepoName,
|
||||
github.Zipball,
|
||||
&github.RepositoryContentGetOptions{Ref: version},
|
||||
true,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve prior template release %q archive: %w", version, err)
|
||||
}
|
||||
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, archiveURL.String(), nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create prior template release %q archive request: %w", version, err)
|
||||
}
|
||||
|
||||
var contents bytes.Buffer
|
||||
response, err := client.Do(ctx, request, &contents)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("download prior template release %q: %w", version, err)
|
||||
}
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("download prior template release %q: unexpected HTTP status %s", version, response.Status)
|
||||
}
|
||||
|
||||
return bytes.NewReader(contents.Bytes()), nil
|
||||
}
|
||||
|
||||
func (t *TemplateManager) bootstrapTemplateOwnershipFromArchive(dir string, archive *bytes.Reader) error {
|
||||
absDir, err := filepath.Abs(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get absolute templates directory: %w", err)
|
||||
}
|
||||
|
||||
manifest := &templateOwnershipManifest{
|
||||
Version: templateOwnershipVersion,
|
||||
Files: make(map[string]string),
|
||||
}
|
||||
|
||||
callback := func(uri string, fileInfo fs.FileInfo, reader io.Reader) error {
|
||||
if fileInfo.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
_, writePath := t.getTemplateOutputLocation(absDir, uri, fileInfo)
|
||||
if writePath == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
if !config.IsTemplate(writePath) {
|
||||
return nil
|
||||
}
|
||||
|
||||
relativePath, err := filepath.Rel(absDir, writePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("make prior template path %q relative to %q: %w", writePath, absDir, err)
|
||||
}
|
||||
|
||||
relativePath = filepath.ToSlash(relativePath)
|
||||
if !config.IsTemplate(relativePath) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := validateTemplateOwnershipPath(relativePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
contents, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read prior template %q: %w", uri, err)
|
||||
}
|
||||
|
||||
manifest.Files[relativePath] = templateDigest(contents)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := updateutils.UnpackAssetWithCallback(updateutils.Zip, archive, callback); err != nil {
|
||||
return fmt.Errorf("%w: unpack prior template release: %v", errTemplateOwnershipArchiveInvalid, err)
|
||||
}
|
||||
|
||||
return writeTemplateOwnership(absDir, manifest)
|
||||
}
|
||||
|
||||
// updateTemplatesAt updates templates at given directory
|
||||
func (t *TemplateManager) updateTemplatesAt(dir string) error {
|
||||
if t.DisablePublicTemplates {
|
||||
gologger.Info().Msgf("Skipping update of public nuclei-templates")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := t.bootstrapTemplateOwnership(dir, config.DefaultConfig.TemplateVersion, fetchTemplateReleaseArchive); err != nil {
|
||||
return errkit.Wrapf(err, "failed to migrate template ownership at %s", dir)
|
||||
}
|
||||
|
||||
// firstly, read checksums from .checksum file these are used to generate stats
|
||||
oldchecksums, err := t.getChecksumFromDir(dir)
|
||||
if err != nil {
|
||||
@@ -184,49 +387,15 @@ func (t *TemplateManager) updateTemplatesAt(dir string) error {
|
||||
}
|
||||
|
||||
// write templates to disk
|
||||
writtenPaths, err := t.writeTemplatesToDisk(ghrd, dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// cleanup orphaned templates that exist locally but weren't in the new release
|
||||
if err := t.cleanupOrphanedTemplates(dir, writtenPaths); err != nil {
|
||||
// log warning but don't fail the update
|
||||
gologger.Warning().Msgf("failed to cleanup orphaned templates: %s", err)
|
||||
} else {
|
||||
// Regenerate metadata (index and checksum) after successful cleanup to ensure
|
||||
// metadata accurately reflects the cleaned template tree. This prevents stale
|
||||
// index entries and checksum entries for deleted templates.
|
||||
if err := t.regenerateTemplateMetadata(dir); err != nil {
|
||||
// Log warning but don't fail the update - metadata will be out of sync
|
||||
// but templates are cleaned up correctly
|
||||
gologger.Warning().Msgf("failed to regenerate template metadata after cleanup: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// get checksums from new templates
|
||||
newchecksums, err := t.getChecksumFromDir(dir)
|
||||
if err != nil {
|
||||
// unlikely this case will happen
|
||||
return errkit.Wrapf(err, "failed to get checksums from %s after update", dir)
|
||||
writtenOutputs, writeErr := t.writeTemplatesToDisk(ghrd, dir)
|
||||
newchecksums, finalizeErr := t.finalizeTemplateWrite(config.DefaultConfig, writtenOutputs, latestVersion, writeErr)
|
||||
if finalizeErr != nil {
|
||||
return errkit.Wrapf(finalizeErr, "failed to finalize template update at %s", dir)
|
||||
}
|
||||
|
||||
// summarize all changes
|
||||
results := t.summarizeChanges(oldchecksums, newchecksums)
|
||||
|
||||
// remove deleted templates
|
||||
for _, deletion := range results.deletions {
|
||||
// the deletion list comes from the on-disk .checksum file; only remove
|
||||
// paths inside the templates directory.
|
||||
if !filepathutil.IsPathWithinDirectory(deletion, dir) {
|
||||
gologger.Warning().Msgf("skipping deletion of %s: path is outside templates directory %s", deletion, dir)
|
||||
continue
|
||||
}
|
||||
if err := os.Remove(deletion); err != nil && !os.IsNotExist(err) {
|
||||
gologger.Warning().Msgf("failed to remove deleted template %s: %s", deletion, err)
|
||||
}
|
||||
}
|
||||
|
||||
// print summary
|
||||
if results.totalCount > 0 {
|
||||
gologger.Info().Msgf("Successfully updated nuclei-templates (%v) to %s. GoodLuck!", ghrd.Latest.GetTagName(), dir)
|
||||
@@ -238,6 +407,7 @@ func (t *TemplateManager) updateTemplatesAt(dir string) error {
|
||||
} else {
|
||||
gologger.Info().Msgf("Successfully updated nuclei-templates (%v) to %s. GoodLuck!", ghrd.Latest.GetTagName(), dir)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -253,26 +423,30 @@ func (t *TemplateManager) summarizeChanges(old, new map[string]string) *template
|
||||
results.additions = append(results.additions, k)
|
||||
}
|
||||
}
|
||||
|
||||
for k := range old {
|
||||
if _, ok := new[k]; !ok {
|
||||
results.deletions = append(results.deletions, k)
|
||||
}
|
||||
}
|
||||
|
||||
results.totalCount = len(results.additions) + len(results.deletions) + len(results.modifications)
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// getAbsoluteFilePath returns an absolute path where a file should be written based on given uri(i.e., files in zip)
|
||||
// if a returned path is empty, it means that file should not be written and skipped
|
||||
func (t *TemplateManager) getAbsoluteFilePath(templateDir, uri string, f fs.FileInfo) string {
|
||||
// getTemplateOutputLocation returns the safe root and absolute output path for an archive entry.
|
||||
// An empty output path means the entry should be skipped.
|
||||
func (t *TemplateManager) getTemplateOutputLocation(templateDir, uri string, f fs.FileInfo) (string, string) {
|
||||
// overwrite .nuclei-ignore every time nuclei-templates are downloaded
|
||||
if f.Name() == config.NucleiIgnoreFileName {
|
||||
return config.DefaultConfig.GetActiveIgnoreFilePath()
|
||||
return config.DefaultConfig.TemplatesDirectory, config.DefaultConfig.GetActiveIgnoreFilePath()
|
||||
}
|
||||
|
||||
// skip all meta files
|
||||
if !strings.EqualFold(f.Name(), config.NewTemplateAdditionsFileName) {
|
||||
if strings.TrimSpace(f.Name()) == "" || strings.HasPrefix(f.Name(), ".") || strings.EqualFold(f.Name(), "README.md") {
|
||||
return ""
|
||||
return "", ""
|
||||
}
|
||||
}
|
||||
|
||||
@@ -290,9 +464,9 @@ func (t *TemplateManager) getAbsoluteFilePath(templateDir, uri string, f fs.File
|
||||
// to outside the configured templates directory.
|
||||
fallbackPath := filepath.Clean(filepath.Join(templateDir, uri))
|
||||
if !filepathutil.IsPathWithinDirectory(fallbackPath, templateDir) {
|
||||
return ""
|
||||
return "", ""
|
||||
}
|
||||
return fallbackPath
|
||||
return templateDir, fallbackPath
|
||||
}
|
||||
// separator is also included in rootDir
|
||||
rootDirectory := uri[:index+1]
|
||||
@@ -300,47 +474,41 @@ func (t *TemplateManager) getAbsoluteFilePath(templateDir, uri string, f fs.File
|
||||
|
||||
// if it is a github meta directory skip it
|
||||
if stringsutil.HasPrefixAny(relPath, ".github", ".git") {
|
||||
return ""
|
||||
return "", ""
|
||||
}
|
||||
|
||||
newPath := filepath.Clean(filepath.Join(templateDir, relPath))
|
||||
|
||||
if !filepathutil.IsPathWithinDirectory(newPath, templateDir) || !filepathutil.IsPathWithinDirectory(filepath.Dir(newPath), templateDir) {
|
||||
// we don't allow LFI
|
||||
return ""
|
||||
return "", ""
|
||||
}
|
||||
|
||||
if filepath.Clean(newPath) == filepath.Clean(templateDir) {
|
||||
// skip writing the folder itself since it already exists
|
||||
return ""
|
||||
return "", ""
|
||||
}
|
||||
|
||||
if relPath != "" && f.IsDir() {
|
||||
// if uri is a directory, create it
|
||||
if err := fileutil.CreateFolder(newPath); err != nil {
|
||||
gologger.Warning().Msgf("uri %v: got %s while installing templates", uri, err)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return newPath
|
||||
return templateDir, newPath
|
||||
}
|
||||
|
||||
// writeTemplatesToDisk writes all templates to disk and returns a map of written file paths
|
||||
// The returned map contains absolute paths of all template files that were successfully written
|
||||
func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownloader, dir string) (*mapsutil.SyncLockMap[string, struct{}], error) {
|
||||
localTemplatesIndex, err := config.GetNucleiTemplatesIndex()
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("failed to get local nuclei-templates index: %s", err)
|
||||
if localTemplatesIndex == nil {
|
||||
localTemplatesIndex = map[string]string{} // no-op
|
||||
}
|
||||
}
|
||||
func isActiveIgnoreFilePath(writePath string) bool {
|
||||
return filepath.Clean(writePath) == filepath.Clean(config.DefaultConfig.GetActiveIgnoreFilePath())
|
||||
}
|
||||
|
||||
// Track all paths that are successfully written during this update
|
||||
writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]()
|
||||
// writeTemplatesToDisk writes release outputs to disk and returns their digests.
|
||||
// The returned map includes every successfully written output; ownership
|
||||
// reconciliation filters it to official template paths.
|
||||
func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownloader, dir string) (map[string]string, error) {
|
||||
writtenOutputs := make(map[string]string)
|
||||
touchedDirectories := make(map[string]struct{})
|
||||
|
||||
callbackFunc := func(uri string, f fs.FileInfo, r io.Reader) error {
|
||||
writePath := t.getAbsoluteFilePath(dir, uri, f)
|
||||
if f.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
rootDir, writePath := t.getTemplateOutputLocation(dir, uri, f)
|
||||
if writePath == "" {
|
||||
// skip writing file
|
||||
return nil
|
||||
@@ -351,64 +519,33 @@ func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownlo
|
||||
// if error occurs, iteration also stops
|
||||
return errkit.Wrapf(err, "failed to read file %s", uri)
|
||||
}
|
||||
// TODO: It might be better to just download index file from nuclei templates repo
|
||||
// instead of creating it from scratch
|
||||
id, _ := config.GetTemplateIDFromReader(bytes.NewReader(bin), uri)
|
||||
if id != "" {
|
||||
// based on template id, check if we are updating a path of official nuclei template
|
||||
if oldPath, ok := localTemplatesIndex[id]; ok {
|
||||
if oldPath != writePath {
|
||||
// write new template at a new path and delete old template
|
||||
if err := os.WriteFile(writePath, bin, f.Mode()); err != nil {
|
||||
return errkit.Wrapf(err, "failed to write file %s", uri)
|
||||
}
|
||||
// Track the new path as written
|
||||
_ = writtenPaths.Set(writePath, struct{}{})
|
||||
// after successful write, remove old template. oldPath comes
|
||||
// from the on-disk .templates-index; only remove paths inside
|
||||
// the templates directory.
|
||||
if !filepathutil.IsPathWithinDirectory(oldPath, dir) {
|
||||
gologger.Warning().Msgf("skipping removal of old template %s: path is outside templates directory %s", oldPath, dir)
|
||||
} else if err := os.Remove(oldPath); err != nil {
|
||||
gologger.Warning().Msgf("failed to remove old template %s: %s", oldPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
outputResult, outputErr := writeTemplateOutput(rootDir, writePath, bin, f.Mode())
|
||||
for _, directory := range outputResult.touchedDirectories {
|
||||
touchedDirectories[directory] = struct{}{}
|
||||
}
|
||||
// no change in template Path of official templates
|
||||
if err := os.WriteFile(writePath, bin, f.Mode()); err != nil {
|
||||
return errkit.Wrapf(err, "failed to write file %s", uri)
|
||||
|
||||
if outputErr != nil {
|
||||
return errkit.Wrapf(outputErr, "failed to write file %s", uri)
|
||||
}
|
||||
// Track successfully written paths
|
||||
_ = writtenPaths.Set(writePath, struct{}{})
|
||||
|
||||
writtenOutputs[writePath] = templateDigest(bin)
|
||||
|
||||
return nil
|
||||
}
|
||||
err = ghrd.DownloadSourceWithCallback(!HideProgressBar, callbackFunc)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "failed to download templates")
|
||||
|
||||
var writeErr error
|
||||
|
||||
if err := ghrd.DownloadSourceWithCallback(!HideProgressBar, callbackFunc); err != nil {
|
||||
writeErr = errkit.Wrap(err, "failed to download templates")
|
||||
}
|
||||
|
||||
if err := config.DefaultConfig.WriteTemplatesConfig(); err != nil {
|
||||
return nil, errkit.Wrap(err, "failed to write templates config")
|
||||
}
|
||||
// update templates version in config file
|
||||
if err := config.DefaultConfig.SetTemplatesVersion(ghrd.Latest.GetTagName()); err != nil {
|
||||
return nil, errkit.Wrap(err, "failed to update templates version")
|
||||
if err := syncTemplateOutputDirectories(touchedDirectories, syncTemplateOwnershipDirectory); err != nil {
|
||||
writeErr = errors.Join(writeErr, errkit.Wrap(err, "failed to sync template output directories"))
|
||||
}
|
||||
|
||||
PurgeEmptyDirectories(dir)
|
||||
|
||||
// generate index of all templates
|
||||
_ = os.Remove(config.DefaultConfig.GetTemplateIndexFilePath())
|
||||
|
||||
index, err := config.GetNucleiTemplatesIndex()
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "failed to get nuclei templates index")
|
||||
}
|
||||
|
||||
if err = config.DefaultConfig.WriteTemplatesIndex(index); err != nil {
|
||||
return nil, errkit.Wrap(err, "failed to write nuclei templates index")
|
||||
if writeErr != nil {
|
||||
return writtenOutputs, writeErr
|
||||
}
|
||||
|
||||
if !HideReleaseNotes {
|
||||
@@ -418,148 +555,162 @@ func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownlo
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("markdown rendering not supported: %v", err)
|
||||
}
|
||||
|
||||
if rendered, err := r.Render(output); err == nil {
|
||||
output = rendered
|
||||
} else {
|
||||
gologger.Error().Msg(err.Error())
|
||||
}
|
||||
|
||||
gologger.Print().Msgf("\n%v\n\n", output)
|
||||
}
|
||||
|
||||
// after installation, create and write checksums to .checksum file
|
||||
if err := t.writeChecksumFileInDir(dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return writtenPaths, nil
|
||||
return writtenOutputs, nil
|
||||
}
|
||||
|
||||
// cleanupOrphanedTemplates removes template files that exist locally but were not part of the new release
|
||||
// It scans the templates directory for template files and deletes those that are not in the writtenPaths set
|
||||
// This function handles empty directories gracefully - if the directory is empty, no orphaned files will be found
|
||||
func (t *TemplateManager) cleanupOrphanedTemplates(dir string, writtenPaths *mapsutil.SyncLockMap[string, struct{}]) error {
|
||||
absDir, err := filepath.Abs(dir)
|
||||
if err != nil {
|
||||
return errkit.Wrapf(err, "failed to get absolute path of templates directory")
|
||||
}
|
||||
// Use Clean to normalize the path consistently (handles Windows paths better)
|
||||
absDir = filepath.Clean(absDir)
|
||||
func syncTemplateOutputDirectories(directories map[string]struct{}, syncDirectory func(*os.Root, string) error) error {
|
||||
var syncErrors error
|
||||
|
||||
// If directory doesn't exist, there's nothing to clean up
|
||||
if !fileutil.FolderExists(absDir) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Normalize all written paths to absolute paths for comparison
|
||||
normalizedWrittenPaths := mapsutil.NewSyncLockMap[string, struct{}]()
|
||||
for path := range writtenPaths.GetAll() {
|
||||
absPath, err := filepath.Abs(path)
|
||||
if err == nil {
|
||||
// Use Clean to normalize the path consistently (handles Windows paths better)
|
||||
absPath = filepath.Clean(absPath)
|
||||
_ = normalizedWrittenPaths.Set(absPath, struct{}{})
|
||||
}
|
||||
}
|
||||
|
||||
// Get custom template directories to exclude
|
||||
customDirs := config.DefaultConfig.GetAllCustomTemplateDirs()
|
||||
customDirAbs := make([]string, 0, len(customDirs))
|
||||
for _, customDir := range customDirs {
|
||||
if absCustomDir, err := filepath.Abs(customDir); err == nil {
|
||||
// Use Clean to normalize the path consistently (handles Windows paths better)
|
||||
absCustomDir = filepath.Clean(absCustomDir)
|
||||
customDirAbs = append(customDirAbs, absCustomDir)
|
||||
}
|
||||
}
|
||||
|
||||
var orphanedFiles []string
|
||||
|
||||
// Walk the templates directory to find all template files
|
||||
err = filepath.WalkDir(absDir, func(path string, d fs.DirEntry, err error) error {
|
||||
for directory := range directories {
|
||||
root, err := os.OpenRoot(directory)
|
||||
if err != nil {
|
||||
// Log but continue walking
|
||||
gologger.Debug().Msgf("error accessing path %s during orphan cleanup: %s", path, err)
|
||||
return nil
|
||||
syncErrors = errors.Join(syncErrors, fmt.Errorf("open output directory %q: %w", directory, err))
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip directories
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
syncErr := syncDirectory(root, ".")
|
||||
closeErr := root.Close()
|
||||
if syncErr != nil {
|
||||
syncErrors = errors.Join(syncErrors, fmt.Errorf("sync output directory %q: %w", directory, syncErr))
|
||||
}
|
||||
|
||||
absPath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
// Use Clean to normalize the path consistently (handles Windows paths better)
|
||||
absPath = filepath.Clean(absPath)
|
||||
|
||||
// Skip custom template directories
|
||||
if filepathutil.IsPathWithinAnyDirectory(absPath, customDirAbs...) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Only process template files
|
||||
if !config.IsTemplate(absPath) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip if this file was written in the new release
|
||||
if normalizedWrittenPaths.Has(absPath) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// This is an orphaned template file
|
||||
orphanedFiles = append(orphanedFiles, absPath)
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return errkit.Wrapf(err, "failed to walk templates directory for orphan cleanup")
|
||||
}
|
||||
|
||||
// Delete orphaned files
|
||||
for _, orphanPath := range orphanedFiles {
|
||||
if err := os.Remove(orphanPath); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
gologger.Warning().Msgf("failed to remove orphaned template %s: %s", orphanPath, err)
|
||||
}
|
||||
} else {
|
||||
gologger.Debug().Msgf("removed orphaned template: %s", orphanPath)
|
||||
if closeErr != nil {
|
||||
syncErrors = errors.Join(syncErrors, fmt.Errorf("close output directory %q: %w", directory, closeErr))
|
||||
}
|
||||
}
|
||||
|
||||
if len(orphanedFiles) > 0 {
|
||||
gologger.Info().Msgf("cleaned up %d orphaned template file(s)", len(orphanedFiles))
|
||||
}
|
||||
|
||||
return nil
|
||||
return syncErrors
|
||||
}
|
||||
|
||||
// regenerateTemplateMetadata regenerates template index and checksum files after cleanup operations.
|
||||
// This ensures the metadata accurately reflects the current state of template files on disk.
|
||||
func (t *TemplateManager) regenerateTemplateMetadata(dir string) error {
|
||||
// Purge empty directories that may have been left after cleanup
|
||||
// templateOutputWriteResult records filesystem state that must be finalized
|
||||
// even when the associated write returns an error.
|
||||
type templateOutputWriteResult struct {
|
||||
touchedDirectories []string
|
||||
}
|
||||
|
||||
func writeTemplateOutput(rootDir, writePath string, contents []byte, mode fs.FileMode) (templateOutputWriteResult, error) {
|
||||
var result templateOutputWriteResult
|
||||
if isActiveIgnoreFilePath(writePath) {
|
||||
if err := config.DefaultConfig.WriteActiveIgnoreFile(contents); err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.touchedDirectories = append(result.touchedDirectories, filepath.Dir(writePath))
|
||||
return result, nil
|
||||
}
|
||||
|
||||
relativePath, err := filepath.Rel(rootDir, writePath)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
if relativePath == ".." || strings.HasPrefix(relativePath, ".."+string(os.PathSeparator)) || filepath.IsAbs(relativePath) {
|
||||
return result, fmt.Errorf("output path %q escapes root %q", writePath, rootDir)
|
||||
}
|
||||
|
||||
root, err := os.OpenRoot(rootDir)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("open output root %q: %w", rootDir, err)
|
||||
}
|
||||
defer func() { _ = root.Close() }()
|
||||
|
||||
parent := filepath.Dir(relativePath)
|
||||
if parent != "." {
|
||||
parentsToSync, err := createTemplateDirectories(root, parent)
|
||||
for _, parentToSync := range parentsToSync {
|
||||
result.touchedDirectories = append(result.touchedDirectories, filepath.Clean(filepath.Join(rootDir, parentToSync)))
|
||||
}
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("create output parent %q: %w", parent, err)
|
||||
}
|
||||
}
|
||||
result.touchedDirectories = append(result.touchedDirectories, filepath.Dir(writePath))
|
||||
|
||||
randomSuffix := make([]byte, 16)
|
||||
if _, err := rand.Read(randomSuffix); err != nil {
|
||||
return result, fmt.Errorf("generate temporary output name: %w", err)
|
||||
}
|
||||
|
||||
temporaryPath := filepath.Join(parent, fmt.Sprintf("%s%x", templateOutputTemporaryPrefix, randomSuffix))
|
||||
temporaryMode := mode.Perm()
|
||||
preserveMode := false
|
||||
|
||||
if info, err := root.Lstat(relativePath); err == nil && info.Mode().IsRegular() {
|
||||
temporaryMode = 0o600
|
||||
mode = info.Mode()
|
||||
preserveMode = true
|
||||
} else if err != nil && !os.IsNotExist(err) {
|
||||
return result, fmt.Errorf("inspect existing output %q: %w", relativePath, err)
|
||||
}
|
||||
|
||||
temporary, err := root.OpenFile(temporaryPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, temporaryMode)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("create temporary output %q: %w", temporaryPath, err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = temporary.Close()
|
||||
_ = root.Remove(temporaryPath)
|
||||
}()
|
||||
|
||||
if _, err := temporary.Write(contents); err != nil {
|
||||
return result, fmt.Errorf("write temporary output %q: %w", temporaryPath, err)
|
||||
}
|
||||
|
||||
if preserveMode {
|
||||
if err := temporary.Chmod(mode.Perm()); err != nil {
|
||||
return result, fmt.Errorf("preserve output mode %q: %w", temporaryPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A release contains thousands of files, so syncing every temporary file
|
||||
// makes installation latency scale with storage flush latency. Closing and
|
||||
// renaming keeps replacement atomic; touched directories are synced once
|
||||
// after extraction and before ownership or version finalization.
|
||||
if err := temporary.Close(); err != nil {
|
||||
return result, fmt.Errorf("close temporary output %q: %w", temporaryPath, err)
|
||||
}
|
||||
|
||||
if err := root.Rename(temporaryPath, relativePath); err != nil {
|
||||
return result, fmt.Errorf("replace output %q: %w", relativePath, err)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// regenerateTemplateMetadata rebuilds the index and checksums from the finalized on-disk tree.
|
||||
func (t *TemplateManager) regenerateTemplateMetadata(cfg *config.Config) (map[string]string, error) {
|
||||
dir := cfg.TemplatesDirectory
|
||||
|
||||
// Purge empty directories before rebuilding metadata.
|
||||
PurgeEmptyDirectories(dir)
|
||||
|
||||
// Ensure templates directory exists (it may have been purged if empty)
|
||||
// Ensure the templates directory exists if the finalized tree is empty.
|
||||
if !fileutil.FolderExists(dir) {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return errkit.Wrapf(err, "failed to recreate templates directory %s after purge", dir)
|
||||
return nil, errkit.Wrapf(err, "failed to recreate templates directory %s after purge", dir)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove old index file and regenerate it from current templates on disk
|
||||
indexFilePath := config.DefaultConfig.GetTemplateIndexFilePath()
|
||||
indexFilePath := cfg.GetTemplateIndexFilePath()
|
||||
if err := os.Remove(indexFilePath); err != nil && !os.IsNotExist(err) {
|
||||
return errkit.Wrapf(err, "failed to remove old index file %s", indexFilePath)
|
||||
return nil, errkit.Wrapf(err, "failed to remove old index file %s", indexFilePath)
|
||||
}
|
||||
|
||||
// Force regeneration by ensuring the file doesn't exist (handles Windows file handle issues)
|
||||
// GetNucleiTemplatesIndex will scan the directory if the file doesn't exist
|
||||
index, err := config.GetNucleiTemplatesIndex()
|
||||
if err != nil {
|
||||
return errkit.Wrap(err, "failed to regenerate nuclei templates index after cleanup")
|
||||
return nil, errkit.Wrap(err, "failed to regenerate nuclei templates index")
|
||||
}
|
||||
|
||||
// Filter out any entries that don't actually exist on disk (Windows file deletion timing issues)
|
||||
@@ -570,16 +721,20 @@ func (t *TemplateManager) regenerateTemplateMetadata(dir string) error {
|
||||
}
|
||||
}
|
||||
|
||||
if err = config.DefaultConfig.WriteTemplatesIndex(filteredIndex); err != nil {
|
||||
return errkit.Wrap(err, "failed to write nuclei templates index after cleanup")
|
||||
if err = cfg.WriteTemplatesIndex(filteredIndex); err != nil {
|
||||
return nil, errkit.Wrap(err, "failed to write regenerated nuclei templates index")
|
||||
}
|
||||
|
||||
// Regenerate checksum file to reflect current templates on disk
|
||||
if err := t.writeChecksumFileInDir(dir); err != nil {
|
||||
return errkit.Wrap(err, "failed to regenerate checksum file after cleanup")
|
||||
checksumMap, err := t.calculateChecksumMap(dir)
|
||||
if err != nil {
|
||||
return nil, errkit.Wrap(err, "failed to regenerate checksum map")
|
||||
}
|
||||
|
||||
return nil
|
||||
if err := writeChecksumMap(cfg, checksumMap); err != nil {
|
||||
return nil, errkit.Wrap(err, "failed to write regenerated checksum file")
|
||||
}
|
||||
|
||||
return checksumMap, nil
|
||||
}
|
||||
|
||||
// getChecksumFromDir returns a map containing checksums (md5 hash) of all yaml files (with .yaml extension)
|
||||
@@ -598,23 +753,31 @@ func (t *TemplateManager) getChecksumFromDir(dir string) (map[string]string, err
|
||||
if len(tmparr) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
allChecksums[tmparr[0]] = tmparr[1]
|
||||
}
|
||||
|
||||
return allChecksums, nil
|
||||
}
|
||||
}
|
||||
|
||||
return t.calculateChecksumMap(dir)
|
||||
}
|
||||
|
||||
// writeChecksumFileInDir creates checksums of all yaml files in given directory
|
||||
// and writes them to a file named .checksum
|
||||
// writeChecksumFileInDir creates checksums of all YAML files in dir and writes
|
||||
// them to the configured checksum file.
|
||||
func (t *TemplateManager) writeChecksumFileInDir(dir string) error {
|
||||
checksumMap, err := t.calculateChecksumMap(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return writeChecksumMap(config.DefaultConfig, checksumMap)
|
||||
}
|
||||
|
||||
func writeChecksumMap(cfg *config.Config, checksumMap map[string]string) error {
|
||||
var buff bytes.Buffer
|
||||
|
||||
for k, v := range checksumMap {
|
||||
buff.WriteString(k)
|
||||
buff.WriteString(",")
|
||||
@@ -622,7 +785,19 @@ func (t *TemplateManager) writeChecksumFileInDir(dir string) error {
|
||||
buff.WriteString(";")
|
||||
}
|
||||
|
||||
return os.WriteFile(config.DefaultConfig.GetChecksumFilePath(), buff.Bytes(), checkSumFilePerm)
|
||||
return os.WriteFile(cfg.GetChecksumFilePath(), buff.Bytes(), checkSumFilePerm)
|
||||
}
|
||||
|
||||
func isTemplateOutputTemporary(name string) bool {
|
||||
suffix, found := strings.CutPrefix(name, templateOutputTemporaryPrefix)
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
if isLowerHex(suffix, 32) {
|
||||
return true
|
||||
}
|
||||
pathDigest, token, found := strings.Cut(suffix, "-")
|
||||
return found && isLowerHex(pathDigest, 32) && isLowerHex(token, 32)
|
||||
}
|
||||
|
||||
// getChecksumMap returns a map containing checksums (md5 hash) of all yaml files (with .yaml extension)
|
||||
@@ -638,6 +813,7 @@ func (t *TemplateManager) calculateChecksumMap(dir string) (map[string]string, e
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%x", md5.Sum(bin)), nil
|
||||
}
|
||||
|
||||
@@ -645,6 +821,18 @@ func (t *TemplateManager) calculateChecksumMap(dir string) (map[string]string, e
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !d.IsDir() && isTemplateOutputTemporary(filepath.Base(path)) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if filepath.Dir(path) == filepath.Clean(dir) {
|
||||
name := filepath.Base(path)
|
||||
if name == templateOwnershipFileName || strings.HasPrefix(name, templateOwnershipTemporaryPrefix) || strings.HasPrefix(name, templateOwnershipRetiredPrefix) || strings.HasPrefix(name, templateOwnershipRestorePrefix) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// skip checksums of custom templates i.e github and s3
|
||||
if filepathutil.IsPathWithinAnyDirectory(path, config.DefaultConfig.GetAllCustomTemplateDirs()...) {
|
||||
return nil
|
||||
@@ -656,9 +844,12 @@ func (t *TemplateManager) calculateChecksumMap(dir string) (map[string]string, e
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
checksumMap[path] = checksum
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return checksumMap, errkit.Wrap(err, "failed to calculate checksums of templates")
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
//go:build !windows
|
||||
|
||||
package installer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestWriteTemplateOutputHonorsUmask(t *testing.T) {
|
||||
previousUmask := syscall.Umask(0o077)
|
||||
t.Cleanup(func() { syscall.Umask(previousUmask) })
|
||||
|
||||
outputPath := filepath.Join(t.TempDir(), "official.yaml")
|
||||
_, err := writeTemplateOutput(filepath.Dir(outputPath), outputPath, []byte("release"), 0o666)
|
||||
require.NoError(t, err)
|
||||
info, err := os.Stat(outputPath)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, os.FileMode(0o600), info.Mode().Perm())
|
||||
}
|
||||
|
||||
func TestCopyQuarantinedTemplatePreservesMode(t *testing.T) {
|
||||
templatesDir := t.TempDir()
|
||||
quarantinePath := templateOwnershipRetiredPrefix + "mode"
|
||||
require.NoError(t, os.WriteFile(filepath.Join(templatesDir, quarantinePath), []byte("modified"), 0o644))
|
||||
|
||||
previousUmask := syscall.Umask(0o077)
|
||||
t.Cleanup(func() { syscall.Umask(previousUmask) })
|
||||
|
||||
root, err := os.OpenRoot(templatesDir)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, root.Close()) })
|
||||
|
||||
require.NoError(t, copyQuarantinedTemplate(root, "retired.yaml", quarantinePath))
|
||||
info, err := root.Stat("retired.yaml")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, os.FileMode(0o644), info.Mode().Perm())
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,42 @@
|
||||
//go:build darwin
|
||||
|
||||
package installer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func renameTemplateRestoreNoReplace(root *os.Root, temporaryPath, retiredPath string) error {
|
||||
sourceDirectory, err := root.Open(filepath.Dir(temporaryPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = sourceDirectory.Close() }()
|
||||
|
||||
destinationDirectory, err := root.Open(filepath.Dir(retiredPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = destinationDirectory.Close() }()
|
||||
|
||||
err = unix.RenameatxNp(int(sourceDirectory.Fd()), filepath.Base(temporaryPath), int(destinationDirectory.Fd()), filepath.Base(retiredPath), unix.RENAME_EXCL)
|
||||
return mapRenameNoReplaceError(err)
|
||||
}
|
||||
|
||||
func mapRenameNoReplaceError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch {
|
||||
case errors.Is(err, unix.ENOTSUP), errors.Is(err, unix.ENOSYS), errors.Is(err, unix.EINVAL):
|
||||
return fmt.Errorf("%w: %w", errors.ErrUnsupported, err)
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
//go:build linux
|
||||
|
||||
package installer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func renameTemplateRestoreNoReplace(root *os.Root, temporaryPath, retiredPath string) error {
|
||||
sourceDirectory, err := root.Open(filepath.Dir(temporaryPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = sourceDirectory.Close() }()
|
||||
|
||||
destinationDirectory, err := root.Open(filepath.Dir(retiredPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = destinationDirectory.Close() }()
|
||||
|
||||
err = unix.Renameat2(int(sourceDirectory.Fd()), filepath.Base(temporaryPath), int(destinationDirectory.Fd()), filepath.Base(retiredPath), unix.RENAME_NOREPLACE)
|
||||
return mapRenameNoReplaceError(err)
|
||||
}
|
||||
|
||||
func mapRenameNoReplaceError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch {
|
||||
case errors.Is(err, unix.ENOSYS), errors.Is(err, unix.EINVAL), errors.Is(err, unix.EOPNOTSUPP):
|
||||
return fmt.Errorf("%w: %w", errors.ErrUnsupported, err)
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
//go:build linux
|
||||
|
||||
package installer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCopyQuarantinedTemplateWithoutHardLinks(t *testing.T) {
|
||||
root, err := os.OpenRoot(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, root.Close()) })
|
||||
|
||||
quarantinePath := templateOwnershipRetiredPrefix + "no-hard-links"
|
||||
contents := []byte("locally modified contents")
|
||||
require.NoError(t, root.WriteFile(quarantinePath, contents, 0o640))
|
||||
openQuarantine, err := root.OpenFile(quarantinePath, os.O_RDWR, 0)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, openQuarantine.Close()) })
|
||||
|
||||
err = restoreQuarantinedTemplateWithLink(root, "retired.yaml", quarantinePath, func(string, string) error {
|
||||
return errors.ErrUnsupported
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, openQuarantine.Truncate(0))
|
||||
_, err = openQuarantine.WriteAt([]byte("late local write"), 0)
|
||||
require.NoError(t, err)
|
||||
restored, err := root.ReadFile("retired.yaml")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []byte("late local write"), restored)
|
||||
info, err := root.Stat("retired.yaml")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, os.FileMode(0o640), info.Mode().Perm())
|
||||
}
|
||||
|
||||
func TestResumeTemplateRestoreStateWithoutHardLinks(t *testing.T) {
|
||||
root, err := os.OpenRoot(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, root.Close()) })
|
||||
|
||||
retiredPath := "retired.yaml"
|
||||
quarantinePath := templateOwnershipRetiredPrefix + "pre-publication"
|
||||
contents := []byte("locally modified contents")
|
||||
require.NoError(t, root.WriteFile(quarantinePath, contents, 0o600))
|
||||
state, err := newTemplateRestoreState(retiredPath)
|
||||
require.NoError(t, err)
|
||||
state.kind = templateRestoreCopy
|
||||
state.digest = templateDigest(contents)
|
||||
require.NoError(t, root.WriteFile(state.temporaryPath, contents, 0o600))
|
||||
require.NoError(t, createTemplateRestoreState(root, state))
|
||||
|
||||
completed, err := resumeTemplateRestoreState(root, state, quarantinePath, func(string, string) error {
|
||||
return errors.ErrUnsupported
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.False(t, completed)
|
||||
require.NoError(t, restoreQuarantinedTemplate(root, retiredPath, quarantinePath))
|
||||
restored, err := root.ReadFile(retiredPath)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, contents, restored)
|
||||
_, err = root.Lstat(state.statePath)
|
||||
require.ErrorIs(t, err, os.ErrNotExist)
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
//go:build !linux && !darwin && !windows
|
||||
|
||||
package installer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
)
|
||||
|
||||
func renameTemplateRestoreNoReplace(_ *os.Root, _, _ string) error {
|
||||
return errors.ErrUnsupported
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
//go:build windows
|
||||
|
||||
package installer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"unsafe"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
type templateRestoreRenameInformation struct {
|
||||
replaceIfExists uint32
|
||||
rootDirectory windows.Handle
|
||||
fileNameLength uint32
|
||||
fileName [1]uint16
|
||||
}
|
||||
|
||||
func renameTemplateRestoreNoReplace(root *os.Root, temporaryPath, retiredPath string) error {
|
||||
sourceDirectory, err := root.Open(filepath.Dir(temporaryPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = sourceDirectory.Close() }()
|
||||
|
||||
destinationDirectory, err := root.Open(filepath.Dir(retiredPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = destinationDirectory.Close() }()
|
||||
|
||||
objectName, err := windows.NewNTUnicodeString(filepath.Base(temporaryPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
attributes := &windows.OBJECT_ATTRIBUTES{
|
||||
Length: uint32(unsafe.Sizeof(windows.OBJECT_ATTRIBUTES{})),
|
||||
RootDirectory: windows.Handle(sourceDirectory.Fd()),
|
||||
ObjectName: objectName,
|
||||
Attributes: windows.OBJ_CASE_INSENSITIVE,
|
||||
}
|
||||
|
||||
var source windows.Handle
|
||||
var status windows.IO_STATUS_BLOCK
|
||||
|
||||
if err := windows.NtCreateFile(
|
||||
&source,
|
||||
windows.SYNCHRONIZE|windows.DELETE,
|
||||
attributes,
|
||||
&status,
|
||||
nil,
|
||||
0,
|
||||
windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE,
|
||||
windows.FILE_OPEN,
|
||||
windows.FILE_OPEN_REPARSE_POINT|windows.FILE_OPEN_FOR_BACKUP_INTENT|windows.FILE_SYNCHRONOUS_IO_NONALERT,
|
||||
0,
|
||||
0,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
defer windows.CloseHandle(source)
|
||||
|
||||
targetName, err := windows.UTF16FromString(filepath.Base(retiredPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
targetNameLength := (len(targetName) - 1) * 2
|
||||
|
||||
var header templateRestoreRenameInformation
|
||||
|
||||
buffer := make([]byte, int(unsafe.Offsetof(header.fileName))+targetNameLength)
|
||||
info := (*templateRestoreRenameInformation)(unsafe.Pointer(&buffer[0]))
|
||||
info.rootDirectory = windows.Handle(destinationDirectory.Fd())
|
||||
info.fileNameLength = uint32(targetNameLength)
|
||||
copy((*[windows.MAX_LONG_PATH]uint16)(unsafe.Pointer(&info.fileName[0]))[:targetNameLength/2:targetNameLength/2], targetName)
|
||||
|
||||
return windows.NtSetInformationFile(source, &status, &buffer[0], uint32(len(buffer)), windows.FileRenameInformation)
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
//go:build !windows
|
||||
|
||||
package installer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLoadTemplateRestoreStateRejectsFIFO(t *testing.T) {
|
||||
templatesDir := t.TempDir()
|
||||
root, err := os.OpenRoot(templatesDir)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, root.Close()) })
|
||||
|
||||
state := templateRestoreState{
|
||||
retiredPath: "retired.yaml",
|
||||
statePath: templateRestoreStatePrefix("retired.yaml") + strings.Repeat("a", 32),
|
||||
}
|
||||
require.NoError(t, syscall.Mkfifo(filepath.Join(templatesDir, state.statePath), 0o600))
|
||||
require.ErrorContains(t, loadTemplateRestoreState(root, &state), "is not a regular file")
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
//go:build !windows
|
||||
|
||||
package installer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
)
|
||||
|
||||
func syncTemplateOwnershipFile(root *os.Root, relativePath string, mode fs.FileMode) error {
|
||||
file, err := root.Open(relativePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := file.Chmod(mode.Perm()); err != nil {
|
||||
_ = file.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
return errors.Join(file.Sync(), file.Close())
|
||||
}
|
||||
|
||||
func syncTemplateOwnershipDirectory(root *os.Root, relativePath string) error {
|
||||
directory, err := root.Open(relativePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = directory.Close() }()
|
||||
|
||||
return directory.Sync()
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
)
|
||||
|
||||
func syncTemplateOwnershipFile(root *os.Root, relativePath string, mode fs.FileMode) error {
|
||||
return syncTemplateOwnershipFileWithOpen(root, relativePath, mode, func() (*os.File, error) {
|
||||
return root.OpenFile(relativePath, os.O_RDWR, 0)
|
||||
})
|
||||
}
|
||||
|
||||
func syncTemplateOwnershipFileWithOpen(root *os.Root, relativePath string, mode fs.FileMode, open func() (*os.File, error)) error {
|
||||
if err := root.Chmod(relativePath, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := open()
|
||||
if err != nil {
|
||||
return errors.Join(err, root.Chmod(relativePath, mode.Perm()))
|
||||
}
|
||||
|
||||
if err := file.Chmod(mode.Perm()); err != nil {
|
||||
return errors.Join(err, file.Close(), root.Chmod(relativePath, mode.Perm()))
|
||||
}
|
||||
|
||||
return errors.Join(file.Sync(), file.Close())
|
||||
}
|
||||
|
||||
// Windows cannot flush the read-only directory handle exposed by os.Root.
|
||||
func syncTemplateOwnershipDirectory(_ *os.Root, _ string) error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSyncTemplateOwnershipDirectory(t *testing.T) {
|
||||
root, err := os.OpenRoot(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, root.Close()) })
|
||||
require.NoError(t, syncTemplateOwnershipDirectory(root, "."))
|
||||
}
|
||||
|
||||
func TestSyncTemplateOwnershipFileRestoresModeAfterOpenFailure(t *testing.T) {
|
||||
root, err := os.OpenRoot(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, root.Close()) })
|
||||
require.NoError(t, root.WriteFile("restored.yaml", []byte("restored"), 0o444))
|
||||
originalInfo, err := root.Stat("restored.yaml")
|
||||
require.NoError(t, err)
|
||||
openErr := errors.New("open failed")
|
||||
|
||||
err = syncTemplateOwnershipFileWithOpen(root, "restored.yaml", originalInfo.Mode(), func() (*os.File, error) {
|
||||
return nil, openErr
|
||||
})
|
||||
require.ErrorIs(t, err, openErr)
|
||||
info, err := root.Stat("restored.yaml")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, originalInfo.Mode().Perm(), info.Mode().Perm())
|
||||
}
|
||||
|
||||
func TestRenameTemplateRestoreNoReplaceUsesOpenedRoot(t *testing.T) {
|
||||
parent := t.TempDir()
|
||||
openedParent := filepath.Join(parent, "opened")
|
||||
currentParent := filepath.Join(parent, "current")
|
||||
rootPath := filepath.Join(openedParent, "templates")
|
||||
currentRootPath := filepath.Join(currentParent, "templates")
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(rootPath, "source"), 0o755))
|
||||
require.NoError(t, os.Mkdir(filepath.Join(rootPath, "destination"), 0o755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(currentRootPath, "source"), 0o755))
|
||||
require.NoError(t, os.Mkdir(filepath.Join(currentRootPath, "destination"), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(currentRootPath, "source", "temporary.yaml"), []byte("wrong root"), 0o600))
|
||||
|
||||
t.Chdir(openedParent)
|
||||
root, err := os.OpenRoot("templates")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, root.Close()) })
|
||||
require.NoError(t, root.WriteFile(filepath.Join("source", "temporary.yaml"), []byte("restored"), 0o600))
|
||||
|
||||
t.Chdir(currentParent)
|
||||
require.NoError(t, renameTemplateRestoreNoReplace(root, filepath.Join("source", "temporary.yaml"), filepath.Join("destination", "retired.yaml")))
|
||||
require.FileExists(t, filepath.Join(rootPath, "destination", "retired.yaml"))
|
||||
require.NoFileExists(t, filepath.Join(currentRootPath, "destination", "retired.yaml"))
|
||||
|
||||
require.NoError(t, os.Rename(filepath.Join(rootPath, "source"), filepath.Join(rootPath, "moved-source")))
|
||||
require.NoError(t, os.Rename(filepath.Join(rootPath, "destination"), filepath.Join(rootPath, "moved-destination")))
|
||||
}
|
||||
|
||||
func TestSyncTemplateOwnershipFile(t *testing.T) {
|
||||
root, err := os.OpenRoot(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, root.Close()) })
|
||||
require.NoError(t, root.WriteFile("restored.yaml", []byte("restored"), 0o444))
|
||||
require.NoError(t, syncTemplateOwnershipFile(root, "restored.yaml", 0o444))
|
||||
info, err := root.Stat("restored.yaml")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, os.FileMode(0o444), info.Mode().Perm())
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
+347
-390
@@ -1,14 +1,16 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
mapsutil "github.com/projectdiscovery/utils/maps"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -59,6 +61,9 @@ func TestTemplateInstallation(t *testing.T) {
|
||||
require.Greater(t, counter, 1000)
|
||||
// every time we install templates, it should override the ignore file with latest one
|
||||
require.FileExists(t, config.DefaultConfig.GetActiveIgnoreFilePath())
|
||||
ownership, err := loadTemplateOwnership(templatesTempDir)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, ownership.Files, "fresh installation should record official template ownership")
|
||||
t.Logf("Installed %d templates", counter)
|
||||
}
|
||||
|
||||
@@ -101,277 +106,262 @@ func TestIsOutdatedVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupOrphanedTemplates(t *testing.T) {
|
||||
HideProgressBar = true
|
||||
func TestWriteTemplateOutput(t *testing.T) {
|
||||
t.Run("replaces a final symlink without modifying its target", func(t *testing.T) {
|
||||
templatesDir := t.TempDir()
|
||||
externalPath := filepath.Join(t.TempDir(), "external.yaml")
|
||||
require.NoError(t, os.WriteFile(externalPath, []byte("external"), 0o644))
|
||||
|
||||
outputPath := filepath.Join(templatesDir, "official.yaml")
|
||||
if err := os.Symlink(externalPath, outputPath); err != nil {
|
||||
t.Skipf("symlinks are unavailable: %v", err)
|
||||
}
|
||||
|
||||
_, err := writeTemplateOutput(templatesDir, outputPath, []byte("release"), 0o644)
|
||||
require.NoError(t, err)
|
||||
require.FileExists(t, outputPath)
|
||||
info, err := os.Lstat(outputPath)
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, info.Mode()&os.ModeSymlink)
|
||||
contents, err := os.ReadFile(outputPath)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "release", string(contents))
|
||||
require.FileExists(t, externalPath)
|
||||
contents, err = os.ReadFile(externalPath)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "external", string(contents))
|
||||
})
|
||||
|
||||
t.Run("rejects a parent symlink outside the templates directory", func(t *testing.T) {
|
||||
templatesDir := t.TempDir()
|
||||
externalDir := t.TempDir()
|
||||
linkedParent := filepath.Join(templatesDir, "nested")
|
||||
if err := os.Symlink(externalDir, linkedParent); err != nil {
|
||||
t.Skipf("symlinks are unavailable: %v", err)
|
||||
}
|
||||
|
||||
_, err := writeTemplateOutput(templatesDir, filepath.Join(linkedParent, "official.yaml"), []byte("release"), 0o644)
|
||||
require.Error(t, err)
|
||||
require.NoFileExists(t, filepath.Join(externalDir, "official.yaml"))
|
||||
})
|
||||
|
||||
t.Run("preserves permissions on an existing regular file", func(t *testing.T) {
|
||||
templatesDir := t.TempDir()
|
||||
outputPath := filepath.Join(templatesDir, "official.yaml")
|
||||
require.NoError(t, os.WriteFile(outputPath, []byte("previous"), 0o600))
|
||||
previousInfo, err := os.Stat(outputPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = writeTemplateOutput(templatesDir, outputPath, []byte("release"), 0o777)
|
||||
require.NoError(t, err)
|
||||
info, err := os.Stat(outputPath)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, previousInfo.Mode().Perm(), info.Mode().Perm())
|
||||
})
|
||||
|
||||
t.Run("reports every directory touched by nested output creation", func(t *testing.T) {
|
||||
templatesDir := t.TempDir()
|
||||
firstParent := filepath.Join(templatesDir, "first")
|
||||
outputParent := filepath.Join(firstParent, "second")
|
||||
outputPath := filepath.Join(outputParent, "official.yaml")
|
||||
|
||||
result, err := writeTemplateOutput(templatesDir, outputPath, []byte("release"), 0o644)
|
||||
require.NoError(t, err)
|
||||
require.ElementsMatch(t, []string{templatesDir, firstParent, outputParent}, result.touchedDirectories)
|
||||
})
|
||||
|
||||
t.Run("reports the parent touched by a failed replacement", func(t *testing.T) {
|
||||
templatesDir := t.TempDir()
|
||||
outputPath := filepath.Join(templatesDir, "existing-directory")
|
||||
require.NoError(t, os.Mkdir(outputPath, 0o755))
|
||||
|
||||
result, err := writeTemplateOutput(templatesDir, outputPath, []byte("release"), 0o644)
|
||||
require.Error(t, err)
|
||||
require.Equal(t, []string{templatesDir}, result.touchedDirectories)
|
||||
})
|
||||
}
|
||||
|
||||
func TestSyncTemplateOutputDirectories(t *testing.T) {
|
||||
t.Run("syncs each distinct directory once", func(t *testing.T) {
|
||||
directories := map[string]struct{}{
|
||||
t.TempDir(): {},
|
||||
t.TempDir(): {},
|
||||
}
|
||||
syncCalls := 0
|
||||
|
||||
err := syncTemplateOutputDirectories(directories, func(*os.Root, string) error {
|
||||
syncCalls++
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, len(directories), syncCalls)
|
||||
})
|
||||
|
||||
t.Run("continues after a sync failure", func(t *testing.T) {
|
||||
directories := map[string]struct{}{
|
||||
t.TempDir(): {},
|
||||
t.TempDir(): {},
|
||||
}
|
||||
syncErr := errors.New("sync failed")
|
||||
syncCalls := 0
|
||||
|
||||
err := syncTemplateOutputDirectories(directories, func(*os.Root, string) error {
|
||||
syncCalls++
|
||||
if syncCalls == 1 {
|
||||
return syncErr
|
||||
}
|
||||
return nil
|
||||
})
|
||||
require.ErrorIs(t, err, syncErr)
|
||||
require.Equal(t, len(directories), syncCalls)
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkWriteTemplateOutputs(b *testing.B) {
|
||||
b.Run("batched", func(b *testing.B) {
|
||||
benchmarkWriteTemplateOutputs(b, false)
|
||||
})
|
||||
b.Run("per-output-sync", func(b *testing.B) {
|
||||
benchmarkWriteTemplateOutputs(b, true)
|
||||
})
|
||||
}
|
||||
|
||||
func benchmarkWriteTemplateOutputs(b *testing.B, syncEachOutput bool) {
|
||||
rootDir := b.TempDir()
|
||||
contents := []byte("id: benchmark\ninfo:\n name: Benchmark\n author: test\n severity: info\n")
|
||||
touchedDirectories := make(map[string]struct{})
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
|
||||
for index := 0; index < b.N; index++ {
|
||||
writePath := filepath.Join(rootDir, fmt.Sprintf("group-%d", index%16), fmt.Sprintf("template-%d.yaml", index))
|
||||
outputResult, err := writeTemplateOutput(rootDir, writePath, contents, 0o644)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
for _, directory := range outputResult.touchedDirectories {
|
||||
touchedDirectories[directory] = struct{}{}
|
||||
}
|
||||
|
||||
if syncEachOutput {
|
||||
output, err := os.OpenFile(writePath, os.O_RDWR, 0)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
syncErr := output.Sync()
|
||||
closeErr := output.Close()
|
||||
if err := errors.Join(syncErr, closeErr); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := syncTemplateOutputDirectories(touchedDirectories, syncTemplateOwnershipDirectory); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetTemplateOutputLocationRoutesIgnoreFile(t *testing.T) {
|
||||
previousConfig := config.DefaultConfig
|
||||
templatesDir := t.TempDir()
|
||||
cfg := &config.Config{}
|
||||
cfg.SetTemplatesDir(templatesDir)
|
||||
config.DefaultConfig = cfg
|
||||
t.Cleanup(func() { config.DefaultConfig = previousConfig })
|
||||
|
||||
ignoreEntry := filepath.Join(t.TempDir(), config.NucleiIgnoreFileName)
|
||||
require.NoError(t, os.WriteFile(ignoreEntry, nil, 0o600))
|
||||
info, err := os.Stat(ignoreEntry)
|
||||
require.NoError(t, err)
|
||||
rootDir, writePath := (&TemplateManager{}).getTemplateOutputLocation(templatesDir, "nuclei-templates/"+config.NucleiIgnoreFileName, info)
|
||||
require.Equal(t, templatesDir, rootDir)
|
||||
require.Equal(t, cfg.GetActiveIgnoreFilePath(), writePath)
|
||||
}
|
||||
|
||||
func TestWriteTemplateOutputValidatesIgnoreFile(t *testing.T) {
|
||||
previousConfig := config.DefaultConfig
|
||||
templatesDir := t.TempDir()
|
||||
cfg := &config.Config{}
|
||||
cfg.SetTemplatesDir(templatesDir)
|
||||
config.DefaultConfig = cfg
|
||||
t.Cleanup(func() { config.DefaultConfig = previousConfig })
|
||||
|
||||
path := cfg.GetActiveIgnoreFilePath()
|
||||
valid := []byte("tags: [weak]\n")
|
||||
require.NoError(t, os.WriteFile(path, valid, 0o600))
|
||||
|
||||
_, err := writeTemplateOutput(templatesDir, path, []byte("<html>blocked</html>\n"), 0o644)
|
||||
require.Error(t, err)
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, valid, got)
|
||||
|
||||
payload := []byte("files: [blocked.yaml]\n")
|
||||
_, err = writeTemplateOutput(templatesDir, path, payload, 0o644)
|
||||
require.NoError(t, err)
|
||||
got, err = os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, payload, got)
|
||||
}
|
||||
|
||||
func TestCommitTemplateVersionRestoresMemoryOnWriteFailure(t *testing.T) {
|
||||
stateDir := filepath.Join(t.TempDir(), "state")
|
||||
require.NoError(t, os.WriteFile(stateDir, nil, 0o600))
|
||||
cfg := &config.Config{TemplateVersion: "v1.2.3"}
|
||||
cfg.SetStateDir(stateDir)
|
||||
|
||||
err := commitTemplateVersion(cfg, "v2.0.0")
|
||||
require.Error(t, err)
|
||||
require.Equal(t, "v1.2.3", cfg.TemplateVersion)
|
||||
}
|
||||
|
||||
func TestFinalizeTemplateReleaseCommitsVersionLast(t *testing.T) {
|
||||
newConfig := func(t *testing.T) *config.Config {
|
||||
t.Helper()
|
||||
configDir := t.TempDir()
|
||||
templatesDir := t.TempDir()
|
||||
cfg := &config.Config{Logger: gologger.DefaultLogger}
|
||||
cfg.SetConfigDir(configDir)
|
||||
cfg.SetTemplatesDir(templatesDir)
|
||||
require.NoError(t, os.WriteFile(cfg.GetActiveIgnoreFilePath(), []byte("ignore contents"), 0o600))
|
||||
cfg.TemplateVersion = "v1.0.0"
|
||||
|
||||
previousConfig := config.DefaultConfig
|
||||
config.DefaultConfig = cfg
|
||||
t.Cleanup(func() { config.DefaultConfig = previousConfig })
|
||||
return cfg
|
||||
}
|
||||
|
||||
t.Run("commits after successful finalization", func(t *testing.T) {
|
||||
cfg := newConfig(t)
|
||||
templatePath := filepath.Join(cfg.TemplatesDirectory, "official.yaml")
|
||||
require.NoError(t, os.WriteFile(templatePath, []byte("id: official\ninfo:\n name: Official\n author: test\n severity: info\n"), 0644))
|
||||
|
||||
tm := &TemplateManager{}
|
||||
_, err := tm.finalizeTemplateRelease(cfg, newWrittenTemplates(t, templatePath), "v2.0.0")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "v2.0.0", cfg.TemplateVersion)
|
||||
})
|
||||
|
||||
t.Run("preserves version when metadata finalization fails", func(t *testing.T) {
|
||||
cfg := newConfig(t)
|
||||
templatePath := filepath.Join(cfg.TemplatesDirectory, "official.yaml")
|
||||
require.NoError(t, os.WriteFile(templatePath, []byte("id: official\ninfo:\n name: Official\n author: test\n severity: info\n"), 0644))
|
||||
require.NoError(t, os.Mkdir(cfg.GetTemplateIndexFilePath(), 0755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(cfg.GetTemplateIndexFilePath(), "keep"), []byte("keep"), 0600))
|
||||
|
||||
tm := &TemplateManager{}
|
||||
_, err := tm.finalizeTemplateRelease(cfg, newWrittenTemplates(t, templatePath), "v2.0.0")
|
||||
require.Error(t, err)
|
||||
require.Equal(t, "v1.0.0", cfg.TemplateVersion)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCalculateChecksumMap(t *testing.T) {
|
||||
tm := &TemplateManager{}
|
||||
|
||||
t.Run("removes orphaned templates", func(t *testing.T) {
|
||||
// Create temporary directories
|
||||
tmpDir, err := os.MkdirTemp("", "nuclei-cleanup-test-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
|
||||
cfgdir, err := os.MkdirTemp("", "nuclei-config-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(cfgdir)
|
||||
}()
|
||||
|
||||
config.DefaultConfig.SetConfigDir(cfgdir)
|
||||
config.DefaultConfig.SetTemplatesDir(tmpDir)
|
||||
|
||||
// Create subdirectories for templates
|
||||
templatesDir1 := filepath.Join(tmpDir, "cves", "2023")
|
||||
templatesDir2 := filepath.Join(tmpDir, "exposures", "configs")
|
||||
require.NoError(t, os.MkdirAll(templatesDir1, 0755))
|
||||
require.NoError(t, os.MkdirAll(templatesDir2, 0755))
|
||||
|
||||
// Create template files
|
||||
template1 := filepath.Join(templatesDir1, "CVE-2023-1234.yaml")
|
||||
template2 := filepath.Join(templatesDir1, "CVE-2023-5678.yaml")
|
||||
template3 := filepath.Join(templatesDir2, "git-config-exposure.yaml")
|
||||
orphanedTemplate1 := filepath.Join(templatesDir1, "old-template.yaml")
|
||||
orphanedTemplate2 := filepath.Join(templatesDir2, "removed-template.yaml")
|
||||
|
||||
// Write valid template files
|
||||
templateContent := `id: test-template
|
||||
info:
|
||||
name: Test Template
|
||||
author: test
|
||||
severity: info`
|
||||
require.NoError(t, os.WriteFile(template1, []byte(templateContent), 0644))
|
||||
require.NoError(t, os.WriteFile(template2, []byte(templateContent), 0644))
|
||||
require.NoError(t, os.WriteFile(template3, []byte(templateContent), 0644))
|
||||
require.NoError(t, os.WriteFile(orphanedTemplate1, []byte(templateContent), 0644))
|
||||
require.NoError(t, os.WriteFile(orphanedTemplate2, []byte(templateContent), 0644))
|
||||
|
||||
// Simulate written paths from new release (only template1, template2, template3)
|
||||
writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]()
|
||||
absTemplate1, _ := filepath.Abs(template1)
|
||||
absTemplate2, _ := filepath.Abs(template2)
|
||||
absTemplate3, _ := filepath.Abs(template3)
|
||||
// Normalize paths consistently (same as cleanupOrphanedTemplates does)
|
||||
absTemplate1 = filepath.Clean(absTemplate1)
|
||||
absTemplate2 = filepath.Clean(absTemplate2)
|
||||
absTemplate3 = filepath.Clean(absTemplate3)
|
||||
_ = writtenPaths.Set(absTemplate1, struct{}{})
|
||||
_ = writtenPaths.Set(absTemplate2, struct{}{})
|
||||
_ = writtenPaths.Set(absTemplate3, struct{}{})
|
||||
|
||||
// Run cleanup
|
||||
err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify orphaned templates were removed
|
||||
require.NoFileExists(t, orphanedTemplate1, "orphaned template should be removed")
|
||||
require.NoFileExists(t, orphanedTemplate2, "orphaned template should be removed")
|
||||
|
||||
// Verify non-orphaned templates still exist
|
||||
require.FileExists(t, template1, "template from new release should exist")
|
||||
require.FileExists(t, template2, "template from new release should exist")
|
||||
require.FileExists(t, template3, "template from new release should exist")
|
||||
})
|
||||
|
||||
t.Run("preserves custom templates", func(t *testing.T) {
|
||||
// Create temporary directories
|
||||
tmpDir, err := os.MkdirTemp("", "nuclei-cleanup-custom-test-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
|
||||
cfgdir, err := os.MkdirTemp("", "nuclei-config-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(cfgdir)
|
||||
}()
|
||||
|
||||
config.DefaultConfig.SetConfigDir(cfgdir)
|
||||
config.DefaultConfig.SetTemplatesDir(tmpDir)
|
||||
|
||||
// Create custom template directory
|
||||
customGitHubDir := filepath.Join(tmpDir, "github", "owner", "repo")
|
||||
require.NoError(t, os.MkdirAll(customGitHubDir, 0755))
|
||||
|
||||
// Create custom template file
|
||||
customTemplate := filepath.Join(customGitHubDir, "custom-template.yaml")
|
||||
templateContent := `id: custom-template
|
||||
info:
|
||||
name: Custom Template
|
||||
author: test
|
||||
severity: info`
|
||||
require.NoError(t, os.WriteFile(customTemplate, []byte(templateContent), 0644))
|
||||
|
||||
// Empty written paths (simulating no custom templates in new release)
|
||||
writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]()
|
||||
|
||||
// Run cleanup
|
||||
err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify custom template was NOT removed
|
||||
require.FileExists(t, customTemplate, "custom template should be preserved")
|
||||
})
|
||||
|
||||
t.Run("removes orphaned templates from custom dir sibling prefixes", func(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "nuclei-cleanup-custom-prefix-test-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
|
||||
cfgdir, err := os.MkdirTemp("", "nuclei-config-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(cfgdir)
|
||||
}()
|
||||
|
||||
config.DefaultConfig.SetConfigDir(cfgdir)
|
||||
config.DefaultConfig.SetTemplatesDir(tmpDir)
|
||||
|
||||
customGitHubDir := filepath.Join(tmpDir, "github", "owner", "repo")
|
||||
require.NoError(t, os.MkdirAll(customGitHubDir, 0755))
|
||||
customTemplate := filepath.Join(customGitHubDir, "custom-template.yaml")
|
||||
require.NoError(t, os.WriteFile(customTemplate, []byte(`id: custom-template
|
||||
info:
|
||||
name: Custom Template
|
||||
author: test
|
||||
severity: info`), 0644))
|
||||
|
||||
siblingDir := filepath.Join(tmpDir, "github-evil")
|
||||
require.NoError(t, os.MkdirAll(siblingDir, 0755))
|
||||
siblingTemplate := filepath.Join(siblingDir, "orphaned-template.yaml")
|
||||
require.NoError(t, os.WriteFile(siblingTemplate, []byte(`id: orphaned-template
|
||||
info:
|
||||
name: Orphaned Template
|
||||
author: test
|
||||
severity: info`), 0644))
|
||||
|
||||
writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]()
|
||||
|
||||
err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.FileExists(t, customTemplate, "custom template should be preserved")
|
||||
require.NoFileExists(t, siblingTemplate, "custom directory sibling prefix should not be preserved")
|
||||
})
|
||||
|
||||
t.Run("skips non-template files", func(t *testing.T) {
|
||||
// Create temporary directories
|
||||
tmpDir, err := os.MkdirTemp("", "nuclei-cleanup-nontemplate-test-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
|
||||
cfgdir, err := os.MkdirTemp("", "nuclei-config-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(cfgdir)
|
||||
}()
|
||||
|
||||
config.DefaultConfig.SetConfigDir(cfgdir)
|
||||
config.DefaultConfig.SetTemplatesDir(tmpDir)
|
||||
|
||||
// Create non-template files
|
||||
readmeFile := filepath.Join(tmpDir, "README.md")
|
||||
configFile := filepath.Join(tmpDir, "cves.json")
|
||||
checksumFile := filepath.Join(tmpDir, ".checksum")
|
||||
|
||||
require.NoError(t, os.WriteFile(readmeFile, []byte("# Templates"), 0644))
|
||||
require.NoError(t, os.WriteFile(configFile, []byte("{}"), 0644))
|
||||
require.NoError(t, os.WriteFile(checksumFile, []byte(""), 0644))
|
||||
|
||||
// Empty written paths
|
||||
writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]()
|
||||
|
||||
// Run cleanup
|
||||
err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify non-template files were NOT removed
|
||||
require.FileExists(t, readmeFile, "README.md should be preserved")
|
||||
require.FileExists(t, configFile, "config file should be preserved")
|
||||
require.FileExists(t, checksumFile, "checksum file should be preserved")
|
||||
})
|
||||
|
||||
t.Run("handles empty written paths", func(t *testing.T) {
|
||||
// Create temporary directories
|
||||
tmpDir, err := os.MkdirTemp("", "nuclei-cleanup-empty-test-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
|
||||
cfgdir, err := os.MkdirTemp("", "nuclei-config-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(cfgdir)
|
||||
}()
|
||||
|
||||
config.DefaultConfig.SetConfigDir(cfgdir)
|
||||
config.DefaultConfig.SetTemplatesDir(tmpDir)
|
||||
|
||||
// Create template files
|
||||
template1 := filepath.Join(tmpDir, "template1.yaml")
|
||||
templateContent := `id: test-template
|
||||
info:
|
||||
name: Test Template
|
||||
author: test
|
||||
severity: info`
|
||||
require.NoError(t, os.WriteFile(template1, []byte(templateContent), 0644))
|
||||
|
||||
// Empty written paths
|
||||
writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]()
|
||||
|
||||
// Run cleanup
|
||||
err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify template was removed (since it's not in written paths)
|
||||
require.NoFileExists(t, template1, "template should be removed when not in written paths")
|
||||
})
|
||||
|
||||
t.Run("handles relative and absolute paths correctly", func(t *testing.T) {
|
||||
// Create temporary directories
|
||||
tmpDir, err := os.MkdirTemp("", "nuclei-cleanup-path-test-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
|
||||
cfgdir, err := os.MkdirTemp("", "nuclei-config-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(cfgdir)
|
||||
}()
|
||||
|
||||
config.DefaultConfig.SetConfigDir(cfgdir)
|
||||
config.DefaultConfig.SetTemplatesDir(tmpDir)
|
||||
|
||||
// Create template file
|
||||
template1 := filepath.Join(tmpDir, "template1.yaml")
|
||||
templateContent := `id: test-template
|
||||
info:
|
||||
name: Test Template
|
||||
author: test
|
||||
severity: info`
|
||||
require.NoError(t, os.WriteFile(template1, []byte(templateContent), 0644))
|
||||
|
||||
// Use relative path in written paths
|
||||
writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]()
|
||||
_ = writtenPaths.Set(template1, struct{}{}) // relative path
|
||||
|
||||
// Run cleanup - should normalize paths correctly
|
||||
err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify template was NOT removed (it was in written paths)
|
||||
require.FileExists(t, template1, "template should be preserved when in written paths")
|
||||
})
|
||||
previousTemplatesDir := config.DefaultConfig.TemplatesDirectory
|
||||
t.Cleanup(func() { config.DefaultConfig.SetTemplatesDir(previousTemplatesDir) })
|
||||
|
||||
t.Run("checksums custom dir sibling prefixes", func(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "nuclei-checksum-custom-prefix-test-*")
|
||||
@@ -380,13 +370,6 @@ info:
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
|
||||
cfgdir, err := os.MkdirTemp("", "nuclei-config-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(cfgdir)
|
||||
}()
|
||||
|
||||
config.DefaultConfig.SetConfigDir(cfgdir)
|
||||
config.DefaultConfig.SetTemplatesDir(tmpDir)
|
||||
|
||||
customGitHubDir := filepath.Join(tmpDir, "github")
|
||||
@@ -413,50 +396,39 @@ info:
|
||||
require.Contains(t, checksums, siblingTemplate, "custom directory sibling prefix should be checksummed")
|
||||
})
|
||||
|
||||
t.Run("handles empty templates directory", func(t *testing.T) {
|
||||
// Create temporary directories
|
||||
tmpDir, err := os.MkdirTemp("", "nuclei-cleanup-empty-dir-test-*")
|
||||
t.Run("excludes internal temporary files from checksums", func(t *testing.T) {
|
||||
templatesDir := t.TempDir()
|
||||
config.DefaultConfig.SetTemplatesDir(templatesDir)
|
||||
|
||||
templateFile := filepath.Join(templatesDir, "template.yaml")
|
||||
ownershipTemporary := filepath.Join(templatesDir, templateOwnershipFileName+".tmp-interrupted")
|
||||
retiredQuarantine := filepath.Join(templatesDir, templateOwnershipRetiredPrefix+"interrupted")
|
||||
nestedDir := filepath.Join(templatesDir, "nested")
|
||||
nestedOwnershipTemporary := filepath.Join(nestedDir, templateOwnershipFileName+".tmp-user")
|
||||
outputTemporary := filepath.Join(nestedDir, templateOutputTemporaryPrefix+strings.Repeat("a", 32))
|
||||
restoreTemporary := filepath.Join(nestedDir, templateOutputTemporaryPrefix+strings.Repeat("b", 32)+"-"+strings.Repeat("c", 32))
|
||||
restoreState := filepath.Join(templatesDir, templateOwnershipRestorePrefix+strings.Repeat("d", 64)+"-"+strings.Repeat("e", 32))
|
||||
userFileWithSimilarName := filepath.Join(nestedDir, templateOutputTemporaryPrefix+"user.yaml")
|
||||
require.NoError(t, os.Mkdir(nestedDir, 0755))
|
||||
require.NoError(t, os.WriteFile(templateFile, []byte("template"), 0644))
|
||||
require.NoError(t, os.WriteFile(ownershipTemporary, []byte("partial manifest"), 0600))
|
||||
require.NoError(t, os.WriteFile(retiredQuarantine, []byte("retired template"), 0600))
|
||||
require.NoError(t, os.WriteFile(nestedOwnershipTemporary, []byte("user file"), 0644))
|
||||
require.NoError(t, os.WriteFile(outputTemporary, []byte("interrupted write"), 0600))
|
||||
require.NoError(t, os.WriteFile(restoreTemporary, []byte("interrupted restore"), 0600))
|
||||
require.NoError(t, os.WriteFile(restoreState, nil, 0600))
|
||||
require.NoError(t, os.WriteFile(userFileWithSimilarName, []byte("user file"), 0644))
|
||||
|
||||
checksums, err := tm.calculateChecksumMap(templatesDir)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
|
||||
cfgdir, err := os.MkdirTemp("", "nuclei-config-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(cfgdir)
|
||||
}()
|
||||
|
||||
config.DefaultConfig.SetConfigDir(cfgdir)
|
||||
config.DefaultConfig.SetTemplatesDir(tmpDir)
|
||||
|
||||
// Directory exists but is empty (user deleted all templates)
|
||||
require.True(t, fileutil.FolderExists(tmpDir), "templates directory should exist")
|
||||
|
||||
// Written paths from new release
|
||||
writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]()
|
||||
|
||||
// Run cleanup - should handle empty directory gracefully
|
||||
err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths)
|
||||
require.NoError(t, err, "cleanup should handle empty directory without error")
|
||||
|
||||
// Directory should still exist after cleanup
|
||||
require.True(t, fileutil.FolderExists(tmpDir), "templates directory should still exist")
|
||||
})
|
||||
|
||||
t.Run("handles non-existent directory gracefully", func(t *testing.T) {
|
||||
// Use a non-existent directory path
|
||||
nonExistentDir := "/tmp/nuclei-test-non-existent-dir-12345"
|
||||
|
||||
// Ensure it doesn't exist
|
||||
_ = os.RemoveAll(nonExistentDir)
|
||||
require.False(t, fileutil.FolderExists(nonExistentDir), "directory should not exist")
|
||||
|
||||
writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]()
|
||||
|
||||
// Run cleanup - should handle non-existent directory gracefully
|
||||
err := tm.cleanupOrphanedTemplates(nonExistentDir, writtenPaths)
|
||||
require.NoError(t, err, "cleanup should handle non-existent directory without error")
|
||||
require.Contains(t, checksums, templateFile)
|
||||
require.NotContains(t, checksums, ownershipTemporary)
|
||||
require.NotContains(t, checksums, retiredQuarantine)
|
||||
require.NotContains(t, checksums, outputTemporary)
|
||||
require.NotContains(t, checksums, restoreTemporary)
|
||||
require.NotContains(t, checksums, restoreState)
|
||||
require.Contains(t, checksums, nestedOwnershipTemporary, "only root-level ownership artifacts are internal metadata")
|
||||
require.Contains(t, checksums, userFileWithSimilarName, "only names matching the complete internal write pattern are excluded")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -500,7 +472,7 @@ info:
|
||||
require.NoError(t, os.WriteFile(template2, []byte(template2Content), 0644))
|
||||
|
||||
// Regenerate metadata
|
||||
err = tm.regenerateTemplateMetadata(tmpDir)
|
||||
_, err = tm.regenerateTemplateMetadata(config.DefaultConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify index file was created
|
||||
@@ -524,7 +496,7 @@ info:
|
||||
require.Contains(t, checksums, template2, "checksum should contain template2")
|
||||
})
|
||||
|
||||
t.Run("excludes deleted templates from index after cleanup", func(t *testing.T) {
|
||||
t.Run("excludes retired templates from the regenerated index", func(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "nuclei-metadata-cleanup-test-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
@@ -543,7 +515,7 @@ info:
|
||||
// Create template files
|
||||
template1 := filepath.Join(tmpDir, "kept-template.yaml")
|
||||
template2 := filepath.Join(tmpDir, "deleted-template.yaml")
|
||||
orphanedTemplate := filepath.Join(tmpDir, "orphaned-template.yaml")
|
||||
retiredTemplate := filepath.Join(tmpDir, "retired-template.yaml")
|
||||
|
||||
template1Content := `id: test-template-1
|
||||
info:
|
||||
@@ -555,21 +527,21 @@ info:
|
||||
name: Test Template 2
|
||||
author: test
|
||||
severity: info`
|
||||
orphanedContent := `id: test-template-orphaned
|
||||
retiredContent := `id: test-template-retired
|
||||
info:
|
||||
name: Test Template Orphaned
|
||||
name: Test Template Retired
|
||||
author: test
|
||||
severity: info`
|
||||
|
||||
require.NoError(t, os.WriteFile(template1, []byte(template1Content), 0644))
|
||||
require.NoError(t, os.WriteFile(template2, []byte(template2Content), 0644))
|
||||
require.NoError(t, os.WriteFile(orphanedTemplate, []byte(orphanedContent), 0644))
|
||||
require.NoError(t, os.WriteFile(retiredTemplate, []byte(retiredContent), 0644))
|
||||
|
||||
// Create initial index with all templates (simulating state before cleanup)
|
||||
// Create initial index with all previously owned templates.
|
||||
initialIndex := map[string]string{
|
||||
"test-template-1": template1,
|
||||
"test-template-2": template2,
|
||||
"test-template-orphaned": orphanedTemplate,
|
||||
"test-template-1": template1,
|
||||
"test-template-2": template2,
|
||||
"test-template-retired": retiredTemplate,
|
||||
}
|
||||
err = config.DefaultConfig.WriteTemplatesIndex(initialIndex)
|
||||
require.NoError(t, err)
|
||||
@@ -577,33 +549,25 @@ info:
|
||||
// Verify initial index contains all templates
|
||||
index, err := config.GetNucleiTemplatesIndex()
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, index, "test-template-orphaned", "initial index should contain orphaned template")
|
||||
require.Contains(t, index, "test-template-retired", "initial index should contain the retired template")
|
||||
|
||||
// Simulate cleanup: remove orphaned template
|
||||
writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]()
|
||||
absTemplate1, _ := filepath.Abs(template1)
|
||||
// Normalize path consistently (same as cleanupOrphanedTemplates does)
|
||||
absTemplate1 = filepath.Clean(absTemplate1)
|
||||
_ = writtenPaths.Set(absTemplate1, struct{}{})
|
||||
// Arrange the post-reconciliation tree directly; reconciliation has dedicated coverage.
|
||||
require.NoError(t, os.Remove(template2))
|
||||
require.NoError(t, os.Remove(retiredTemplate))
|
||||
|
||||
err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths)
|
||||
require.NoError(t, err)
|
||||
require.NoFileExists(t, orphanedTemplate, "orphaned template should be deleted")
|
||||
require.NoFileExists(t, template2, "template2 should be deleted since it's not in writtenPaths")
|
||||
|
||||
// Regenerate metadata after cleanup
|
||||
err = tm.regenerateTemplateMetadata(tmpDir)
|
||||
// Regenerate metadata from the arranged post-reconciliation tree.
|
||||
_, err = tm.regenerateTemplateMetadata(config.DefaultConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify index no longer contains deleted template
|
||||
// The regenerated index excludes templates retired by the release.
|
||||
index, err = config.GetNucleiTemplatesIndex()
|
||||
require.NoError(t, err)
|
||||
require.NotContains(t, index, "test-template-orphaned", "index should not contain deleted orphaned template")
|
||||
require.NotContains(t, index, "test-template-retired", "index should not contain a retired template")
|
||||
require.Contains(t, index, "test-template-1", "index should still contain kept template")
|
||||
require.NotContains(t, index, "test-template-2", "index should not contain template that was deleted but not cleaned")
|
||||
})
|
||||
|
||||
t.Run("excludes deleted templates from checksum after cleanup", func(t *testing.T) {
|
||||
t.Run("excludes retired templates from the regenerated checksum", func(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "nuclei-checksum-cleanup-test-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
@@ -621,7 +585,7 @@ info:
|
||||
|
||||
// Create template files
|
||||
keptTemplate := filepath.Join(tmpDir, "kept.yaml")
|
||||
orphanedTemplate := filepath.Join(tmpDir, "orphaned.yaml")
|
||||
retiredTemplate := filepath.Join(tmpDir, "retired.yaml")
|
||||
|
||||
templateContent := `id: test-template
|
||||
info:
|
||||
@@ -630,40 +594,34 @@ info:
|
||||
severity: info`
|
||||
|
||||
require.NoError(t, os.WriteFile(keptTemplate, []byte(templateContent), 0644))
|
||||
require.NoError(t, os.WriteFile(orphanedTemplate, []byte(templateContent), 0644))
|
||||
require.NoError(t, os.WriteFile(retiredTemplate, []byte(templateContent), 0644))
|
||||
|
||||
// Create initial checksum with both templates
|
||||
err = tm.writeChecksumFileInDir(tmpDir)
|
||||
checksumMap, err := tm.calculateChecksumMap(tmpDir)
|
||||
require.NoError(t, err)
|
||||
err = writeChecksumMap(config.DefaultConfig, checksumMap)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify initial checksum contains both templates
|
||||
initialChecksums, err := tm.getChecksumFromDir(tmpDir)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, initialChecksums, orphanedTemplate, "initial checksum should contain orphaned template")
|
||||
require.Contains(t, initialChecksums, retiredTemplate, "initial checksum should contain the retired template")
|
||||
|
||||
// Simulate cleanup: remove orphaned template
|
||||
writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]()
|
||||
absKept, _ := filepath.Abs(keptTemplate)
|
||||
// Normalize path consistently (same as cleanupOrphanedTemplates does)
|
||||
absKept = filepath.Clean(absKept)
|
||||
_ = writtenPaths.Set(absKept, struct{}{})
|
||||
// Arrange the post-reconciliation tree directly; reconciliation has dedicated coverage.
|
||||
require.NoError(t, os.Remove(retiredTemplate))
|
||||
|
||||
err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths)
|
||||
require.NoError(t, err)
|
||||
require.NoFileExists(t, orphanedTemplate, "orphaned template should be deleted")
|
||||
|
||||
// Regenerate metadata after cleanup
|
||||
err = tm.regenerateTemplateMetadata(tmpDir)
|
||||
// Regenerate metadata from the arranged post-reconciliation tree.
|
||||
_, err = tm.regenerateTemplateMetadata(config.DefaultConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify checksum no longer contains deleted template
|
||||
// The regenerated checksum excludes templates retired by the release.
|
||||
checksums, err := tm.getChecksumFromDir(tmpDir)
|
||||
require.NoError(t, err)
|
||||
require.NotContains(t, checksums, orphanedTemplate, "checksum should not contain deleted orphaned template")
|
||||
require.NotContains(t, checksums, retiredTemplate, "checksum should not contain a retired template")
|
||||
require.Contains(t, checksums, keptTemplate, "checksum should still contain kept template")
|
||||
})
|
||||
|
||||
t.Run("cleanup and metadata regeneration integration", func(t *testing.T) {
|
||||
t.Run("reconciles ownership before metadata regeneration", func(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "nuclei-integration-test-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
@@ -682,12 +640,12 @@ info:
|
||||
// Create multiple templates
|
||||
template1 := filepath.Join(tmpDir, "cves", "2023", "cve1.yaml")
|
||||
template2 := filepath.Join(tmpDir, "cves", "2023", "cve2.yaml")
|
||||
orphaned1 := filepath.Join(tmpDir, "cves", "2022", "old-cve.yaml")
|
||||
orphaned2 := filepath.Join(tmpDir, "exposures", "old-exposure.yaml")
|
||||
retired1 := filepath.Join(tmpDir, "cves", "2022", "old-cve.yaml")
|
||||
retired2 := filepath.Join(tmpDir, "exposures", "old-exposure.yaml")
|
||||
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(template1), 0755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(orphaned1), 0755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(orphaned2), 0755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(retired1), 0755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(retired2), 0755))
|
||||
|
||||
template1Content := `id: cve1
|
||||
info:
|
||||
@@ -699,12 +657,12 @@ info:
|
||||
name: CVE2
|
||||
author: test
|
||||
severity: info`
|
||||
orphaned1Content := `id: old-cve
|
||||
retired1Content := `id: old-cve
|
||||
info:
|
||||
name: Old CVE
|
||||
author: test
|
||||
severity: info`
|
||||
orphaned2Content := `id: old-exposure
|
||||
retired2Content := `id: old-exposure
|
||||
info:
|
||||
name: Old Exposure
|
||||
author: test
|
||||
@@ -712,27 +670,26 @@ info:
|
||||
|
||||
require.NoError(t, os.WriteFile(template1, []byte(template1Content), 0644))
|
||||
require.NoError(t, os.WriteFile(template2, []byte(template2Content), 0644))
|
||||
require.NoError(t, os.WriteFile(orphaned1, []byte(orphaned1Content), 0644))
|
||||
require.NoError(t, os.WriteFile(orphaned2, []byte(orphaned2Content), 0644))
|
||||
require.NoError(t, os.WriteFile(retired1, []byte(retired1Content), 0644))
|
||||
require.NoError(t, os.WriteFile(retired2, []byte(retired2Content), 0644))
|
||||
seedTemplateOwnership(t, tmpDir, template1, template2, retired1, retired2)
|
||||
|
||||
// Simulate written paths from new release
|
||||
writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]()
|
||||
// The current release retains template1 and template2.
|
||||
absTemplate1, _ := filepath.Abs(template1)
|
||||
absTemplate2, _ := filepath.Abs(template2)
|
||||
// Normalize paths consistently (same as cleanupOrphanedTemplates does)
|
||||
// Normalize paths consistently with ownership reconciliation.
|
||||
absTemplate1 = filepath.Clean(absTemplate1)
|
||||
absTemplate2 = filepath.Clean(absTemplate2)
|
||||
_ = writtenPaths.Set(absTemplate1, struct{}{})
|
||||
_ = writtenPaths.Set(absTemplate2, struct{}{})
|
||||
writtenTemplates := newWrittenTemplates(t, absTemplate1, absTemplate2)
|
||||
|
||||
// Perform cleanup
|
||||
err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths)
|
||||
// Reconcile prior ownership with the current release.
|
||||
err = reconcileTemplateOwnership(tmpDir, writtenTemplates)
|
||||
require.NoError(t, err)
|
||||
require.NoFileExists(t, orphaned1, "orphaned template 1 should be deleted")
|
||||
require.NoFileExists(t, orphaned2, "orphaned template 2 should be deleted")
|
||||
require.NoFileExists(t, retired1, "retired template 1 should be deleted")
|
||||
require.NoFileExists(t, retired2, "retired template 2 should be deleted")
|
||||
|
||||
// Regenerate metadata (simulating what updateTemplatesAt does)
|
||||
err = tm.regenerateTemplateMetadata(tmpDir)
|
||||
_, err = tm.regenerateTemplateMetadata(config.DefaultConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify index only contains kept templates
|
||||
@@ -748,12 +705,12 @@ info:
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, checksums, template1, "checksum should contain kept template1")
|
||||
require.Contains(t, checksums, template2, "checksum should contain kept template2")
|
||||
require.NotContains(t, checksums, orphaned1, "checksum should not contain deleted template")
|
||||
require.NotContains(t, checksums, orphaned2, "checksum should not contain deleted template")
|
||||
require.NotContains(t, checksums, retired1, "checksum should not contain deleted template")
|
||||
require.NotContains(t, checksums, retired2, "checksum should not contain deleted template")
|
||||
|
||||
// Verify empty directories are purged
|
||||
require.False(t, fileutil.FolderExists(filepath.Dir(orphaned1)), "empty directory should be purged")
|
||||
require.False(t, fileutil.FolderExists(filepath.Dir(orphaned2)), "empty directory should be purged")
|
||||
require.False(t, fileutil.FolderExists(filepath.Dir(retired1)), "empty directory should be purged")
|
||||
require.False(t, fileutil.FolderExists(filepath.Dir(retired2)), "empty directory should be purged")
|
||||
})
|
||||
|
||||
t.Run("handles empty templates directory", func(t *testing.T) {
|
||||
@@ -776,7 +733,7 @@ info:
|
||||
require.NoError(t, os.MkdirAll(tmpDir, 0755))
|
||||
|
||||
// Regenerate metadata on empty directory
|
||||
err = tm.regenerateTemplateMetadata(tmpDir)
|
||||
_, err = tm.regenerateTemplateMetadata(config.DefaultConfig)
|
||||
require.NoError(t, err, "should handle empty directory without error")
|
||||
|
||||
// Index should exist but be empty or minimal
|
||||
@@ -820,7 +777,7 @@ info:
|
||||
severity: info`), 0644))
|
||||
|
||||
// Regenerate metadata (should purge empty directories)
|
||||
err = tm.regenerateTemplateMetadata(tmpDir)
|
||||
_, err = tm.regenerateTemplateMetadata(config.DefaultConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify empty directories were purged
|
||||
|
||||
@@ -1,13 +1,25 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/projectdiscovery/utils/generic"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return f(req)
|
||||
}
|
||||
|
||||
func TestVersionCheck(t *testing.T) {
|
||||
ignoreHash, err := NucleiVersionCheck()
|
||||
require.Nil(t, err)
|
||||
@@ -17,3 +29,86 @@ func TestVersionCheck(t *testing.T) {
|
||||
t.Errorf("something went wrong got empty response nuclei-version=%v templates-version=%v ignore-hash=%v", cfg.LatestNucleiVersion, cfg.LatestNucleiTemplatesVersion, ignoreHash)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateIgnoreFileRejectsHTMLAndPreservesExisting(t *testing.T) {
|
||||
html := []byte("<!DOCTYPE HTML>\n<html>\n<head>\n<meta name=\"description\" content=\"Zscaler: blocked\">\n</head>\n</html>\n")
|
||||
withIgnoreDownload(t, http.StatusOK, html)
|
||||
|
||||
cfg := config.DefaultConfig
|
||||
root := t.TempDir()
|
||||
previous := cfg.TemplatesDirectory
|
||||
t.Cleanup(func() { cfg.SetTemplatesDir(previous) })
|
||||
cfg.SetTemplatesDir(root)
|
||||
|
||||
valid := []byte("tags: [weak]\nfiles: [blocked.yaml]\n")
|
||||
path := cfg.GetActiveIgnoreFilePath()
|
||||
require.NoError(t, os.WriteFile(path, valid, 0o600))
|
||||
|
||||
err := UpdateIgnoreFile()
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), fmt.Sprintf("%q", path))
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, valid, got)
|
||||
}
|
||||
|
||||
func TestUpdateIgnoreFileWritesValidContents(t *testing.T) {
|
||||
payload := []byte("tags: [dos]\nfiles: [http/test.yaml]\n")
|
||||
withIgnoreDownload(t, http.StatusOK, payload)
|
||||
|
||||
cfg := config.DefaultConfig
|
||||
root := t.TempDir()
|
||||
previous := cfg.TemplatesDirectory
|
||||
t.Cleanup(func() { cfg.SetTemplatesDir(previous) })
|
||||
cfg.SetTemplatesDir(root)
|
||||
|
||||
require.NoError(t, UpdateIgnoreFile())
|
||||
|
||||
got, err := os.ReadFile(cfg.GetActiveIgnoreFilePath())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, payload, got)
|
||||
}
|
||||
|
||||
func TestUpdateIgnoreFileRejectsUnexpectedStatus(t *testing.T) {
|
||||
withIgnoreDownload(t, http.StatusForbidden, []byte("tags: [weak]\n"))
|
||||
|
||||
cfg := config.DefaultConfig
|
||||
root := t.TempDir()
|
||||
previous := cfg.TemplatesDirectory
|
||||
t.Cleanup(func() { cfg.SetTemplatesDir(previous) })
|
||||
cfg.SetTemplatesDir(root)
|
||||
|
||||
err := UpdateIgnoreFile()
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "unexpected HTTP status")
|
||||
_, statErr := os.Stat(cfg.GetActiveIgnoreFilePath())
|
||||
require.ErrorIs(t, statErr, os.ErrNotExist)
|
||||
}
|
||||
|
||||
func withIgnoreDownload(t *testing.T, status int, body []byte) {
|
||||
t.Helper()
|
||||
|
||||
previous := retryableHttpClient
|
||||
t.Cleanup(func() { retryableHttpClient = previous })
|
||||
|
||||
retryableHttpClient = retryablehttp.NewClient(retryablehttp.Options{
|
||||
RetryMax: 0,
|
||||
HttpClient: &http.Client{
|
||||
Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
header := make(http.Header)
|
||||
header.Set("Content-Type", "text/plain")
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Status: http.StatusText(status),
|
||||
Body: io.NopCloser(strings.NewReader(string(body))),
|
||||
Header: header,
|
||||
Request: req,
|
||||
Proto: "HTTP/1.1",
|
||||
ProtoMajor: 1,
|
||||
ProtoMinor: 1,
|
||||
}, nil
|
||||
}),
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -64,19 +64,19 @@ func TestZipSlip(t *testing.T) {
|
||||
|
||||
for _, filePathFromZip := range filePathsFromZip {
|
||||
var tmp fs.FileInfo = &tempFileInfo{name: filePathFromZip}
|
||||
writePath := tm.getAbsoluteFilePath(configuredTemplateDirectory, filePathFromZip, tmp)
|
||||
_, writePath := tm.getTemplateOutputLocation(configuredTemplateDirectory, filePathFromZip, tmp)
|
||||
require.Equal(t, "", writePath, filePathFromZip)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("positive no-slash fallback", func(t *testing.T) {
|
||||
// Entry names with no slash exercise the fallback branch in
|
||||
// getAbsoluteFilePath. Legitimate single-name entries must still be
|
||||
// getTemplateOutputLocation. Legitimate single-name entries must still be
|
||||
// written into templateDir (regression test for the containment check
|
||||
// added to that branch).
|
||||
tm := TemplateManager{}
|
||||
var tmp fs.FileInfo = &tempFileInfo{name: "single-file.yaml"}
|
||||
writePath := tm.getAbsoluteFilePath(configuredTemplateDirectory, "single-file.yaml", tmp)
|
||||
_, writePath := tm.getTemplateOutputLocation(configuredTemplateDirectory, "single-file.yaml", tmp)
|
||||
require.Equal(t, filepath.Join(configuredTemplateDirectory, "single-file.yaml"), writePath)
|
||||
})
|
||||
|
||||
@@ -89,7 +89,7 @@ func TestZipSlip(t *testing.T) {
|
||||
|
||||
for filePathFromZip, expectedWritePath := range filePathsFromZip {
|
||||
var tmp fs.FileInfo = &tempFileInfo{name: filePathFromZip}
|
||||
writePath := tm.getAbsoluteFilePath(configuredTemplateDirectory, filePathFromZip, tmp)
|
||||
_, writePath := tm.getTemplateOutputLocation(configuredTemplateDirectory, filePathFromZip, tmp)
|
||||
require.Equal(t, expectedWritePath, writePath, filePathFromZip)
|
||||
}
|
||||
})
|
||||
@@ -101,7 +101,7 @@ func TestZipSlip(t *testing.T) {
|
||||
|
||||
tm := TemplateManager{}
|
||||
var tmp fs.FileInfo = &tempFileInfo{name: "nuclei-templates/cves/test.yaml"}
|
||||
writePath := tm.getAbsoluteFilePath(aliasDir, "nuclei-templates/cves/test.yaml", tmp)
|
||||
_, writePath := tm.getTemplateOutputLocation(aliasDir, "nuclei-templates/cves/test.yaml", tmp)
|
||||
require.Equal(t, filepath.Join(aliasDir, "cves", "test.yaml"), writePath)
|
||||
})
|
||||
|
||||
@@ -109,7 +109,7 @@ func TestZipSlip(t *testing.T) {
|
||||
// path component is itself a symlink that points outside the configured
|
||||
// templates directory. Because both the entry's leaf and its parent on
|
||||
// disk may be missing, a naive lexical containment check could miss the
|
||||
// escape. canonicalizePath inside IsPathWithinDirectory walks up to the
|
||||
// escape. canonicalizePath inside IsPathWithinDirectory walks up to the
|
||||
// nearest existing ancestor (the symlink) and resolves it, which is what
|
||||
// makes this rejection sound. This test pins that behavior.
|
||||
t.Run("negative symlinked ancestor escapes templateDir", func(t *testing.T) {
|
||||
@@ -117,7 +117,7 @@ func TestZipSlip(t *testing.T) {
|
||||
outsideDir := t.TempDir()
|
||||
// Plant a symlink "evil" inside templateDir that points outside.
|
||||
// A malicious zip entry that traverses through it must be rejected
|
||||
// before it ever reaches WriteFile / CreateFolder.
|
||||
// before it reaches the root-confined write boundary.
|
||||
require.NoError(t, os.Symlink(outsideDir, filepath.Join(templateDir, "evil")))
|
||||
|
||||
tm := TemplateManager{}
|
||||
@@ -128,7 +128,7 @@ func TestZipSlip(t *testing.T) {
|
||||
}
|
||||
for _, entry := range entries {
|
||||
var tmp fs.FileInfo = &tempFileInfo{name: entry}
|
||||
writePath := tm.getAbsoluteFilePath(templateDir, entry, tmp)
|
||||
_, writePath := tm.getTemplateOutputLocation(templateDir, entry, tmp)
|
||||
require.Equal(t, "", writePath, entry)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -45,6 +45,15 @@ type ExecuteOptions struct {
|
||||
|
||||
TimeoutVariants *types.Timeouts
|
||||
|
||||
// ProxyURL is the HTTP(S) proxy URL for JS net dials via HTTP CONNECT.
|
||||
// SOCKS proxies are handled by fastdialer and must not be set here.
|
||||
ProxyURL string
|
||||
|
||||
// CustomHeaders contains global CLI headers for the nuclei/http module.
|
||||
// Raw nuclei/net connections construct their own wire payloads and do not
|
||||
// consume these headers.
|
||||
CustomHeaders []string
|
||||
|
||||
// Manually exported objects
|
||||
exports map[string]interface{}
|
||||
}
|
||||
|
||||
@@ -137,8 +137,22 @@ func (s *session) prepareCommon() {
|
||||
_ = s.config.runtime.Set(k, v)
|
||||
}
|
||||
|
||||
executionCtx := s.config.ctx
|
||||
if s.config.opts.TimeoutVariants != nil {
|
||||
s.config.runtime.SetContextValue("timeoutVariants", s.config.opts.TimeoutVariants)
|
||||
executionCtx = context.WithValue(executionCtx, "timeoutVariants", s.config.opts.TimeoutVariants) //nolint:staticcheck
|
||||
}
|
||||
if s.config.opts.ProxyURL != "" {
|
||||
s.config.runtime.SetContextValue("proxyURL", s.config.opts.ProxyURL)
|
||||
executionCtx = context.WithValue(executionCtx, "proxyURL", s.config.opts.ProxyURL) //nolint:staticcheck
|
||||
}
|
||||
if len(s.config.opts.CustomHeaders) > 0 {
|
||||
headers := append([]string(nil), s.config.opts.CustomHeaders...)
|
||||
s.config.runtime.SetContextValue("customHeaders", headers)
|
||||
executionCtx = context.WithValue(executionCtx, "customHeaders", headers) //nolint:staticcheck
|
||||
}
|
||||
s.config.runtime.SetContextValue("executionId", s.config.opts.ExecutionId)
|
||||
s.config.runtime.SetContextValue("ctx", s.config.ctx)
|
||||
s.config.runtime.SetContextValue("ctx", executionCtx)
|
||||
enableRequire(s.config.runtime)
|
||||
}
|
||||
|
||||
@@ -207,6 +221,9 @@ func (s *session) cleanupCommon() {
|
||||
}
|
||||
s.config.runtime.RemoveContextValue("executionId")
|
||||
s.config.runtime.RemoveContextValue("ctx")
|
||||
s.config.runtime.RemoveContextValue("timeoutVariants")
|
||||
s.config.runtime.RemoveContextValue("proxyURL")
|
||||
s.config.runtime.RemoveContextValue("customHeaders")
|
||||
}
|
||||
|
||||
func (s *session) cleanupPath() {
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package compiler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/goja"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSessionInjectsProxyAndTimeoutContext(t *testing.T) {
|
||||
runtime := goja.New()
|
||||
program, err := goja.Compile("test.js", `
|
||||
function run() {
|
||||
return { success: true };
|
||||
}
|
||||
run();
|
||||
`, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
var sawProxy string
|
||||
var sawTimeout time.Duration
|
||||
var sawHeaders []string
|
||||
proxyURL := "http://127.0.0.1:8080"
|
||||
customHeaders := []string{"X-Global: value"}
|
||||
timeouts := &types.Timeouts{
|
||||
TcpReadTimeout: 12 * time.Second,
|
||||
JsCompilerExecutionTimeout: 5 * time.Second,
|
||||
}
|
||||
|
||||
opts := &ExecuteOptions{
|
||||
ExecutionId: "session-proxy-test",
|
||||
ProxyURL: proxyURL,
|
||||
CustomHeaders: customHeaders,
|
||||
TimeoutVariants: timeouts,
|
||||
Callback: func(rt *goja.Runtime) error {
|
||||
if v, ok := rt.GetContextValue("proxyURL"); ok {
|
||||
sawProxy, _ = v.(string)
|
||||
}
|
||||
if v, ok := rt.GetContextValue("timeoutVariants"); ok {
|
||||
if tv, ok := v.(*types.Timeouts); ok && tv != nil {
|
||||
sawTimeout = tv.TcpReadTimeout
|
||||
}
|
||||
}
|
||||
if v, ok := rt.GetContextValue("customHeaders"); ok {
|
||||
sawHeaders, _ = v.([]string)
|
||||
}
|
||||
if v, ok := rt.GetContextValue("ctx"); ok {
|
||||
executionCtx, _ := v.(context.Context)
|
||||
require.Equal(t, proxyURL, executionCtx.Value("proxyURL"))
|
||||
require.Equal(t, timeouts, executionCtx.Value("timeoutVariants"))
|
||||
require.Equal(t, customHeaders, executionCtx.Value("customHeaders"))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
_, err = executeWithRuntime(context.Background(), runtime, program, NewExecuteArgs(), opts, nil)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "http://127.0.0.1:8080", sawProxy)
|
||||
require.Equal(t, 12*time.Second, sawTimeout)
|
||||
require.Equal(t, customHeaders, sawHeaders)
|
||||
|
||||
_, ok := runtime.GetContextValue("proxyURL")
|
||||
require.False(t, ok, "proxyURL should be cleaned up after execution")
|
||||
_, ok = runtime.GetContextValue("timeoutVariants")
|
||||
require.False(t, ok, "timeoutVariants should be cleaned up after execution")
|
||||
_, ok = runtime.GetContextValue("customHeaders")
|
||||
require.False(t, ok, "customHeaders should be cleaned up after execution")
|
||||
}
|
||||
|
||||
func TestSessionAppliesOptionsToJSHTTP(t *testing.T) {
|
||||
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
require.Equal(t, "example.com", r.Host)
|
||||
_, _ = io.WriteString(w, r.Header.Get("X-Global"))
|
||||
}))
|
||||
t.Cleanup(proxy.Close)
|
||||
|
||||
executionID := "session-js-http"
|
||||
require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: executionID}))
|
||||
t.Cleanup(func() { protocolstate.Close(executionID) })
|
||||
|
||||
source := `
|
||||
const http = require("nuclei/http");
|
||||
http.Get("http://example.com/through-proxy").Body;
|
||||
`
|
||||
program, err := goja.Compile("test.js", source, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
result, err := executeWithRuntime(t.Context(), createNewRuntime(), program, NewExecuteArgs(), &ExecuteOptions{
|
||||
ExecutionId: executionID,
|
||||
ProxyURL: proxy.URL,
|
||||
CustomHeaders: []string{"X-Global: inherited"},
|
||||
TimeoutVariants: &types.Timeouts{
|
||||
HttpTimeout: 2 * time.Second,
|
||||
JsCompilerExecutionTimeout: 5 * time.Second,
|
||||
},
|
||||
}, nil)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "inherited", result.String())
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/logrusorgru/aurora/v4"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
|
||||
jsnet "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/net"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
@@ -127,13 +128,7 @@ func initBuiltInFunc(runtime *goja.Runtime) {
|
||||
return false, errkit.New("isPortOpen: host or port is empty")
|
||||
}
|
||||
|
||||
executionId := ctx.Value("executionId").(string)
|
||||
dialer := protocolstate.GetDialersWithId(executionId)
|
||||
if dialer == nil {
|
||||
panic("dialers with executionId " + executionId + " not found")
|
||||
}
|
||||
|
||||
conn, err := dialer.Fastdialer.Dial(ctx, "tcp", net.JoinHostPort(host, port))
|
||||
conn, err := jsnet.Dial(ctx, "tcp", net.JoinHostPort(host, port))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/projectdiscovery/goja"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"golang.org/x/net/publicsuffix"
|
||||
)
|
||||
|
||||
@@ -87,9 +88,9 @@ func NewClient(call goja.ConstructorCall, runtime *goja.Runtime) *goja.Object {
|
||||
nj: utils.NewNucleiJS(runtime),
|
||||
FollowRedirects: true,
|
||||
MaxRedirects: defaultMaxRedirects,
|
||||
TimeoutSeconds: defaultTimeoutSeconds,
|
||||
TimeoutSeconds: timeoutSecondsFromRuntime(runtime),
|
||||
MaxBodyBytes: defaultMaxBodyBytes,
|
||||
headers: make(http.Header),
|
||||
headers: customHeadersFromRuntime(runtime),
|
||||
}
|
||||
c.nj.ObjectSig = "Client(options?)"
|
||||
|
||||
@@ -230,7 +231,7 @@ func Post(ctx context.Context, rawURL, body string) (*Response, error) {
|
||||
// @example
|
||||
// ```javascript
|
||||
// const http = require('nuclei/http');
|
||||
// const resp = http.Request('OPTIONS', 'https://example.com', '');
|
||||
// const resp = http.Request("OPTIONS", "https://example.com", "");
|
||||
// ```
|
||||
func Request(ctx context.Context, method, rawURL, body string) (*Response, error) {
|
||||
return defaultClient().Request(ctx, method, rawURL, body)
|
||||
@@ -240,13 +241,14 @@ func defaultClient() *Client {
|
||||
return &Client{
|
||||
FollowRedirects: true,
|
||||
MaxRedirects: defaultMaxRedirects,
|
||||
TimeoutSeconds: defaultTimeoutSeconds,
|
||||
TimeoutSeconds: 0,
|
||||
MaxBodyBytes: defaultMaxBodyBytes,
|
||||
headers: make(http.Header),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) do(ctx context.Context, method, rawURL, body string) (*Response, error) {
|
||||
timeout := requestTimeout(ctx, c.TimeoutSeconds)
|
||||
c.init()
|
||||
|
||||
executionID := executionIDFrom(ctx, c)
|
||||
@@ -281,8 +283,12 @@ func (c *Client) do(ctx context.Context, method, rawURL, body string) (*Response
|
||||
tlsConfig.ServerName = host
|
||||
}
|
||||
|
||||
proxy, err := proxyFromContext(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
Proxy: proxy,
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
return dialers.Fastdialer.Dial(ctx, network, addr)
|
||||
},
|
||||
@@ -294,12 +300,13 @@ func (c *Client) do(ctx context.Context, method, rawURL, body string) (*Response
|
||||
MaxIdleConns: 4,
|
||||
MaxIdleConnsPerHost: 4,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
ResponseHeaderTimeout: time.Duration(c.TimeoutSeconds) * time.Second,
|
||||
ResponseHeaderTimeout: timeout,
|
||||
}
|
||||
defer transport.CloseIdleConnections()
|
||||
|
||||
httpClient := &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: time.Duration(c.TimeoutSeconds) * time.Second,
|
||||
Timeout: timeout,
|
||||
}
|
||||
if c.jar != nil {
|
||||
httpClient.Jar = c.jar
|
||||
@@ -330,9 +337,10 @@ func (c *Client) do(ctx context.Context, method, rawURL, body string) (*Response
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
applyCustomHeaders(req.Header, customHeadersFromContext(ctx))
|
||||
for k, vals := range c.headers {
|
||||
for _, v := range vals {
|
||||
req.Header.Add(k, v)
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
}
|
||||
if req.Header.Get("User-Agent") == "" {
|
||||
@@ -366,6 +374,80 @@ func (c *Client) do(ctx context.Context, method, rawURL, body string) (*Response
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func timeoutSecondsFromRuntime(runtime *goja.Runtime) int {
|
||||
if runtime != nil {
|
||||
if value, ok := runtime.GetContextValue("timeoutVariants"); ok {
|
||||
if timeouts, ok := value.(*types.Timeouts); ok && timeouts != nil && timeouts.HttpTimeout > 0 {
|
||||
return int(timeouts.HttpTimeout / time.Second)
|
||||
}
|
||||
}
|
||||
}
|
||||
return defaultTimeoutSeconds
|
||||
}
|
||||
|
||||
func requestTimeout(ctx context.Context, seconds int) time.Duration {
|
||||
if seconds > 0 {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
if ctx != nil {
|
||||
if timeouts, ok := ctx.Value("timeoutVariants").(*types.Timeouts); ok && timeouts != nil && timeouts.HttpTimeout > 0 {
|
||||
return timeouts.HttpTimeout
|
||||
}
|
||||
}
|
||||
return defaultTimeoutSeconds * time.Second
|
||||
}
|
||||
|
||||
func proxyFromContext(ctx context.Context) (func(*http.Request) (*url.URL, error), error) {
|
||||
if ctx == nil {
|
||||
return http.ProxyFromEnvironment, nil
|
||||
}
|
||||
raw, _ := ctx.Value("proxyURL").(string)
|
||||
if raw == "" {
|
||||
return http.ProxyFromEnvironment, nil
|
||||
}
|
||||
proxyURL, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http: invalid proxy URL: %w", err)
|
||||
}
|
||||
if proxyURL.Host == "" || (proxyURL.Scheme != "http" && proxyURL.Scheme != "https") {
|
||||
return nil, fmt.Errorf("http: proxy URL must use http or https")
|
||||
}
|
||||
return http.ProxyURL(proxyURL), nil
|
||||
}
|
||||
|
||||
func customHeadersFromRuntime(runtime *goja.Runtime) http.Header {
|
||||
headers := make(http.Header)
|
||||
if runtime == nil {
|
||||
return headers
|
||||
}
|
||||
if value, ok := runtime.GetContextValue("customHeaders"); ok {
|
||||
if raw, ok := value.([]string); ok {
|
||||
applyCustomHeaders(headers, raw)
|
||||
}
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
func customHeadersFromContext(ctx context.Context) []string {
|
||||
if ctx == nil {
|
||||
return nil
|
||||
}
|
||||
headers, _ := ctx.Value("customHeaders").([]string)
|
||||
return headers
|
||||
}
|
||||
|
||||
func applyCustomHeaders(headers http.Header, rawHeaders []string) {
|
||||
for _, raw := range rawHeaders {
|
||||
key, value, ok := strings.Cut(raw, ":")
|
||||
key = strings.TrimSpace(key)
|
||||
value = strings.TrimSpace(value)
|
||||
if !ok || key == "" || value == "" {
|
||||
continue
|
||||
}
|
||||
headers.Set(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
func executionIDFrom(ctx context.Context, c *Client) string {
|
||||
if c != nil && c.nj != nil {
|
||||
if id := c.nj.ExecutionId(); id != "" {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/goja"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
@@ -192,6 +193,113 @@ func TestClientSetHeader(t *testing.T) {
|
||||
require.Equal(t, "from-client", resp.Body)
|
||||
}
|
||||
|
||||
func TestClientInheritsGlobalHeaders(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.WriteString(w, r.Header.Get("X-Global")+"|"+r.Header.Get("X-Override"))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
executionID := initExec(t, &types.Options{})
|
||||
client := newRuntimeClient(t, executionID, Options{})
|
||||
client.SetHeader("X-Override", "client")
|
||||
|
||||
ctx := withExec(executionID)
|
||||
ctx = context.WithValue(ctx, "customHeaders", []string{ //nolint:staticcheck
|
||||
"X-Global: from-cli",
|
||||
"X-Override: global",
|
||||
"invalid",
|
||||
})
|
||||
resp, err := client.Get(ctx, srv.URL)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "from-cli|client", resp.Body)
|
||||
}
|
||||
|
||||
func TestClientUsesConfiguredProxy(t *testing.T) {
|
||||
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
require.Equal(t, "example.com", r.Host)
|
||||
require.Equal(t, "/through-proxy", r.URL.Path)
|
||||
_, _ = io.WriteString(w, "proxied")
|
||||
}))
|
||||
t.Cleanup(proxy.Close)
|
||||
|
||||
executionID := initExec(t, &types.Options{})
|
||||
client := newRuntimeClient(t, executionID, Options{})
|
||||
ctx := withExec(executionID)
|
||||
ctx = context.WithValue(ctx, "proxyURL", proxy.URL) //nolint:staticcheck
|
||||
|
||||
resp, err := client.Get(ctx, "http://example.com/through-proxy")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "proxied", resp.Body)
|
||||
}
|
||||
|
||||
func TestClientUsesConfiguredProxyForHTTPS(t *testing.T) {
|
||||
target := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = io.WriteString(w, "secure-proxied")
|
||||
}))
|
||||
t.Cleanup(target.Close)
|
||||
|
||||
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
require.Equal(t, http.MethodConnect, r.Method)
|
||||
upstream, err := net.Dial("tcp", r.Host)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = upstream.Close() }()
|
||||
|
||||
hijacker, ok := w.(http.Hijacker)
|
||||
require.True(t, ok)
|
||||
client, rw, err := hijacker.Hijack()
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = client.Close() }()
|
||||
|
||||
_, err = rw.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rw.Flush())
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
_, _ = io.Copy(upstream, client)
|
||||
close(done)
|
||||
}()
|
||||
_, _ = io.Copy(client, upstream)
|
||||
<-done
|
||||
}))
|
||||
t.Cleanup(proxy.Close)
|
||||
|
||||
executionID := initExec(t, &types.Options{})
|
||||
client := newRuntimeClient(t, executionID, Options{})
|
||||
ctx := withExec(executionID)
|
||||
ctx = context.WithValue(ctx, "proxyURL", proxy.URL) //nolint:staticcheck
|
||||
|
||||
resp, err := client.Get(ctx, target.URL)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "secure-proxied", resp.Body)
|
||||
}
|
||||
|
||||
func TestClientRejectsInvalidConfiguredProxy(t *testing.T) {
|
||||
executionID := initExec(t, &types.Options{})
|
||||
client := newRuntimeClient(t, executionID, Options{})
|
||||
ctx := withExec(executionID)
|
||||
ctx = context.WithValue(ctx, "proxyURL", "socks5://127.0.0.1:1080") //nolint:staticcheck
|
||||
|
||||
_, err := client.Get(ctx, "http://example.com/")
|
||||
require.ErrorContains(t, err, "must use http or https")
|
||||
}
|
||||
|
||||
func TestClientUsesConfiguredHTTPTimeout(t *testing.T) {
|
||||
timeouts := &types.Timeouts{HttpTimeout: 7 * time.Second}
|
||||
ctx := context.WithValue(context.Background(), "timeoutVariants", timeouts) //nolint:staticcheck
|
||||
require.Equal(t, 7*time.Second, requestTimeout(ctx, 0))
|
||||
require.Equal(t, 2*time.Second, requestTimeout(ctx, 2), "client option must override the global timeout")
|
||||
|
||||
runtime := goja.New()
|
||||
runtime.SetContextValue("executionId", "timeout-test")
|
||||
runtime.SetContextValue("timeoutVariants", timeouts)
|
||||
obj, err := runtime.New(runtime.ToValue(NewClient))
|
||||
require.NoError(t, err)
|
||||
client, ok := obj.Export().(*Client)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, 7, client.TimeoutSeconds)
|
||||
}
|
||||
|
||||
func TestClientHead(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
require.Equal(t, http.MethodHead, r.Method)
|
||||
|
||||
+184
-11
@@ -1,11 +1,15 @@
|
||||
package net
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
@@ -18,6 +22,162 @@ var (
|
||||
defaultTimeout = time.Duration(5) * time.Second
|
||||
)
|
||||
|
||||
// getTimeoutFromContext returns the configured TCP timeout or defaultTimeout.
|
||||
func getTimeoutFromContext(ctx context.Context) time.Duration {
|
||||
if tv, ok := ctx.Value("timeoutVariants").(*types.Timeouts); ok && tv != nil {
|
||||
if tv.TcpReadTimeout > 0 {
|
||||
return tv.TcpReadTimeout
|
||||
}
|
||||
}
|
||||
return defaultTimeout
|
||||
}
|
||||
|
||||
func proxyURLFromContext(ctx context.Context) string {
|
||||
if proxyURL, ok := ctx.Value("proxyURL").(string); ok {
|
||||
return proxyURL
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func executionIDFromContext(ctx context.Context) (string, error) {
|
||||
executionId, _ := ctx.Value("executionId").(string)
|
||||
if executionId == "" {
|
||||
return "", fmt.Errorf("executionId missing from context")
|
||||
}
|
||||
return executionId, nil
|
||||
}
|
||||
|
||||
type dialFunc func(ctx context.Context, network, address string) (net.Conn, error)
|
||||
|
||||
// dialHTTPProxy establishes a TCP connection through an HTTP CONNECT proxy.
|
||||
func dialHTTPProxy(ctx context.Context, dial dialFunc, proxyURL string, address string, timeout time.Duration) (net.Conn, error) {
|
||||
parsed, err := url.Parse(proxyURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid proxy URL: %w", err)
|
||||
}
|
||||
proxyAddr := parsed.Host
|
||||
if _, _, splitErr := net.SplitHostPort(proxyAddr); splitErr != nil {
|
||||
defaultPort := "8080"
|
||||
if parsed.Scheme == "https" {
|
||||
defaultPort = "443"
|
||||
}
|
||||
proxyAddr = net.JoinHostPort(proxyAddr, defaultPort)
|
||||
}
|
||||
|
||||
proxyConn, err := dial(ctx, "tcp", proxyAddr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not connect to proxy %s: %w", proxyAddr, err)
|
||||
}
|
||||
if parsed.Scheme == "https" {
|
||||
config := &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10}
|
||||
if hostname := parsed.Hostname(); hostname != "" {
|
||||
config.ServerName = hostname
|
||||
}
|
||||
tlsConn := tls.Client(proxyConn, config)
|
||||
if err := tlsConn.HandshakeContext(ctx); err != nil {
|
||||
_ = proxyConn.Close()
|
||||
return nil, fmt.Errorf("could not establish TLS connection to proxy %s: %w", proxyAddr, err)
|
||||
}
|
||||
proxyConn = tlsConn
|
||||
}
|
||||
|
||||
connectReq := &http.Request{
|
||||
Method: http.MethodConnect,
|
||||
URL: &url.URL{Opaque: address},
|
||||
Host: address,
|
||||
Header: make(http.Header),
|
||||
}
|
||||
if parsed.User != nil {
|
||||
connectReq.Header.Set("Proxy-Authorization", "Basic "+basicAuth(parsed.User))
|
||||
}
|
||||
|
||||
_ = proxyConn.SetDeadline(time.Now().Add(timeout))
|
||||
if err := connectReq.Write(proxyConn); err != nil {
|
||||
_ = proxyConn.Close()
|
||||
return nil, fmt.Errorf("could not send CONNECT request: %w", err)
|
||||
}
|
||||
|
||||
br := bufio.NewReader(proxyConn)
|
||||
resp, err := http.ReadResponse(br, connectReq)
|
||||
if err != nil {
|
||||
_ = proxyConn.Close()
|
||||
return nil, fmt.Errorf("could not read CONNECT response: %w", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_ = proxyConn.Close()
|
||||
return nil, fmt.Errorf("proxy CONNECT returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// reset deadline after successful CONNECT
|
||||
_ = proxyConn.SetDeadline(time.Time{})
|
||||
|
||||
// preserve any bytes the buffered reader consumed past the CONNECT response
|
||||
if br.Buffered() > 0 {
|
||||
return &bufferedConn{Conn: proxyConn, reader: br}, nil
|
||||
}
|
||||
return proxyConn, nil
|
||||
}
|
||||
|
||||
// basicAuth encodes proxy user credentials.
|
||||
func basicAuth(user *url.Userinfo) string {
|
||||
username := user.Username()
|
||||
password, _ := user.Password()
|
||||
return base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
|
||||
}
|
||||
|
||||
func redactProxyURL(proxyURL string) string {
|
||||
parsed, err := url.Parse(proxyURL)
|
||||
if err != nil {
|
||||
return "<invalid proxy URL>"
|
||||
}
|
||||
parsed.User = nil
|
||||
return parsed.String()
|
||||
}
|
||||
|
||||
// bufferedConn preserves bytes buffered during the CONNECT handshake.
|
||||
type bufferedConn struct {
|
||||
net.Conn
|
||||
reader *bufio.Reader
|
||||
}
|
||||
|
||||
func (c *bufferedConn) Read(b []byte) (int, error) {
|
||||
return c.reader.Read(b)
|
||||
}
|
||||
|
||||
// Dial opens a connection using the execution dialer.
|
||||
// When an HTTP(S) proxy is present in context, TCP dials use HTTP CONNECT.
|
||||
// Misconfigured proxies return an error instead of silently dialing direct.
|
||||
func Dial(ctx context.Context, protocol, address string) (net.Conn, error) {
|
||||
executionId, err := executionIDFromContext(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dialer := protocolstate.GetDialersWithId(executionId)
|
||||
if dialer == nil {
|
||||
return nil, fmt.Errorf("dialers not initialized for %s", executionId)
|
||||
}
|
||||
|
||||
if proxyURL := proxyURLFromContext(ctx); proxyURL != "" {
|
||||
if protocol != "tcp" {
|
||||
return nil, fmt.Errorf("http proxy does not support protocol %q", protocol)
|
||||
}
|
||||
parsed, err := url.Parse(proxyURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid proxy URL %s: %w", redactProxyURL(proxyURL), err)
|
||||
}
|
||||
switch parsed.Scheme {
|
||||
case "http", "https":
|
||||
return dialHTTPProxy(ctx, dialer.Fastdialer.Dial, proxyURL, address, getTimeoutFromContext(ctx))
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported proxy scheme %q in %s (expected http or https)", parsed.Scheme, redactProxyURL(proxyURL))
|
||||
}
|
||||
}
|
||||
|
||||
return dialer.Fastdialer.Dial(ctx, protocol, address)
|
||||
}
|
||||
|
||||
// Open opens a new connection to the address with a timeout.
|
||||
// supported protocols: tcp, udp
|
||||
// @example
|
||||
@@ -26,19 +186,15 @@ var (
|
||||
// const conn = net.Open('tcp', 'acme.com:80');
|
||||
// ```
|
||||
func Open(ctx context.Context, protocol, address string) (*NetConn, error) {
|
||||
executionId := ctx.Value("executionId").(string)
|
||||
dialer := protocolstate.GetDialersWithId(executionId)
|
||||
if dialer == nil {
|
||||
return nil, fmt.Errorf("dialers not initialized for %s", executionId)
|
||||
}
|
||||
conn, err := dialer.Fastdialer.Dial(ctx, protocol, address)
|
||||
timeout := getTimeoutFromContext(ctx)
|
||||
conn, err := Dial(ctx, protocol, address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &NetConn{conn: conn, timeout: defaultTimeout}, nil
|
||||
return &NetConn{conn: conn, timeout: timeout}, nil
|
||||
}
|
||||
|
||||
// Open opens a new connection to the address with a timeout.
|
||||
// OpenTLS opens a new TLS connection to the address with a timeout.
|
||||
// supported protocols: tcp, udp
|
||||
// @example
|
||||
// ```javascript
|
||||
@@ -46,6 +202,7 @@ func Open(ctx context.Context, protocol, address string) (*NetConn, error) {
|
||||
// const conn = net.OpenTLS('tcp', 'acme.com:443');
|
||||
// ```
|
||||
func OpenTLS(ctx context.Context, protocol, address string) (*NetConn, error) {
|
||||
timeout := getTimeoutFromContext(ctx)
|
||||
config := &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10}
|
||||
host, _, _ := net.SplitHostPort(address)
|
||||
if host != "" {
|
||||
@@ -53,17 +210,33 @@ func OpenTLS(ctx context.Context, protocol, address string) (*NetConn, error) {
|
||||
c.ServerName = host
|
||||
config = c
|
||||
}
|
||||
executionId := ctx.Value("executionId").(string)
|
||||
|
||||
if proxyURLFromContext(ctx) != "" {
|
||||
conn, err := Dial(ctx, protocol, address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tlsConn := tls.Client(conn, config)
|
||||
if err := tlsConn.HandshakeContext(ctx); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("TLS handshake failed: %w", err)
|
||||
}
|
||||
return &NetConn{conn: tlsConn, timeout: timeout}, nil
|
||||
}
|
||||
|
||||
executionId, err := executionIDFromContext(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dialer := protocolstate.GetDialersWithId(executionId)
|
||||
if dialer == nil {
|
||||
return nil, fmt.Errorf("dialers not initialized for %s", executionId)
|
||||
}
|
||||
|
||||
conn, err := dialer.Fastdialer.DialTLSWithConfig(ctx, protocol, address, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &NetConn{conn: conn, timeout: defaultTimeout}, nil
|
||||
return &NetConn{conn: conn, timeout: timeout}, nil
|
||||
}
|
||||
|
||||
type (
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
package net
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGetTimeoutFromContext(t *testing.T) {
|
||||
t.Run("default when missing", func(t *testing.T) {
|
||||
require.Equal(t, defaultTimeout, getTimeoutFromContext(context.Background()))
|
||||
})
|
||||
|
||||
t.Run("reads TcpReadTimeout", func(t *testing.T) {
|
||||
expected := 15 * time.Second
|
||||
tv := &types.Timeouts{TcpReadTimeout: expected}
|
||||
ctx := context.WithValue(context.Background(), "timeoutVariants", tv) // nolint: staticcheck
|
||||
require.Equal(t, expected, getTimeoutFromContext(ctx))
|
||||
})
|
||||
}
|
||||
|
||||
func TestBufferedConn(t *testing.T) {
|
||||
t.Run("read drains buffered bytes first", func(t *testing.T) {
|
||||
underlying := &bytes.Buffer{}
|
||||
underlying.WriteString("hello world")
|
||||
|
||||
br := bufio.NewReader(underlying)
|
||||
_, _ = br.Peek(5)
|
||||
|
||||
conn := &bufferedConn{
|
||||
Conn: &fakeConn{},
|
||||
reader: br,
|
||||
}
|
||||
|
||||
buf := make([]byte, 64)
|
||||
n, err := conn.Read(buf)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "hello world", string(buf[:n]))
|
||||
})
|
||||
|
||||
t.Run("read returns EOF after buffer is drained", func(t *testing.T) {
|
||||
underlying := bytes.NewBufferString("data")
|
||||
br := bufio.NewReader(underlying)
|
||||
conn := &bufferedConn{Conn: &fakeConn{}, reader: br}
|
||||
|
||||
buf := make([]byte, 64)
|
||||
n, err := conn.Read(buf)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "data", string(buf[:n]))
|
||||
|
||||
_, err = conn.Read(buf)
|
||||
require.ErrorIs(t, err, io.EOF)
|
||||
})
|
||||
}
|
||||
|
||||
// directDial is a plain net.Dialer for tests (no fastdialer dependency).
|
||||
var directDial = (&net.Dialer{Timeout: 5 * time.Second}).DialContext
|
||||
|
||||
func TestDialHTTPProxy(t *testing.T) {
|
||||
t.Run("successful CONNECT tunnel", func(t *testing.T) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = ln.Close() }()
|
||||
|
||||
targetPayload := "hello from target"
|
||||
go func() {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
req, err := http.ReadRequest(bufio.NewReader(conn))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = req.Body.Close()
|
||||
|
||||
_, _ = conn.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n"))
|
||||
_, _ = conn.Write([]byte(targetPayload))
|
||||
}()
|
||||
|
||||
ctx := context.Background()
|
||||
proxyURL := fmt.Sprintf("http://%s", ln.Addr().String())
|
||||
conn, err := dialHTTPProxy(ctx, directDial, proxyURL, "example.com:443", 5*time.Second)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
buf := make([]byte, 64)
|
||||
n, err := conn.Read(buf)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, targetPayload, string(buf[:n]))
|
||||
})
|
||||
|
||||
t.Run("proxy returns non-200 status", func(t *testing.T) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = ln.Close() }()
|
||||
|
||||
go func() {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
req, err := http.ReadRequest(bufio.NewReader(conn))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = req.Body.Close()
|
||||
_, _ = conn.Write([]byte("HTTP/1.1 403 Forbidden\r\n\r\n"))
|
||||
}()
|
||||
|
||||
ctx := context.Background()
|
||||
proxyURL := fmt.Sprintf("http://%s", ln.Addr().String())
|
||||
_, err = dialHTTPProxy(ctx, directDial, proxyURL, "example.com:443", 5*time.Second)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "403")
|
||||
})
|
||||
|
||||
t.Run("invalid proxy URL", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
_, err := dialHTTPProxy(ctx, directDial, "://bad-url", "example.com:443", 5*time.Second)
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("successful HTTPS CONNECT tunnel", func(t *testing.T) {
|
||||
targetPayload := "hello from secure proxy"
|
||||
proxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodConnect {
|
||||
http.Error(w, "expected CONNECT", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
hijacker, ok := w.(http.Hijacker)
|
||||
if !ok {
|
||||
http.Error(w, "hijacking not supported", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
conn, rw, err := hijacker.Hijack()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
_, _ = rw.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n")
|
||||
_, _ = rw.WriteString(targetPayload)
|
||||
_ = rw.Flush()
|
||||
}))
|
||||
defer proxy.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
conn, err := dialHTTPProxy(ctx, directDial, proxy.URL, "example.com:443", 5*time.Second)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
buf := make([]byte, 64)
|
||||
n, err := conn.Read(buf)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, targetPayload, string(buf[:n]))
|
||||
})
|
||||
|
||||
t.Run("unreachable proxy", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
_, err := dialHTTPProxy(ctx, directDial, "http://127.0.0.1:1", "example.com:443", 1*time.Second)
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("proxy auth header", func(t *testing.T) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = ln.Close() }()
|
||||
|
||||
gotAuth := make(chan string, 1)
|
||||
go func() {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
req, err := http.ReadRequest(bufio.NewReader(conn))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = req.Body.Close()
|
||||
gotAuth <- req.Header.Get("Proxy-Authorization")
|
||||
_, _ = conn.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n"))
|
||||
}()
|
||||
|
||||
proxyURL := fmt.Sprintf("http://user:secret@%s", ln.Addr().String())
|
||||
conn, err := dialHTTPProxy(context.Background(), directDial, proxyURL, "example.com:443", 5*time.Second)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
require.Equal(t, "Basic "+basicAuth(url.UserPassword("user", "secret")), <-gotAuth)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDial(t *testing.T) {
|
||||
t.Run("missing execution id", func(t *testing.T) {
|
||||
_, err := Dial(context.Background(), "tcp", "127.0.0.1:1")
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "executionId")
|
||||
})
|
||||
|
||||
t.Run("unsupported proxy scheme fails hard", func(t *testing.T) {
|
||||
opts := &types.Options{ExecutionId: "net-dial-scheme-" + t.Name()}
|
||||
require.NoError(t, protocolstate.Init(opts))
|
||||
t.Cleanup(func() { protocolstate.Close(opts.ExecutionId) })
|
||||
|
||||
ctx := context.WithValue(context.Background(), "executionId", opts.ExecutionId) // nolint: staticcheck
|
||||
ctx = context.WithValue(ctx, "proxyURL", "socks5://127.0.0.1:1080") // nolint: staticcheck
|
||||
_, err := Dial(ctx, "tcp", "example.com:80")
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "unsupported proxy scheme")
|
||||
})
|
||||
|
||||
t.Run("invalid proxy url fails hard", func(t *testing.T) {
|
||||
opts := &types.Options{ExecutionId: "net-dial-invalid-" + t.Name()}
|
||||
require.NoError(t, protocolstate.Init(opts))
|
||||
t.Cleanup(func() { protocolstate.Close(opts.ExecutionId) })
|
||||
|
||||
ctx := context.WithValue(context.Background(), "executionId", opts.ExecutionId) // nolint: staticcheck
|
||||
ctx = context.WithValue(ctx, "proxyURL", "://bad") // nolint: staticcheck
|
||||
_, err := Dial(ctx, "tcp", "example.com:80")
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "invalid proxy URL")
|
||||
})
|
||||
|
||||
t.Run("udp with http proxy fails hard", func(t *testing.T) {
|
||||
opts := &types.Options{ExecutionId: "net-dial-udp-" + t.Name()}
|
||||
require.NoError(t, protocolstate.Init(opts))
|
||||
t.Cleanup(func() { protocolstate.Close(opts.ExecutionId) })
|
||||
|
||||
ctx := context.WithValue(context.Background(), "executionId", opts.ExecutionId) // nolint: staticcheck
|
||||
ctx = context.WithValue(ctx, "proxyURL", "http://127.0.0.1:8080") // nolint: staticcheck
|
||||
_, err := Dial(ctx, "udp", "example.com:53")
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "does not support protocol")
|
||||
})
|
||||
|
||||
t.Run("open through local http proxy", func(t *testing.T) {
|
||||
target, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = target.Close() }()
|
||||
|
||||
go func() {
|
||||
conn, err := target.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
_, _ = conn.Write([]byte("pong"))
|
||||
}()
|
||||
|
||||
proxyLn, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = proxyLn.Close() }()
|
||||
|
||||
go func() {
|
||||
conn, err := proxyLn.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
req, err := http.ReadRequest(bufio.NewReader(conn))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = req.Body.Close()
|
||||
require.Equal(t, http.MethodConnect, req.Method)
|
||||
require.Equal(t, target.Addr().String(), req.Host)
|
||||
|
||||
upstream, err := net.Dial("tcp", target.Addr().String())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() { _ = upstream.Close() }()
|
||||
|
||||
_, _ = conn.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n"))
|
||||
go func() { _, _ = io.Copy(upstream, conn) }()
|
||||
_, _ = io.Copy(conn, upstream)
|
||||
}()
|
||||
|
||||
opts := &types.Options{ExecutionId: "net-dial-proxy-" + t.Name()}
|
||||
require.NoError(t, protocolstate.Init(opts))
|
||||
t.Cleanup(func() { protocolstate.Close(opts.ExecutionId) })
|
||||
|
||||
ctx := context.WithValue(context.Background(), "executionId", opts.ExecutionId) // nolint: staticcheck
|
||||
ctx = context.WithValue(ctx, "proxyURL", "http://"+proxyLn.Addr().String()) // nolint: staticcheck
|
||||
ctx = context.WithValue(ctx, "timeoutVariants", &types.Timeouts{TcpReadTimeout: 5 * time.Second}) // nolint: staticcheck
|
||||
|
||||
nconn, err := Open(ctx, "tcp", target.Addr().String())
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = nconn.Close() }()
|
||||
|
||||
buf := make([]byte, 16)
|
||||
n, err := nconn.conn.Read(buf)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "pong", string(buf[:n]))
|
||||
})
|
||||
}
|
||||
|
||||
func TestRedactProxyURL(t *testing.T) {
|
||||
require.Equal(t, "http://proxy.example.com:8080", redactProxyURL("http://user:[email protected]:8080"))
|
||||
require.Equal(t, "http://proxy.example.com:8080", redactProxyURL("http://proxy.example.com:8080"))
|
||||
require.Equal(t, "<invalid proxy URL>", redactProxyURL("://bad-url"))
|
||||
}
|
||||
|
||||
// fakeConn is a minimal net.Conn for testing bufferedConn.
|
||||
type fakeConn struct{ net.Conn }
|
||||
|
||||
func (f *fakeConn) Read([]byte) (int, error) { return 0, io.EOF }
|
||||
func (f *fakeConn) Write([]byte) (int, error) { return 0, nil }
|
||||
func (f *fakeConn) Close() error { return nil }
|
||||
func (f *fakeConn) LocalAddr() net.Addr { return nil }
|
||||
func (f *fakeConn) RemoteAddr() net.Addr { return nil }
|
||||
func (f *fakeConn) SetDeadline(time.Time) error { return nil }
|
||||
func (f *fakeConn) SetReadDeadline(time.Time) error { return nil }
|
||||
func (f *fakeConn) SetWriteDeadline(time.Time) error { return nil }
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/projectdiscovery/fastdialer/fastdialer"
|
||||
"github.com/projectdiscovery/mapcidr/asn"
|
||||
"github.com/projectdiscovery/networkpolicy"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpcache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/expand"
|
||||
mapsutil "github.com/projectdiscovery/utils/maps"
|
||||
@@ -328,6 +329,7 @@ func Close(executionId string) {
|
||||
dialers.Delete(executionId)
|
||||
|
||||
if dialers.IsEmpty() {
|
||||
httpcache.Close()
|
||||
StopActiveMemGuardian()
|
||||
}
|
||||
}
|
||||
@@ -174,6 +174,10 @@ type Request struct {
|
||||
// DisableCookie is an optional setting that disables cookie reuse
|
||||
DisableCookie bool `yaml:"disable-cookie,omitempty" json:"disable-cookie,omitempty" jsonschema:"title=optional disable cookie reuse,description=Optional setting that disables cookie reuse"`
|
||||
|
||||
// description: |
|
||||
// DisableHTTPCache turns off HTTP caching for this request. It cannot turn caching on when -http-cache is unset.
|
||||
DisableHTTPCache bool `yaml:"disable-http-cache,omitempty" json:"disable-http-cache,omitempty" jsonschema:"title=disable HTTP cache,description=Turns off HTTP caching for this request; cannot enable cache when -http-cache is unset"`
|
||||
|
||||
// description: |
|
||||
// Enables force reading of the entire raw unsafe request body ignoring
|
||||
// any specified content length headers.
|
||||
@@ -354,10 +358,11 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
|
||||
}
|
||||
|
||||
connectionConfiguration := &httpclientpool.Configuration{
|
||||
Threads: request.Threads,
|
||||
MaxRedirects: request.MaxRedirects,
|
||||
NoTimeout: false,
|
||||
DisableCookie: request.DisableCookie,
|
||||
Threads: request.Threads,
|
||||
MaxRedirects: request.MaxRedirects,
|
||||
NoTimeout: false,
|
||||
DisableCookie: request.DisableCookie,
|
||||
DisableHTTPCache: request.DisableHTTPCache,
|
||||
Connection: &httpclientpool.ConnectionConfiguration{
|
||||
DisableKeepAlive: disableKeepAlive,
|
||||
},
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
// Package httpcache provides RFC 9111 compliant HTTP response caching for nuclei.
|
||||
// It wraps http.RoundTripper to cache responses based on standard HTTP caching headers.
|
||||
package httpcache
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/sandrolain/httpcache"
|
||||
"github.com/sandrolain/httpcache/leveldbcache"
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
)
|
||||
|
||||
const defaultMaxCacheBytes = 256 << 20 // 256 MiB
|
||||
|
||||
var (
|
||||
cacheMu sync.Mutex
|
||||
cache httpcache.Cache
|
||||
diskDB *leveldb.DB
|
||||
maxCacheBytes int64 = defaultMaxCacheBytes
|
||||
)
|
||||
|
||||
// GetCache returns the shared cache, opening LevelDB on first use.
|
||||
// After Close, the next call opens a new handle.
|
||||
func GetCache() httpcache.Cache {
|
||||
cacheMu.Lock()
|
||||
defer cacheMu.Unlock()
|
||||
if cache != nil {
|
||||
return cache
|
||||
}
|
||||
|
||||
cacheDir := filepath.Join(config.DefaultConfig.GetCacheDir(), "httpcache")
|
||||
opened, db, err := openDiskCache(cacheDir, maxCacheBytes)
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("HTTP cache: could not open %q, using memory: %s", cacheDir, err)
|
||||
cache = newBoundedCache(httpcache.NewMemoryCache(), "", maxCacheBytes)
|
||||
return cache
|
||||
}
|
||||
gologger.Verbose().Msgf("HTTP cache initialized at %q (max %d bytes)", cacheDir, maxCacheBytes)
|
||||
diskDB = db
|
||||
cache = opened
|
||||
return cache
|
||||
}
|
||||
|
||||
// Close releases the LevelDB handle. Safe if the cache was never opened or
|
||||
// is using the memory fallback. The next GetCache call reopens it.
|
||||
func Close() {
|
||||
cacheMu.Lock()
|
||||
defer cacheMu.Unlock()
|
||||
if diskDB != nil {
|
||||
if err := diskDB.Close(); err != nil {
|
||||
gologger.Warning().Msgf("HTTP cache: close: %s", err)
|
||||
}
|
||||
diskDB = nil
|
||||
}
|
||||
cache = nil
|
||||
}
|
||||
|
||||
// NewTransportWrapper returns a function that wraps an [http.RoundTripper] with
|
||||
// caching.
|
||||
//
|
||||
// This is designed to be used with retryablehttp's WrapTransport option.
|
||||
func NewTransportWrapper() func(http.RoundTripper) http.RoundTripper {
|
||||
c := GetCache()
|
||||
|
||||
return func(rt http.RoundTripper) http.RoundTripper {
|
||||
transport := &httpcache.Transport{
|
||||
Transport: rt,
|
||||
Cache: c,
|
||||
MarkCachedResponses: false,
|
||||
SkipServerErrorsFromCache: true,
|
||||
AsyncRevalidateTimeout: 10 * time.Second,
|
||||
IsPublicCache: false,
|
||||
EnableVarySeparation: true,
|
||||
ShouldCache: shouldCache,
|
||||
CacheKeyHeaders: []string{
|
||||
"Authorization",
|
||||
"Cookie",
|
||||
"Accept-Encoding",
|
||||
"Accept-Language",
|
||||
"Accept",
|
||||
"Origin",
|
||||
},
|
||||
DisableWarningHeader: true,
|
||||
}
|
||||
return transport
|
||||
}
|
||||
}
|
||||
|
||||
func openDiskCache(dir string, maxBytes int64) (httpcache.Cache, *leveldb.DB, error) {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, nil, fmt.Errorf("create cache dir: %w", err)
|
||||
}
|
||||
|
||||
db, err := leveldb.OpenFile(dir, &opt.Options{
|
||||
WriteBuffer: 1 << 20,
|
||||
CompactionTableSize: 2 << 20,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return newBoundedCache(leveldbcache.NewWithDB(db), dir, maxBytes), db, nil
|
||||
}
|
||||
|
||||
func shouldCache(resp *http.Response) bool {
|
||||
return isCacheableStatus(resp.StatusCode)
|
||||
}
|
||||
|
||||
func isCacheableStatus(code int) bool {
|
||||
return code >= 200 && code < 300
|
||||
}
|
||||
|
||||
func newBoundedCache(inner httpcache.Cache, dir string, maxBytes int64) *statusFilterCache {
|
||||
return &statusFilterCache{
|
||||
inner: inner,
|
||||
dir: dir,
|
||||
maxBytes: maxBytes,
|
||||
}
|
||||
}
|
||||
|
||||
// statusFilterCache drops non-2xx entries. sandrolain/httpcache v1.4.0 stores
|
||||
// 301/404/405/410/414/501 without calling ShouldCache.
|
||||
// When dir is set, Set is skipped once the LevelDB directory reaches maxBytes.
|
||||
type statusFilterCache struct {
|
||||
inner httpcache.Cache
|
||||
dir string
|
||||
maxBytes int64
|
||||
setCount atomic.Uint32
|
||||
fullWarned atomic.Bool
|
||||
}
|
||||
|
||||
func (c *statusFilterCache) Get(key string) ([]byte, bool) {
|
||||
data, ok := c.inner.Get(key)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
if !storedResponseIs2xx(data) {
|
||||
c.inner.Delete(key)
|
||||
return nil, false
|
||||
}
|
||||
return data, true
|
||||
}
|
||||
|
||||
func (c *statusFilterCache) Set(key string, responseBytes []byte) {
|
||||
if !storedResponseIs2xx(responseBytes) {
|
||||
return
|
||||
}
|
||||
if c.maxBytes > 0 && c.overBudget(len(responseBytes)) {
|
||||
if c.fullWarned.CompareAndSwap(false, true) {
|
||||
gologger.Warning().Msgf("HTTP cache: at %d byte cap, not storing more responses", c.maxBytes)
|
||||
}
|
||||
return
|
||||
}
|
||||
c.inner.Set(key, responseBytes)
|
||||
}
|
||||
|
||||
func (c *statusFilterCache) Delete(key string) {
|
||||
c.inner.Delete(key)
|
||||
}
|
||||
|
||||
func (c *statusFilterCache) overBudget(incoming int) bool {
|
||||
if c.dir == "" {
|
||||
return false
|
||||
}
|
||||
// Directory walk is not free; sample every 32 writes after the first.
|
||||
n := c.setCount.Add(1)
|
||||
if n > 1 && n%32 != 0 {
|
||||
return false
|
||||
}
|
||||
return dirSize(c.dir)+int64(incoming) >= c.maxBytes
|
||||
}
|
||||
|
||||
func dirSize(root string) int64 {
|
||||
var total int64
|
||||
_ = filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
total += info.Size()
|
||||
return nil
|
||||
})
|
||||
return total
|
||||
}
|
||||
|
||||
func storedResponseIs2xx(data []byte) bool {
|
||||
resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(data)), nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
return isCacheableStatus(resp.StatusCode)
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package httpcache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/http/httputil"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
libcache "github.com/sandrolain/httpcache"
|
||||
)
|
||||
|
||||
func dumpedResponse(t *testing.T, status int) []byte {
|
||||
t.Helper()
|
||||
resp := &http.Response{
|
||||
StatusCode: status,
|
||||
Status: http.StatusText(status),
|
||||
Proto: "HTTP/1.1",
|
||||
ProtoMajor: 1,
|
||||
ProtoMinor: 1,
|
||||
Header: make(http.Header),
|
||||
Body: http.NoBody,
|
||||
}
|
||||
resp.Header.Set("Cache-Control", "max-age=60")
|
||||
data, err := httputil.DumpResponse(resp, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func TestStatusFilterCacheRejectsNon2xx(t *testing.T) {
|
||||
inner := libcache.NewMemoryCache()
|
||||
c := &statusFilterCache{inner: inner}
|
||||
|
||||
okBody := dumpedResponse(t, http.StatusOK)
|
||||
c.Set("ok", okBody)
|
||||
got, ok := c.Get("ok")
|
||||
if !ok {
|
||||
t.Fatal("2xx response was not stored")
|
||||
}
|
||||
if string(got) != string(okBody) {
|
||||
t.Fatal("2xx cache payload changed")
|
||||
}
|
||||
|
||||
c.Set("missing", dumpedResponse(t, http.StatusNotFound))
|
||||
if _, ok := c.Get("missing"); ok {
|
||||
t.Fatal("404 response was stored")
|
||||
}
|
||||
|
||||
inner.Set("stale-404", dumpedResponse(t, http.StatusMovedPermanently))
|
||||
if _, ok := c.Get("stale-404"); ok {
|
||||
t.Fatal("existing 301 was served")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransportDoesNotCache404(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.Header().Set("Cache-Control", "max-age=60")
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_, _ = io.WriteString(w, "gone")
|
||||
}))
|
||||
t.Cleanup(ts.Close)
|
||||
|
||||
rt := &libcache.Transport{
|
||||
Transport: ts.Client().Transport,
|
||||
Cache: &statusFilterCache{inner: libcache.NewMemoryCache()},
|
||||
SkipServerErrorsFromCache: true,
|
||||
AsyncRevalidateTimeout: time.Second,
|
||||
ShouldCache: shouldCache,
|
||||
DisableWarningHeader: true,
|
||||
}
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := rt.RoundTrip(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("status = %d", resp.StatusCode)
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
|
||||
if got := hits.Load(); got != 2 {
|
||||
t.Fatalf("server hits = %d, want 2 (404 must not be reused from cache)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransportCaches200(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
n := hits.Add(1)
|
||||
w.Header().Set("Cache-Control", "max-age=60")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = fmt.Fprintf(w, "%d", n)
|
||||
}))
|
||||
t.Cleanup(ts.Close)
|
||||
|
||||
rt := &libcache.Transport{
|
||||
Transport: ts.Client().Transport,
|
||||
Cache: &statusFilterCache{inner: libcache.NewMemoryCache()},
|
||||
SkipServerErrorsFromCache: true,
|
||||
AsyncRevalidateTimeout: time.Second,
|
||||
ShouldCache: shouldCache,
|
||||
DisableWarningHeader: true,
|
||||
}
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := rt.RoundTrip(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(body) != "1" {
|
||||
t.Fatalf("body = %q, want cached first response", body)
|
||||
}
|
||||
}
|
||||
|
||||
if got := hits.Load(); got != 1 {
|
||||
t.Fatalf("server hits = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskCacheCloses(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
c, db, err := openDiskCache(dir, defaultMaxCacheBytes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := dumpedResponse(t, http.StatusOK)
|
||||
c.Set("a", body)
|
||||
if _, ok := c.Get("a"); !ok {
|
||||
t.Fatal("value missing before close")
|
||||
}
|
||||
if err := db.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
c2, db2, err := openDiskCache(dir, defaultMaxCacheBytes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db2.Close() })
|
||||
if _, ok := c2.Get("a"); !ok {
|
||||
t.Fatal("value missing after reopen")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskCacheRespectsSizeCap(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
c, db, err := openDiskCache(dir, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
c.Set("a", dumpedResponse(t, http.StatusOK))
|
||||
if _, ok := c.Get("a"); ok {
|
||||
t.Fatal("stored a response over a 1-byte directory cap")
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpcache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
@@ -187,6 +188,8 @@ type Configuration struct {
|
||||
NoTimeout bool
|
||||
// DisableCookie disables cookie reuse for the http client (cookiejar impl)
|
||||
DisableCookie bool
|
||||
// DisableHTTPCache disables HTTP caching for this client instance
|
||||
DisableHTTPCache bool
|
||||
// FollowRedirects specifies the redirects flow
|
||||
RedirectFlow RedirectFlow
|
||||
// Connection defines custom connection configuration
|
||||
@@ -226,6 +229,8 @@ func (c *Configuration) Hash() string {
|
||||
builder.WriteString(strconv.Itoa(int(c.RedirectFlow)))
|
||||
builder.WriteString("r")
|
||||
builder.WriteString(strconv.FormatBool(c.DisableCookie))
|
||||
builder.WriteString("h")
|
||||
builder.WriteString(strconv.FormatBool(c.DisableHTTPCache))
|
||||
builder.WriteString("c")
|
||||
builder.WriteString(strconv.FormatBool(c.Connection != nil))
|
||||
if c.Connection != nil {
|
||||
@@ -247,7 +252,7 @@ func (c *Configuration) Hash() string {
|
||||
|
||||
// HasStandardOptions checks whether the configuration requires custom settings
|
||||
func (c *Configuration) HasStandardOptions() bool {
|
||||
return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout && c.ResponseHeaderTimeout == 0
|
||||
return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout && c.ResponseHeaderTimeout == 0 && !c.DisableHTTPCache
|
||||
}
|
||||
|
||||
// GetRawHTTP returns the rawhttp request client
|
||||
@@ -360,6 +365,9 @@ func wrappedGet(options *types.Options, configuration *Configuration, host strin
|
||||
if configuration.ResponseHeaderTimeout > 0 && configuration.ResponseHeaderTimeout > retryableHttpOptions.Timeout {
|
||||
retryableHttpOptions.Timeout = configuration.ResponseHeaderTimeout
|
||||
}
|
||||
if options.EnableHTTPCache && !configuration.DisableHTTPCache {
|
||||
retryableHttpOptions.WrapTransport = httpcache.NewTransportWrapper()
|
||||
}
|
||||
|
||||
maxIdleConnsPerHost := idleConnPoolSize(options.TemplateThreads, configuration.Threads)
|
||||
maxIdleConns := maxIdleConnsPerHost
|
||||
@@ -463,7 +471,6 @@ func wrappedGet(options *types.Options, configuration *Configuration, host strin
|
||||
|
||||
return &connTrackingTransport{base: transport}, nil
|
||||
}
|
||||
|
||||
redirectFlow := configuration.RedirectFlow
|
||||
maxRedirects := configuration.MaxRedirects
|
||||
|
||||
|
||||
@@ -169,6 +169,8 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
|
||||
opts := &compiler.ExecuteOptions{
|
||||
ExecutionId: request.options.Options.ExecutionId,
|
||||
TimeoutVariants: request.options.Options.GetTimeouts(),
|
||||
ProxyURL: request.options.Options.AliveHttpProxy,
|
||||
CustomHeaders: request.options.Options.CustomHeaders,
|
||||
Source: &request.Init,
|
||||
}
|
||||
// register 'export' function to export variables from init code
|
||||
@@ -405,6 +407,8 @@ func (request *Request) executeWithResults(port string, target *contextargs.Cont
|
||||
&compiler.ExecuteOptions{
|
||||
ExecutionId: requestOptions.Options.ExecutionId,
|
||||
TimeoutVariants: requestOptions.Options.GetTimeouts(),
|
||||
ProxyURL: requestOptions.Options.AliveHttpProxy,
|
||||
CustomHeaders: requestOptions.Options.CustomHeaders,
|
||||
Source: &request.PreCondition,
|
||||
},
|
||||
)
|
||||
@@ -642,6 +646,8 @@ func (request *Request) executeRequestWithPayloads(
|
||||
&compiler.ExecuteOptions{
|
||||
ExecutionId: requestOptions.Options.ExecutionId,
|
||||
TimeoutVariants: requestOptions.Options.GetTimeouts(),
|
||||
ProxyURL: requestOptions.Options.AliveHttpProxy,
|
||||
CustomHeaders: requestOptions.Options.CustomHeaders,
|
||||
Source: &request.Code,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -2,9 +2,9 @@ package sarif
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
@@ -13,11 +13,18 @@ import (
|
||||
"github.com/projectdiscovery/sarif"
|
||||
)
|
||||
|
||||
const (
|
||||
maxDetailValueLength = 400
|
||||
maxExtractedValues = 5
|
||||
maxRuleReferences = 8
|
||||
fallbackTemplateRuleID = "nuclei-template"
|
||||
)
|
||||
|
||||
// Exporter is an exporter for nuclei sarif output format.
|
||||
type Exporter struct {
|
||||
sarif *sarif.Report
|
||||
mutex *sync.Mutex
|
||||
rulemap map[string]*int // contains rule-id && ruleIndex
|
||||
rulemap map[string]int // contains rule-id && ruleIndex
|
||||
rules []sarif.ReportingDescriptor
|
||||
options *Options
|
||||
}
|
||||
@@ -35,7 +42,7 @@ func New(options *Options) (*Exporter, error) {
|
||||
sarif: report,
|
||||
mutex: &sync.Mutex{},
|
||||
rules: []sarif.ReportingDescriptor{},
|
||||
rulemap: map[string]*int{},
|
||||
rulemap: map[string]int{},
|
||||
options: options,
|
||||
}
|
||||
return exporter, nil
|
||||
@@ -56,6 +63,7 @@ func (exporter *Exporter) addToolDetails() {
|
||||
FullName: "Nuclei " + config.Version,
|
||||
SemanticVersion: config.Version,
|
||||
DownloadUri: "https://github.com/projectdiscovery/nuclei/releases",
|
||||
InformationUri: "https://github.com/projectdiscovery/nuclei/releases",
|
||||
Rules: exporter.rules,
|
||||
}
|
||||
exporter.sarif.RegisterTool(driver)
|
||||
@@ -68,9 +76,10 @@ func (exporter *Exporter) addToolDetails() {
|
||||
}
|
||||
|
||||
invocation := sarif.Invocation{
|
||||
CommandLine: os.Args[0],
|
||||
Arguments: os.Args[1:],
|
||||
ResponseFiles: []sarif.ArtifactLocation{reportLocation},
|
||||
CommandLine: os.Args[0],
|
||||
Arguments: os.Args[1:],
|
||||
ResponseFiles: []sarif.ArtifactLocation{reportLocation},
|
||||
ExecutionSuccessful: true,
|
||||
}
|
||||
exporter.sarif.RegisterToolInvocation(invocation)
|
||||
}
|
||||
@@ -99,71 +108,67 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
|
||||
defer exporter.mutex.Unlock()
|
||||
|
||||
severity := event.Info.SeverityHolder.Severity.String()
|
||||
resultHeader := fmt.Sprintf("%v (%v) found on %v", event.Info.Name, event.TemplateID, event.Host)
|
||||
ruleID := normalizeValue(event.TemplateID)
|
||||
if ruleID == "" {
|
||||
ruleID = fallbackTemplateRuleID
|
||||
}
|
||||
ruleName := normalizeValue(event.Info.Name)
|
||||
if ruleName == "" {
|
||||
ruleName = ruleID
|
||||
}
|
||||
resultTarget := getResultTarget(event)
|
||||
resultHeader := fmt.Sprintf("%s (%s) found on %s", ruleName, ruleID, event.URL)
|
||||
resultLevel, vulnRating := exporter.getSeverity(severity)
|
||||
if classification := event.Info.Classification; classification != nil && classification.CVSSScore > 0 {
|
||||
vulnRating = fmt.Sprintf("%.1f", classification.CVSSScore)
|
||||
}
|
||||
|
||||
// Extra metadata if generated sarif is uploaded to GitHub security page
|
||||
ghMeta := map[string]interface{}{}
|
||||
ghMeta["tags"] = []string{"security"}
|
||||
ghMeta["security-severity"] = vulnRating
|
||||
ghMeta := map[string]interface{}{
|
||||
"tags": buildRuleTags(event),
|
||||
"security-severity": vulnRating,
|
||||
}
|
||||
|
||||
// rule contain details of template
|
||||
rule := sarif.ReportingDescriptor{
|
||||
Id: event.TemplateID,
|
||||
Name: event.Info.Name,
|
||||
Id: ruleID,
|
||||
Name: ruleName,
|
||||
HelpUri: normalizeValue(event.TemplateURL),
|
||||
ShortDescription: &sarif.MultiformatMessageString{
|
||||
Text: ruleName,
|
||||
},
|
||||
FullDescription: &sarif.MultiformatMessageString{
|
||||
// Points to template URL
|
||||
Text: event.Info.Description + "\nMore details at\n" + event.TemplateURL + "\n",
|
||||
Text: buildRuleDescription(event, ruleName),
|
||||
},
|
||||
Help: &sarif.MultiformatMessageString{
|
||||
Text: buildRuleHelpText(event),
|
||||
Markdown: buildRuleHelpMarkdown(event, ruleName),
|
||||
},
|
||||
Properties: ghMeta,
|
||||
}
|
||||
|
||||
// GitHub Uses ShortDescription as title
|
||||
if event.Info.Description != "" {
|
||||
rule.ShortDescription = &sarif.MultiformatMessageString{
|
||||
Text: resultHeader,
|
||||
}
|
||||
}
|
||||
|
||||
// If rule is added
|
||||
ruleIndex := int(math.Max(0, float64(len(exporter.rules)-1)))
|
||||
if exporter.rulemap[rule.Id] == nil {
|
||||
exporter.rulemap[rule.Id] = &ruleIndex
|
||||
// Add rule once and keep stable rule indices.
|
||||
ruleIndex, found := exporter.rulemap[ruleID]
|
||||
if !found {
|
||||
ruleIndex = len(exporter.rules)
|
||||
exporter.rulemap[ruleID] = ruleIndex
|
||||
exporter.rules = append(exporter.rules, rule)
|
||||
} else {
|
||||
ruleIndex = *exporter.rulemap[rule.Id]
|
||||
}
|
||||
|
||||
// vulnerability target/location
|
||||
location := sarif.Location{
|
||||
Message: &sarif.Message{
|
||||
Text: path.Join(event.Host, event.Path),
|
||||
},
|
||||
PhysicalLocation: sarif.PhysicalLocation{
|
||||
ArtifactLocation: sarif.ArtifactLocation{
|
||||
// GitHub only accepts file:// protocol and local & relative files only
|
||||
// to avoid errors // is used which also translates to file according to specification
|
||||
Uri: "/" + event.Path,
|
||||
Description: &sarif.Message{
|
||||
Text: path.Join(event.Host, event.Path),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
location := buildLocation(event, resultTarget)
|
||||
|
||||
// vulnerability report/result
|
||||
result := &sarif.Result{
|
||||
RuleId: rule.Id,
|
||||
RuleId: ruleID,
|
||||
RuleIndex: ruleIndex,
|
||||
Level: resultLevel,
|
||||
Kind: sarif.Open,
|
||||
Message: &sarif.Message{
|
||||
Text: resultHeader,
|
||||
},
|
||||
Message: buildResultMessage(event, resultHeader, ruleID),
|
||||
Locations: []sarif.Location{location},
|
||||
Rule: sarif.ReportingDescriptorReference{
|
||||
Id: rule.Id,
|
||||
Id: ruleID,
|
||||
},
|
||||
Properties: buildResultProperties(event, resultTarget),
|
||||
}
|
||||
|
||||
exporter.sarif.RegisterResult(*result)
|
||||
@@ -184,7 +189,10 @@ func (exporter *Exporter) Close() error {
|
||||
// links results and rules/templates
|
||||
exporter.addToolDetails()
|
||||
|
||||
bin, err := exporter.sarif.Export()
|
||||
bin, err := exporter.sarif.ExportWithOptions(
|
||||
sarif.WithNormalization(true),
|
||||
sarif.WithEmptyObjectPruning(true),
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to generate sarif report")
|
||||
}
|
||||
@@ -194,3 +202,370 @@ func (exporter *Exporter) Close() error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildRuleDescription(event *output.ResultEvent, ruleName string) string {
|
||||
description := normalizeValue(event.Info.Description)
|
||||
if description == "" {
|
||||
description = ruleName
|
||||
}
|
||||
if templateURL := normalizeValue(event.TemplateURL); templateURL != "" {
|
||||
description += "\nMore details at\n" + templateURL
|
||||
}
|
||||
return description
|
||||
}
|
||||
|
||||
func buildRuleHelpText(event *output.ResultEvent) string {
|
||||
sections := []string{}
|
||||
|
||||
if description := normalizeValue(event.Info.Description); description != "" {
|
||||
sections = append(sections, description)
|
||||
}
|
||||
if impact := normalizeValue(event.Info.Impact); impact != "" {
|
||||
sections = append(sections, "Impact: "+impact)
|
||||
}
|
||||
if remediation := normalizeValue(event.Info.Remediation); remediation != "" {
|
||||
sections = append(sections, "Remediation: "+remediation)
|
||||
}
|
||||
if templatePath := normalizeValue(firstNonEmpty(event.Template, event.TemplatePath)); templatePath != "" {
|
||||
sections = append(sections, "Nuclei Template: "+templatePath)
|
||||
}
|
||||
if refs := getReferences(event); len(refs) > 0 {
|
||||
sections = append(sections, "References: "+strings.Join(refs, ", "))
|
||||
}
|
||||
if templateURL := normalizeValue(event.TemplateURL); templateURL != "" {
|
||||
sections = append(sections, "More details: "+templateURL)
|
||||
}
|
||||
|
||||
if len(sections) == 0 {
|
||||
return "No additional template metadata was provided by the scanner."
|
||||
}
|
||||
|
||||
return strings.Join(sections, "\n")
|
||||
}
|
||||
|
||||
func buildRuleHelpMarkdown(event *output.ResultEvent, ruleName string) string {
|
||||
lines := []string{fmt.Sprintf("### %s", ruleName), ""}
|
||||
|
||||
if description := normalizeValue(event.Info.Description); description != "" {
|
||||
lines = append(lines, description, "")
|
||||
}
|
||||
if impact := normalizeValue(event.Info.Impact); impact != "" {
|
||||
lines = append(lines, fmt.Sprintf("- **Impact:** %s", impact))
|
||||
}
|
||||
if remediation := normalizeValue(event.Info.Remediation); remediation != "" {
|
||||
lines = append(lines, fmt.Sprintf("- **Remediation:** %s", remediation))
|
||||
}
|
||||
if templatePath := normalizeValue(firstNonEmpty(event.Template, event.TemplatePath)); templatePath != "" {
|
||||
lines = append(lines, fmt.Sprintf("- **Nuclei Template:** %s", templatePath))
|
||||
}
|
||||
|
||||
if refs := getReferences(event); len(refs) > 0 {
|
||||
lines = append(lines, "", "**References**")
|
||||
for _, ref := range refs {
|
||||
lines = append(lines, "- "+ref)
|
||||
}
|
||||
}
|
||||
|
||||
if templateURL := normalizeValue(event.TemplateURL); templateURL != "" {
|
||||
lines = append(lines, "", "More details: "+templateURL)
|
||||
}
|
||||
|
||||
return strings.TrimSpace(strings.Join(lines, "\n"))
|
||||
}
|
||||
|
||||
func buildResultMessage(event *output.ResultEvent, resultHeader, ruleID string) *sarif.Message {
|
||||
detailLines := []string{}
|
||||
target := getResultTarget(event)
|
||||
addDetail(&detailLines, "Target", target)
|
||||
if matched := normalizeAndTruncate(event.Matched, maxDetailValueLength); matched != "" && matched != target {
|
||||
addDetail(&detailLines, "Matched At", matched)
|
||||
}
|
||||
if baseURL := normalizeAndTruncate(event.URL, maxDetailValueLength); baseURL != "" {
|
||||
addDetail(&detailLines, "Base URL", baseURL)
|
||||
}
|
||||
if host := normalizeAndTruncate(event.Host, maxDetailValueLength); host != "" && host != target {
|
||||
addDetail(&detailLines, "Host", host)
|
||||
}
|
||||
addDetail(&detailLines, "IP", event.IP)
|
||||
addDetail(&detailLines, "Protocol", event.Type)
|
||||
addDetail(&detailLines, "Matcher", event.MatcherName)
|
||||
addDetail(&detailLines, "Extractor", event.ExtractorName)
|
||||
addDetail(&detailLines, "Nuclei Template", ruleID)
|
||||
if templatePath := normalizeAndTruncate(firstNonEmpty(event.Template, event.TemplatePath), maxDetailValueLength); templatePath != "" {
|
||||
addDetail(&detailLines, "Nuclei Template Path", templatePath)
|
||||
}
|
||||
if extracted := formatExtractedResults(event.ExtractedResults); extracted != "" {
|
||||
addDetail(&detailLines, "Extracted Results", extracted)
|
||||
}
|
||||
|
||||
reproduction := normalizeAndTruncate(event.CURLCommand, maxDetailValueLength*2)
|
||||
|
||||
text := resultHeader
|
||||
if len(detailLines) > 0 {
|
||||
text += "\n" + strings.Join(detailLines, "\n")
|
||||
}
|
||||
if reproduction != "" {
|
||||
text += "\nReproduce:\n"
|
||||
text += "```bash\n" + reproduction + "\n```"
|
||||
}
|
||||
|
||||
markdownLines := []string{resultHeader}
|
||||
if len(detailLines) > 0 {
|
||||
markdownLines = append(markdownLines, "", "\n**Triage Details**:")
|
||||
for _, detail := range detailLines {
|
||||
parts := strings.SplitN(detail, ": ", 2)
|
||||
if len(parts) == 2 {
|
||||
markdownLines = append(markdownLines, fmt.Sprintf("- **%s:** %s", parts[0], parts[1]))
|
||||
continue
|
||||
}
|
||||
markdownLines = append(markdownLines, "- "+detail)
|
||||
}
|
||||
}
|
||||
if reproduction != "" {
|
||||
markdownLines = append(markdownLines, "", "\n**Reproduce**:", "```bash", reproduction, "```")
|
||||
}
|
||||
|
||||
return &sarif.Message{
|
||||
Text: text,
|
||||
Markdown: strings.Join(markdownLines, "\n"),
|
||||
}
|
||||
}
|
||||
|
||||
func buildResultProperties(event *output.ResultEvent, target string) map[string]interface{} {
|
||||
properties := map[string]interface{}{}
|
||||
|
||||
if normalizedTarget := normalizeAndTruncate(target, maxDetailValueLength); normalizedTarget != "" {
|
||||
properties["target"] = normalizedTarget
|
||||
}
|
||||
if matched := normalizeAndTruncate(event.Matched, maxDetailValueLength); matched != "" {
|
||||
properties["matched-at"] = matched
|
||||
}
|
||||
if matcher := normalizeAndTruncate(event.MatcherName, maxDetailValueLength); matcher != "" {
|
||||
properties["matcher"] = matcher
|
||||
}
|
||||
if extractor := normalizeAndTruncate(event.ExtractorName, maxDetailValueLength); extractor != "" {
|
||||
properties["extractor"] = extractor
|
||||
}
|
||||
if len(event.ExtractedResults) > 0 {
|
||||
trimmed := make([]string, 0, maxExtractedValues)
|
||||
for i, value := range event.ExtractedResults {
|
||||
if i >= maxExtractedValues {
|
||||
break
|
||||
}
|
||||
if normalized := normalizeAndTruncate(value, maxDetailValueLength); normalized != "" {
|
||||
trimmed = append(trimmed, normalized)
|
||||
}
|
||||
}
|
||||
if len(trimmed) > 0 {
|
||||
properties["extracted-results"] = trimmed
|
||||
}
|
||||
}
|
||||
if templatePath := normalizeAndTruncate(firstNonEmpty(event.Template, event.TemplatePath), maxDetailValueLength); templatePath != "" {
|
||||
properties["template-path"] = templatePath
|
||||
}
|
||||
|
||||
if len(properties) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return properties
|
||||
}
|
||||
|
||||
func buildLocation(event *output.ResultEvent, resultTarget string) sarif.Location {
|
||||
location := sarif.Location{
|
||||
Message: &sarif.Message{
|
||||
Text: resultTarget,
|
||||
},
|
||||
PhysicalLocation: sarif.PhysicalLocation{
|
||||
ArtifactLocation: sarif.ArtifactLocation{
|
||||
Uri: buildArtifactURI(event),
|
||||
Description: &sarif.Message{
|
||||
Text: resultTarget,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if line := firstPositiveLine(event.Lines); line > 0 {
|
||||
location.PhysicalLocation.Region = &sarif.Region{StartLine: line}
|
||||
}
|
||||
|
||||
return location
|
||||
}
|
||||
|
||||
func buildArtifactURI(event *output.ResultEvent) string {
|
||||
if template := normalizePath(firstNonEmpty(event.Template, event.TemplatePath)); template != "" {
|
||||
if idx := strings.Index(template, "/nuclei-templates/"); idx >= 0 {
|
||||
return strings.TrimLeft(template[idx+len("/nuclei-templates/"):], "/")
|
||||
}
|
||||
return strings.TrimLeft(template, "/")
|
||||
}
|
||||
|
||||
if targetPath := normalizePath(event.Path); targetPath != "" && targetPath != "/" && targetPath != "." {
|
||||
return strings.TrimLeft(targetPath, "/")
|
||||
}
|
||||
|
||||
if templateID := normalizeValue(event.TemplateID); templateID != "" {
|
||||
return path.Join("nuclei-results", templateID+".txt")
|
||||
}
|
||||
|
||||
return "nuclei-results/result.txt"
|
||||
}
|
||||
|
||||
func buildRuleTags(event *output.ResultEvent) []string {
|
||||
tags := []string{"security"}
|
||||
for _, tag := range event.Info.Tags.ToSlice() {
|
||||
normalizedTag := normalizeValue(tag)
|
||||
if normalizedTag == "" {
|
||||
continue
|
||||
}
|
||||
tags = appendIfMissing(tags, normalizedTag)
|
||||
}
|
||||
|
||||
if classification := event.Info.Classification; classification != nil {
|
||||
for _, cwe := range classification.CWEID.ToSlice() {
|
||||
if cweTag := buildCWETag(cwe); cweTag != "" {
|
||||
tags = appendIfMissing(tags, cweTag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return tags
|
||||
}
|
||||
|
||||
func getReferences(event *output.ResultEvent) []string {
|
||||
if event.Info.Reference == nil {
|
||||
return nil
|
||||
}
|
||||
references := event.Info.Reference.ToSlice()
|
||||
if len(references) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
output := make([]string, 0, min(len(references), maxRuleReferences))
|
||||
for i, reference := range references {
|
||||
if i >= maxRuleReferences {
|
||||
break
|
||||
}
|
||||
if normalizedRef := normalizeAndTruncate(reference, maxDetailValueLength); normalizedRef != "" {
|
||||
output = append(output, normalizedRef)
|
||||
}
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
func getResultTarget(event *output.ResultEvent) string {
|
||||
for _, value := range []string{event.Matched, event.URL, event.Host, event.Path} {
|
||||
if normalized := normalizeAndTruncate(value, maxDetailValueLength); normalized != "" {
|
||||
return normalized
|
||||
}
|
||||
}
|
||||
return "unknown target"
|
||||
}
|
||||
|
||||
func formatExtractedResults(values []string) string {
|
||||
if len(values) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
trimmed := make([]string, 0, maxExtractedValues)
|
||||
for i, value := range values {
|
||||
if i >= maxExtractedValues {
|
||||
break
|
||||
}
|
||||
if normalized := normalizeAndTruncate(value, maxDetailValueLength); normalized != "" {
|
||||
trimmed = append(trimmed, normalized)
|
||||
}
|
||||
}
|
||||
|
||||
if len(trimmed) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
if len(values) > len(trimmed) {
|
||||
trimmed = append(trimmed, fmt.Sprintf("... and %d more", len(values)-len(trimmed)))
|
||||
}
|
||||
|
||||
return strings.Join(trimmed, ", ")
|
||||
}
|
||||
|
||||
func firstPositiveLine(lines []int) int {
|
||||
for _, line := range lines {
|
||||
if line > 0 {
|
||||
return line
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func addDetail(lines *[]string, label, value string) {
|
||||
normalizedValue := normalizeAndTruncate(value, maxDetailValueLength)
|
||||
if normalizedValue == "" {
|
||||
return
|
||||
}
|
||||
*lines = append(*lines, fmt.Sprintf("%s: %s", label, normalizedValue))
|
||||
}
|
||||
|
||||
func normalizeAndTruncate(value string, limit int) string {
|
||||
normalized := normalizeValue(value)
|
||||
if normalized == "" {
|
||||
return ""
|
||||
}
|
||||
if limit > 0 && len(normalized) > limit {
|
||||
return normalized[:limit-3] + "..."
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
func normalizeValue(value string) string {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(strings.Fields(trimmed), " ")
|
||||
}
|
||||
|
||||
func normalizePath(value string) string {
|
||||
normalized := strings.TrimSpace(strings.ReplaceAll(value, "\\", "/"))
|
||||
if normalized == "" {
|
||||
return ""
|
||||
}
|
||||
return path.Clean(normalized)
|
||||
}
|
||||
|
||||
func appendIfMissing(values []string, candidate string) []string {
|
||||
for _, existing := range values {
|
||||
if existing == candidate {
|
||||
return values
|
||||
}
|
||||
}
|
||||
return append(values, candidate)
|
||||
}
|
||||
|
||||
func buildCWETag(value string) string {
|
||||
normalized := strings.ToLower(strings.TrimSpace(value))
|
||||
if normalized == "" {
|
||||
return ""
|
||||
}
|
||||
normalized = strings.TrimPrefix(normalized, "cwe-")
|
||||
digitsOnly := strings.Builder{}
|
||||
for _, char := range normalized {
|
||||
if char >= '0' && char <= '9' {
|
||||
digitsOnly.WriteRune(char)
|
||||
}
|
||||
}
|
||||
if digitsOnly.Len() == 0 {
|
||||
return ""
|
||||
}
|
||||
return "external/cwe/cwe-" + digitsOnly.String()
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,479 @@
|
||||
package sarif
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
upstream "github.com/projectdiscovery/sarif"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// makeEvent returns a fully populated ResultEvent suitable for the integration tests.
|
||||
func makeEvent() *output.ResultEvent {
|
||||
return &output.ResultEvent{
|
||||
TemplateID: "test-template",
|
||||
TemplateURL: "https://templates.example.com/test-template.yaml",
|
||||
Template: "/tmp/nuclei-templates/cves/test-template.yaml",
|
||||
TemplatePath: "/tmp/nuclei-templates/cves/test-template.yaml",
|
||||
Type: "http",
|
||||
Host: "example.com",
|
||||
URL: "https://example.com",
|
||||
Path: "/admin",
|
||||
Matched: "https://example.com/admin",
|
||||
IP: "93.184.216.34",
|
||||
MatcherName: "status-200",
|
||||
ExtractorName: "version",
|
||||
ExtractedResults: []string{"v1.2.3", "v1.2.4"},
|
||||
CURLCommand: "curl -X GET https://example.com/admin",
|
||||
Lines: []int{0, 42},
|
||||
Timestamp: time.Now(),
|
||||
Info: model.Info{
|
||||
Name: "Test Finding",
|
||||
Authors: stringslice.StringSlice{Value: "tester"},
|
||||
Description: "A test vulnerability description.",
|
||||
Impact: "Severe impact",
|
||||
Remediation: "Update to latest version",
|
||||
Tags: stringslice.StringSlice{Value: []string{"test", "rce"}},
|
||||
Reference: stringslice.NewRawStringSlice([]string{"https://ref.example.com/a", "https://ref.example.com/b"}),
|
||||
SeverityHolder: severity.Holder{Severity: severity.High},
|
||||
Classification: &model.Classification{
|
||||
CWEID: stringslice.StringSlice{Value: []string{"CWE-79"}},
|
||||
CVSSScore: 8.7,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// helper-level unit tests
|
||||
|
||||
func TestNormalizeValue(t *testing.T) {
|
||||
tests := []struct {
|
||||
in, want string
|
||||
}{
|
||||
{"", ""},
|
||||
{" \t\n ", ""},
|
||||
{" hello world ", "hello world"},
|
||||
{"line1\nline2\tline3", "line1 line2 line3"},
|
||||
{"single", "single"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
require.Equal(t, tt.want, normalizeValue(tt.in), "input=%q", tt.in)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeAndTruncate(t *testing.T) {
|
||||
require.Equal(t, "", normalizeAndTruncate("", 100))
|
||||
require.Equal(t, "hello", normalizeAndTruncate(" hello ", 100))
|
||||
require.Equal(t, "hello", normalizeAndTruncate("hello", 0))
|
||||
|
||||
long := strings.Repeat("x", 500)
|
||||
got := normalizeAndTruncate(long, 100)
|
||||
require.Len(t, got, 100)
|
||||
require.True(t, strings.HasSuffix(got, "..."))
|
||||
require.Equal(t, strings.Repeat("x", 97)+"...", got)
|
||||
}
|
||||
|
||||
func TestNormalizePath(t *testing.T) {
|
||||
require.Equal(t, "", normalizePath(""))
|
||||
require.Equal(t, "", normalizePath(" "))
|
||||
require.Equal(t, "a/b/c.yaml", normalizePath("a\\b\\c.yaml"))
|
||||
require.Equal(t, "/a/b", normalizePath("/a/./b"))
|
||||
require.Equal(t, "a/b", normalizePath("a/b/"))
|
||||
}
|
||||
|
||||
func TestAppendIfMissing(t *testing.T) {
|
||||
out := appendIfMissing([]string{"a", "b"}, "a")
|
||||
require.Equal(t, []string{"a", "b"}, out)
|
||||
out = appendIfMissing([]string{"a", "b"}, "c")
|
||||
require.Equal(t, []string{"a", "b", "c"}, out)
|
||||
}
|
||||
|
||||
func TestBuildCWETag(t *testing.T) {
|
||||
tests := []struct {
|
||||
in, want string
|
||||
}{
|
||||
{"", ""},
|
||||
{" ", ""},
|
||||
{"abc", ""},
|
||||
{"CWE-79", "external/cwe/cwe-79"},
|
||||
{"cwe-22", "external/cwe/cwe-22"},
|
||||
{"79", "external/cwe/cwe-79"},
|
||||
{" CWE-89 ", "external/cwe/cwe-89"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
require.Equal(t, tt.want, buildCWETag(tt.in), "input=%q", tt.in)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirstNonEmpty(t *testing.T) {
|
||||
require.Equal(t, "", firstNonEmpty())
|
||||
require.Equal(t, "", firstNonEmpty("", " "))
|
||||
require.Equal(t, "first", firstNonEmpty("first", "second"))
|
||||
require.Equal(t, "second", firstNonEmpty("", "second", "third"))
|
||||
}
|
||||
|
||||
func TestFirstPositiveLine(t *testing.T) {
|
||||
require.Equal(t, 0, firstPositiveLine(nil))
|
||||
require.Equal(t, 0, firstPositiveLine([]int{}))
|
||||
require.Equal(t, 0, firstPositiveLine([]int{0, -1, 0}))
|
||||
require.Equal(t, 5, firstPositiveLine([]int{0, 0, 5, 7}))
|
||||
require.Equal(t, 7, firstPositiveLine([]int{7}))
|
||||
}
|
||||
|
||||
func TestFormatExtractedResults(t *testing.T) {
|
||||
require.Equal(t, "", formatExtractedResults(nil))
|
||||
require.Equal(t, "", formatExtractedResults([]string{}))
|
||||
require.Equal(t, "", formatExtractedResults([]string{"", " "}))
|
||||
|
||||
require.Equal(t, "a, b", formatExtractedResults([]string{"a", "b"}))
|
||||
|
||||
values := []string{"a", "b", "c", "d", "e", "f", "g"}
|
||||
got := formatExtractedResults(values)
|
||||
require.Contains(t, got, "a, b, c, d, e")
|
||||
require.Contains(t, got, "and 2 more")
|
||||
}
|
||||
|
||||
func TestGetResultTarget(t *testing.T) {
|
||||
require.Equal(t, "https://example.com/match", getResultTarget(&output.ResultEvent{
|
||||
Matched: "https://example.com/match",
|
||||
URL: "https://example.com",
|
||||
}))
|
||||
require.Equal(t, "https://example.com", getResultTarget(&output.ResultEvent{
|
||||
URL: "https://example.com",
|
||||
Host: "example.com",
|
||||
}))
|
||||
require.Equal(t, "example.com", getResultTarget(&output.ResultEvent{Host: "example.com"}))
|
||||
require.Equal(t, "/path", getResultTarget(&output.ResultEvent{Path: "/path"}))
|
||||
require.Equal(t, "unknown target", getResultTarget(&output.ResultEvent{}))
|
||||
}
|
||||
|
||||
func TestBuildArtifactURI(t *testing.T) {
|
||||
require.Equal(t,
|
||||
"cves/test.yaml",
|
||||
buildArtifactURI(&output.ResultEvent{Template: "/home/user/nuclei-templates/cves/test.yaml"}),
|
||||
)
|
||||
require.Equal(t,
|
||||
"cves/test.yaml",
|
||||
buildArtifactURI(&output.ResultEvent{Template: "C:\\src\\nuclei-templates\\cves\\test.yaml"}),
|
||||
)
|
||||
require.Equal(t,
|
||||
"local/template.yaml",
|
||||
buildArtifactURI(&output.ResultEvent{Template: "/local/template.yaml"}),
|
||||
)
|
||||
require.Equal(t,
|
||||
"some/file",
|
||||
buildArtifactURI(&output.ResultEvent{Path: "/some/file"}),
|
||||
)
|
||||
require.Equal(t,
|
||||
"nuclei-results/my-id.txt",
|
||||
buildArtifactURI(&output.ResultEvent{TemplateID: "my-id"}),
|
||||
)
|
||||
require.Equal(t,
|
||||
"nuclei-results/result.txt",
|
||||
buildArtifactURI(&output.ResultEvent{}),
|
||||
)
|
||||
}
|
||||
|
||||
func TestBuildRuleTags(t *testing.T) {
|
||||
event := &output.ResultEvent{
|
||||
Info: model.Info{
|
||||
Tags: stringslice.StringSlice{Value: []string{"rce", "rce", " "}},
|
||||
Classification: &model.Classification{
|
||||
CWEID: stringslice.StringSlice{Value: []string{"CWE-79"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
tags := buildRuleTags(event)
|
||||
require.Equal(t, []string{"security", "rce", "external/cwe/cwe-79"}, tags)
|
||||
}
|
||||
|
||||
func TestGetReferences(t *testing.T) {
|
||||
require.Nil(t, getReferences(&output.ResultEvent{}))
|
||||
|
||||
refs := make([]string, 0, 12)
|
||||
for i := 0; i < 12; i++ {
|
||||
refs = append(refs, "https://example.com/r")
|
||||
}
|
||||
event := &output.ResultEvent{
|
||||
Info: model.Info{Reference: stringslice.NewRawStringSlice(refs)},
|
||||
}
|
||||
got := getReferences(event)
|
||||
require.Len(t, got, maxRuleReferences)
|
||||
}
|
||||
|
||||
func TestAddDetail(t *testing.T) {
|
||||
var lines []string
|
||||
addDetail(&lines, "Label", "")
|
||||
require.Empty(t, lines)
|
||||
addDetail(&lines, "Label", "value")
|
||||
require.Equal(t, []string{"Label: value"}, lines)
|
||||
}
|
||||
|
||||
func TestBuildLocationRegion(t *testing.T) {
|
||||
loc := buildLocation(&output.ResultEvent{Lines: []int{0, 42}}, "target")
|
||||
require.NotNil(t, loc.PhysicalLocation.Region)
|
||||
require.Equal(t, 42, loc.PhysicalLocation.Region.StartLine)
|
||||
|
||||
loc = buildLocation(&output.ResultEvent{Lines: []int{0, 0}}, "target")
|
||||
require.Nil(t, loc.PhysicalLocation.Region)
|
||||
}
|
||||
|
||||
func TestBuildRuleDescription(t *testing.T) {
|
||||
got := buildRuleDescription(&output.ResultEvent{
|
||||
TemplateURL: "https://example.com/t.yaml",
|
||||
Info: model.Info{Description: "desc"},
|
||||
}, "fallback")
|
||||
require.Equal(t, "desc\nMore details at\nhttps://example.com/t.yaml", got)
|
||||
|
||||
got = buildRuleDescription(&output.ResultEvent{}, "fallback name")
|
||||
require.Equal(t, "fallback name", got)
|
||||
}
|
||||
|
||||
func TestBuildRuleHelpText(t *testing.T) {
|
||||
got := buildRuleHelpText(&output.ResultEvent{})
|
||||
require.Equal(t, "No additional template metadata was provided by the scanner.", got)
|
||||
|
||||
event := makeEvent()
|
||||
got = buildRuleHelpText(event)
|
||||
require.Contains(t, got, "Impact: Severe impact")
|
||||
require.Contains(t, got, "Remediation: Update to latest version")
|
||||
require.Contains(t, got, "References:")
|
||||
require.Contains(t, got, "More details:")
|
||||
}
|
||||
|
||||
func TestBuildRuleHelpMarkdown(t *testing.T) {
|
||||
got := buildRuleHelpMarkdown(makeEvent(), "Test Finding")
|
||||
require.Contains(t, got, "### Test Finding")
|
||||
require.Contains(t, got, "- **Impact:** Severe impact")
|
||||
require.Contains(t, got, "- **Remediation:** Update to latest version")
|
||||
require.Contains(t, got, "**References**")
|
||||
}
|
||||
|
||||
func TestBuildResultMessage(t *testing.T) {
|
||||
msg := buildResultMessage(makeEvent(), "header", "test-template")
|
||||
require.NotNil(t, msg)
|
||||
require.Contains(t, msg.Text, "header")
|
||||
require.Contains(t, msg.Text, "Target: https://example.com/admin")
|
||||
require.Contains(t, msg.Text, "IP: 93.184.216.34")
|
||||
require.Contains(t, msg.Text, "Matcher: status-200")
|
||||
require.Contains(t, msg.Text, "Extractor: version")
|
||||
require.Contains(t, msg.Text, "Reproduce:")
|
||||
require.Contains(t, msg.Text, "```bash")
|
||||
require.Contains(t, msg.Markdown, "**Triage Details**")
|
||||
require.Contains(t, msg.Markdown, "- **Target:** https://example.com/admin")
|
||||
}
|
||||
|
||||
func TestBuildResultProperties(t *testing.T) {
|
||||
props := buildResultProperties(makeEvent(), "https://example.com/admin")
|
||||
require.NotNil(t, props)
|
||||
require.Equal(t, "https://example.com/admin", props["target"])
|
||||
require.Equal(t, "status-200", props["matcher"])
|
||||
require.Equal(t, "version", props["extractor"])
|
||||
require.Contains(t, props["template-path"], "test-template.yaml")
|
||||
|
||||
extracted, ok := props["extracted-results"].([]string)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, []string{"v1.2.3", "v1.2.4"}, extracted)
|
||||
|
||||
require.Nil(t, buildResultProperties(&output.ResultEvent{}, ""))
|
||||
}
|
||||
|
||||
// severity table
|
||||
func TestGetSeverity(t *testing.T) {
|
||||
exp, err := New(&Options{File: "x.sarif"})
|
||||
require.NoError(t, err)
|
||||
|
||||
cases := []struct {
|
||||
sev string
|
||||
level upstream.Level
|
||||
score string
|
||||
}{
|
||||
{"critical", upstream.Error, "9.4"},
|
||||
{"high", upstream.Error, "8"},
|
||||
{"medium", upstream.Note, "5"},
|
||||
{"low", upstream.Note, "2"},
|
||||
{"info", upstream.None, "1"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
gotLevel, gotScore := exp.getSeverity(c.sev)
|
||||
require.Equal(t, c.level, gotLevel, "sev=%s", c.sev)
|
||||
require.Equal(t, c.score, gotScore, "sev=%s", c.sev)
|
||||
}
|
||||
}
|
||||
|
||||
// integration: Export + Close round-trip
|
||||
|
||||
func TestNew_InitializesEmpty(t *testing.T) {
|
||||
exp, err := New(&Options{File: "x.sarif"})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, exp.sarif)
|
||||
require.NotNil(t, exp.mutex)
|
||||
require.Empty(t, exp.rules)
|
||||
require.Empty(t, exp.rulemap)
|
||||
}
|
||||
|
||||
func TestClose_NoEventsWritesNoFile(t *testing.T) {
|
||||
out := filepath.Join(t.TempDir(), "report.sarif")
|
||||
exp, err := New(&Options{File: out})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, exp.Close())
|
||||
_, statErr := os.Stat(out)
|
||||
require.True(t, os.IsNotExist(statErr))
|
||||
}
|
||||
|
||||
func TestExport_Close_RoundTrip(t *testing.T) {
|
||||
out := filepath.Join(t.TempDir(), "report.sarif")
|
||||
exp, err := New(&Options{File: out})
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, exp.Export(makeEvent()))
|
||||
require.NoError(t, exp.Close())
|
||||
|
||||
bin, err := os.ReadFile(out)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, bin)
|
||||
|
||||
var log upstream.SarifLog
|
||||
require.NoError(t, json.Unmarshal(bin, &log), "output is not valid JSON / SarifLog")
|
||||
require.Equal(t, "2.1.0", log.Version)
|
||||
require.NotEmpty(t, log.Schema)
|
||||
require.Len(t, log.Runs, 1)
|
||||
|
||||
run := log.Runs[0]
|
||||
require.Equal(t, "Nuclei", run.Tool.Driver.Name)
|
||||
require.Equal(t, "ProjectDiscovery", run.Tool.Driver.Organization)
|
||||
require.NotEmpty(t, run.Tool.Driver.DownloadUri)
|
||||
require.NotEmpty(t, run.Tool.Driver.InformationUri)
|
||||
require.Len(t, run.Tool.Driver.Rules, 1)
|
||||
|
||||
rule := run.Tool.Driver.Rules[0]
|
||||
require.Equal(t, "test-template", rule.Id)
|
||||
require.Equal(t, "Test Finding", rule.Name)
|
||||
require.Equal(t, "https://templates.example.com/test-template.yaml", rule.HelpUri)
|
||||
require.NotNil(t, rule.ShortDescription)
|
||||
require.NotNil(t, rule.FullDescription)
|
||||
require.NotNil(t, rule.Help)
|
||||
|
||||
require.Len(t, run.Result, 1)
|
||||
res := run.Result[0]
|
||||
require.Equal(t, "test-template", res.RuleId)
|
||||
require.Equal(t, 0, res.RuleIndex)
|
||||
require.Equal(t, upstream.Error, res.Level)
|
||||
require.NotNil(t, res.Message)
|
||||
require.Contains(t, res.Message.Text, "Test Finding")
|
||||
require.Contains(t, res.Message.Text, "test-template")
|
||||
require.Len(t, res.Locations, 1)
|
||||
|
||||
require.Len(t, run.Invocations, 1)
|
||||
require.True(t, run.Invocations[0].ExecutionSuccessful)
|
||||
}
|
||||
|
||||
func TestExport_FallbackRuleID(t *testing.T) {
|
||||
out := filepath.Join(t.TempDir(), "report.sarif")
|
||||
exp, err := New(&Options{File: out})
|
||||
require.NoError(t, err)
|
||||
|
||||
event := makeEvent()
|
||||
event.TemplateID = ""
|
||||
event.Info.Name = ""
|
||||
require.NoError(t, exp.Export(event))
|
||||
require.NoError(t, exp.Close())
|
||||
|
||||
require.Len(t, exp.rules, 1)
|
||||
require.Equal(t, fallbackTemplateRuleID, exp.rules[0].Id)
|
||||
require.Equal(t, fallbackTemplateRuleID, exp.rules[0].Name)
|
||||
}
|
||||
|
||||
func TestExport_StableRuleIndices(t *testing.T) {
|
||||
out := filepath.Join(t.TempDir(), "report.sarif")
|
||||
exp, err := New(&Options{File: out})
|
||||
require.NoError(t, err)
|
||||
|
||||
a := makeEvent()
|
||||
a.TemplateID = "rule-a"
|
||||
b := makeEvent()
|
||||
b.TemplateID = "rule-b"
|
||||
|
||||
require.NoError(t, exp.Export(a))
|
||||
require.NoError(t, exp.Export(b))
|
||||
require.NoError(t, exp.Export(a))
|
||||
require.NoError(t, exp.Export(b))
|
||||
|
||||
require.Len(t, exp.rules, 2)
|
||||
require.Equal(t, "rule-a", exp.rules[0].Id)
|
||||
require.Equal(t, "rule-b", exp.rules[1].Id)
|
||||
require.Equal(t, 0, exp.rulemap["rule-a"])
|
||||
require.Equal(t, 1, exp.rulemap["rule-b"])
|
||||
|
||||
require.NoError(t, exp.Close())
|
||||
|
||||
bin, err := os.ReadFile(out)
|
||||
require.NoError(t, err)
|
||||
var log upstream.SarifLog
|
||||
require.NoError(t, json.Unmarshal(bin, &log))
|
||||
require.Len(t, log.Runs, 1)
|
||||
require.Len(t, log.Runs[0].Result, 4)
|
||||
|
||||
for i, want := range []string{"rule-a", "rule-b", "rule-a", "rule-b"} {
|
||||
res := log.Runs[0].Result[i]
|
||||
require.Equal(t, want, res.RuleId, "result %d ruleId", i)
|
||||
require.Equal(t, log.Runs[0].Tool.Driver.Rules[res.RuleIndex].Id, res.RuleId,
|
||||
"result %d index points to wrong rule", i)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExport_CVSSScoreOverridesSeverity(t *testing.T) {
|
||||
out := filepath.Join(t.TempDir(), "report.sarif")
|
||||
exp, err := New(&Options{File: out})
|
||||
require.NoError(t, err)
|
||||
|
||||
event := makeEvent()
|
||||
event.Info.SeverityHolder.Severity = severity.Medium
|
||||
event.Info.Classification.CVSSScore = 7.3
|
||||
require.NoError(t, exp.Export(event))
|
||||
|
||||
require.Len(t, exp.rules, 1)
|
||||
props, ok := exp.rules[0].Properties.(map[string]interface{})
|
||||
require.True(t, ok)
|
||||
require.Equal(t, "7.3", props["security-severity"])
|
||||
}
|
||||
|
||||
func TestExport_Concurrency(t *testing.T) {
|
||||
out := filepath.Join(t.TempDir(), "report.sarif")
|
||||
exp, err := New(&Options{File: out})
|
||||
require.NoError(t, err)
|
||||
|
||||
const workers = 64
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(workers)
|
||||
errs := make(chan error, workers)
|
||||
for i := 0; i < workers; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
errs <- exp.Export(makeEvent())
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for e := range errs {
|
||||
require.NoError(t, e)
|
||||
}
|
||||
|
||||
require.Len(t, exp.rules, 1)
|
||||
require.NoError(t, exp.Close())
|
||||
|
||||
bin, err := os.ReadFile(out)
|
||||
require.NoError(t, err)
|
||||
var log upstream.SarifLog
|
||||
require.NoError(t, json.Unmarshal(bin, &log))
|
||||
require.Len(t, log.Runs[0].Result, workers)
|
||||
}
|
||||
@@ -40,7 +40,7 @@ func BenchmarkParseAcrossEngineLocalCaches(b *testing.B) {
|
||||
engineOptions := make([]*protocols.ExecutorOptions, engineCount)
|
||||
engineParsers := make([]*templates.Parser, engineCount)
|
||||
for i := range engineCount {
|
||||
engineParsers[i] = templates.NewParserWithParsedCache(sharedParser.Cache())
|
||||
engineParsers[i] = templates.NewExecutionParser(sharedParser)
|
||||
engineOptions[i] = executerOpts.Copy()
|
||||
engineOptions[i].Parser = engineParsers[i]
|
||||
}
|
||||
@@ -57,7 +57,7 @@ func BenchmarkParseAcrossEngineLocalCaches(b *testing.B) {
|
||||
}
|
||||
|
||||
for _, parser := range engineParsers {
|
||||
parser.CompiledCache().Purge()
|
||||
parser.PurgeCompiled()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,6 +51,25 @@ func NewParserWithParsedCache(cache *Cache) *Parser {
|
||||
}
|
||||
}
|
||||
|
||||
// NewExecutionParser creates a parser for one engine execution. Parsed template
|
||||
// data is shared with the long-lived parent, while compiled templates remain
|
||||
// private because they retain execution-specific options and mutable state.
|
||||
func NewExecutionParser(parent *Parser) *Parser {
|
||||
if parent == nil {
|
||||
return NewParser()
|
||||
}
|
||||
|
||||
parent.Lock()
|
||||
defer parent.Unlock()
|
||||
|
||||
return &Parser{
|
||||
ShouldValidate: parent.ShouldValidate,
|
||||
NoStrictSyntax: parent.NoStrictSyntax,
|
||||
parsedTemplatesCache: parent.parsedTemplatesCache,
|
||||
compiledTemplatesCache: NewCache(),
|
||||
}
|
||||
}
|
||||
|
||||
// Purge clears the parsed and compiled template caches. It should be called
|
||||
// when the parser is no longer needed (e.g. on engine Close) so a long-running
|
||||
// embedder does not retain every compiled template (a heap-heavy object) for
|
||||
@@ -62,6 +81,14 @@ func (p *Parser) Purge() {
|
||||
p.compiledTemplatesCache.Purge()
|
||||
}
|
||||
|
||||
// PurgeCompiled releases execution-specific compiled templates without
|
||||
// clearing parsed template data that may be shared by other executions.
|
||||
func (p *Parser) PurgeCompiled() {
|
||||
p.Lock()
|
||||
defer p.Unlock()
|
||||
p.compiledTemplatesCache.Purge()
|
||||
}
|
||||
|
||||
// Cache returns the parsed templates cache
|
||||
func (p *Parser) Cache() *Cache {
|
||||
return p.parsedTemplatesCache
|
||||
|
||||
@@ -22,3 +22,41 @@ func TestParserPurgeClearsCaches(t *testing.T) {
|
||||
require.Equal(t, 0, p.ParsedCount(), "parsed cache must be empty after Purge")
|
||||
require.Equal(t, 0, p.CompiledCount(), "compiled cache must be empty after Purge")
|
||||
}
|
||||
|
||||
func TestParserExecutionParserSharesOnlyParsedCache(t *testing.T) {
|
||||
parent := NewParser()
|
||||
parent.ShouldValidate = true
|
||||
parent.NoStrictSyntax = true
|
||||
parent.Cache().Store("parsed", &Template{}, []byte("raw"), nil)
|
||||
parent.CompiledCache().StoreWithoutRaw("parent-compiled", &Template{}, nil)
|
||||
|
||||
execution := NewExecutionParser(parent)
|
||||
sibling := NewExecutionParser(parent)
|
||||
|
||||
require.Same(t, parent.Cache(), execution.Cache())
|
||||
require.Same(t, parent.Cache(), sibling.Cache())
|
||||
require.NotSame(t, parent.CompiledCache(), execution.CompiledCache())
|
||||
require.NotSame(t, execution.CompiledCache(), sibling.CompiledCache())
|
||||
require.True(t, execution.ShouldValidate)
|
||||
require.True(t, execution.NoStrictSyntax)
|
||||
require.Equal(t, 1, execution.ParsedCount())
|
||||
require.Zero(t, execution.CompiledCount())
|
||||
|
||||
execution.CompiledCache().StoreWithoutRaw("execution-compiled", &Template{}, nil)
|
||||
require.Equal(t, 1, execution.CompiledCount())
|
||||
require.Zero(t, sibling.CompiledCount())
|
||||
require.Equal(t, 1, parent.CompiledCount())
|
||||
}
|
||||
|
||||
func TestParserPurgeCompiledPreservesSharedParsedCache(t *testing.T) {
|
||||
parent := NewParser()
|
||||
parent.Cache().Store("parsed", &Template{}, []byte("raw"), nil)
|
||||
execution := NewExecutionParser(parent)
|
||||
execution.CompiledCache().StoreWithoutRaw("compiled", &Template{}, nil)
|
||||
|
||||
execution.PurgeCompiled()
|
||||
|
||||
require.Zero(t, execution.CompiledCount())
|
||||
require.Equal(t, 1, execution.ParsedCount())
|
||||
require.Equal(t, 1, parent.ParsedCount())
|
||||
}
|
||||
@@ -460,7 +460,7 @@ func init() {
|
||||
Value: "HTTP response headers in name:value format",
|
||||
},
|
||||
}
|
||||
HTTPRequestDoc.Fields = make([]encoder.Doc, 39)
|
||||
HTTPRequestDoc.Fields = make([]encoder.Doc, 40)
|
||||
HTTPRequestDoc.Fields[0].Name = "path"
|
||||
HTTPRequestDoc.Fields[0].Type = "[]string"
|
||||
HTTPRequestDoc.Fields[0].Note = ""
|
||||
@@ -599,91 +599,96 @@ func init() {
|
||||
HTTPRequestDoc.Fields[21].Note = ""
|
||||
HTTPRequestDoc.Fields[21].Description = "DisableCookie is an optional setting that disables cookie reuse"
|
||||
HTTPRequestDoc.Fields[21].Comments[encoder.LineComment] = "DisableCookie is an optional setting that disables cookie reuse"
|
||||
HTTPRequestDoc.Fields[22].Name = "read-all"
|
||||
HTTPRequestDoc.Fields[22].Name = "disable-http-cache"
|
||||
HTTPRequestDoc.Fields[22].Type = "bool"
|
||||
HTTPRequestDoc.Fields[22].Note = ""
|
||||
HTTPRequestDoc.Fields[22].Description = "Enables force reading of the entire raw unsafe request body ignoring\nany specified content length headers."
|
||||
HTTPRequestDoc.Fields[22].Comments[encoder.LineComment] = "Enables force reading of the entire raw unsafe request body ignoring"
|
||||
HTTPRequestDoc.Fields[23].Name = "redirects"
|
||||
HTTPRequestDoc.Fields[22].Description = "DisableHTTPCache turns off HTTP caching for this request. It cannot turn caching on when -http-cache is unset."
|
||||
HTTPRequestDoc.Fields[22].Comments[encoder.LineComment] = "DisableHTTPCache turns off HTTP caching for this request. It cannot turn caching on when -http-cache is unset."
|
||||
HTTPRequestDoc.Fields[23].Name = "read-all"
|
||||
HTTPRequestDoc.Fields[23].Type = "bool"
|
||||
HTTPRequestDoc.Fields[23].Note = ""
|
||||
HTTPRequestDoc.Fields[23].Description = "Redirects specifies whether redirects should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
|
||||
HTTPRequestDoc.Fields[23].Comments[encoder.LineComment] = "Redirects specifies whether redirects should be followed by the HTTP Client."
|
||||
HTTPRequestDoc.Fields[24].Name = "host-redirects"
|
||||
HTTPRequestDoc.Fields[23].Description = "Enables force reading of the entire raw unsafe request body ignoring\nany specified content length headers."
|
||||
HTTPRequestDoc.Fields[23].Comments[encoder.LineComment] = "Enables force reading of the entire raw unsafe request body ignoring"
|
||||
HTTPRequestDoc.Fields[24].Name = "redirects"
|
||||
HTTPRequestDoc.Fields[24].Type = "bool"
|
||||
HTTPRequestDoc.Fields[24].Note = ""
|
||||
HTTPRequestDoc.Fields[24].Description = "Redirects specifies whether only redirects to the same host should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
|
||||
HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "Redirects specifies whether only redirects to the same host should be followed by the HTTP Client."
|
||||
HTTPRequestDoc.Fields[25].Name = "protocol-redirects"
|
||||
HTTPRequestDoc.Fields[24].Description = "Redirects specifies whether redirects should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
|
||||
HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "Redirects specifies whether redirects should be followed by the HTTP Client."
|
||||
HTTPRequestDoc.Fields[25].Name = "host-redirects"
|
||||
HTTPRequestDoc.Fields[25].Type = "bool"
|
||||
HTTPRequestDoc.Fields[25].Note = ""
|
||||
HTTPRequestDoc.Fields[25].Description = "ProtocolRedirects specifies whether only redirects within the same protocol should be followed.\n\nWhen set to true with redirects enabled, cross-protocol redirects (e.g. HTTP to HTTPS) will be blocked."
|
||||
HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "ProtocolRedirects specifies whether only redirects within the same protocol should be followed."
|
||||
HTTPRequestDoc.Fields[26].Name = "pipeline"
|
||||
HTTPRequestDoc.Fields[25].Description = "Redirects specifies whether only redirects to the same host should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
|
||||
HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "Redirects specifies whether only redirects to the same host should be followed by the HTTP Client."
|
||||
HTTPRequestDoc.Fields[26].Name = "protocol-redirects"
|
||||
HTTPRequestDoc.Fields[26].Type = "bool"
|
||||
HTTPRequestDoc.Fields[26].Note = ""
|
||||
HTTPRequestDoc.Fields[26].Description = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining\n\nAll requests must be idempotent (GET/POST). This can be used for race conditions/billions requests."
|
||||
HTTPRequestDoc.Fields[26].Comments[encoder.LineComment] = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"
|
||||
HTTPRequestDoc.Fields[27].Name = "unsafe"
|
||||
HTTPRequestDoc.Fields[26].Description = "ProtocolRedirects specifies whether only redirects within the same protocol should be followed.\n\nWhen set to true with redirects enabled, cross-protocol redirects (e.g. HTTP to HTTPS) will be blocked."
|
||||
HTTPRequestDoc.Fields[26].Comments[encoder.LineComment] = "ProtocolRedirects specifies whether only redirects within the same protocol should be followed."
|
||||
HTTPRequestDoc.Fields[27].Name = "pipeline"
|
||||
HTTPRequestDoc.Fields[27].Type = "bool"
|
||||
HTTPRequestDoc.Fields[27].Note = ""
|
||||
HTTPRequestDoc.Fields[27].Description = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests.\n\nThis uses the [rawhttp](https://github.com/projectdiscovery/rawhttp) engine to achieve complete\ncontrol over the request, with no normalization performed by the client."
|
||||
HTTPRequestDoc.Fields[27].Comments[encoder.LineComment] = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests."
|
||||
HTTPRequestDoc.Fields[28].Name = "race"
|
||||
HTTPRequestDoc.Fields[27].Description = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining\n\nAll requests must be idempotent (GET/POST). This can be used for race conditions/billions requests."
|
||||
HTTPRequestDoc.Fields[27].Comments[encoder.LineComment] = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"
|
||||
HTTPRequestDoc.Fields[28].Name = "unsafe"
|
||||
HTTPRequestDoc.Fields[28].Type = "bool"
|
||||
HTTPRequestDoc.Fields[28].Note = ""
|
||||
HTTPRequestDoc.Fields[28].Description = "Race determines if all the request have to be attempted at the same time (Race Condition)\n\nThe actual number of requests that will be sent is determined by the `race_count` field."
|
||||
HTTPRequestDoc.Fields[28].Comments[encoder.LineComment] = "Race determines if all the request have to be attempted at the same time (Race Condition)"
|
||||
HTTPRequestDoc.Fields[29].Name = "req-condition"
|
||||
HTTPRequestDoc.Fields[28].Description = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests.\n\nThis uses the [rawhttp](https://github.com/projectdiscovery/rawhttp) engine to achieve complete\ncontrol over the request, with no normalization performed by the client."
|
||||
HTTPRequestDoc.Fields[28].Comments[encoder.LineComment] = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests."
|
||||
HTTPRequestDoc.Fields[29].Name = "race"
|
||||
HTTPRequestDoc.Fields[29].Type = "bool"
|
||||
HTTPRequestDoc.Fields[29].Note = ""
|
||||
HTTPRequestDoc.Fields[29].Description = "ReqCondition automatically assigns numbers to requests and preserves their history.\n\nThis allows matching on them later for multi-request conditions."
|
||||
HTTPRequestDoc.Fields[29].Comments[encoder.LineComment] = "ReqCondition automatically assigns numbers to requests and preserves their history."
|
||||
HTTPRequestDoc.Fields[30].Name = "stop-at-first-match"
|
||||
HTTPRequestDoc.Fields[29].Description = "Race determines if all the request have to be attempted at the same time (Race Condition)\n\nThe actual number of requests that will be sent is determined by the `race_count` field."
|
||||
HTTPRequestDoc.Fields[29].Comments[encoder.LineComment] = "Race determines if all the request have to be attempted at the same time (Race Condition)"
|
||||
HTTPRequestDoc.Fields[30].Name = "req-condition"
|
||||
HTTPRequestDoc.Fields[30].Type = "bool"
|
||||
HTTPRequestDoc.Fields[30].Note = ""
|
||||
HTTPRequestDoc.Fields[30].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
|
||||
HTTPRequestDoc.Fields[30].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
|
||||
HTTPRequestDoc.Fields[31].Name = "skip-variables-check"
|
||||
HTTPRequestDoc.Fields[30].Description = "ReqCondition automatically assigns numbers to requests and preserves their history.\n\nThis allows matching on them later for multi-request conditions."
|
||||
HTTPRequestDoc.Fields[30].Comments[encoder.LineComment] = "ReqCondition automatically assigns numbers to requests and preserves their history."
|
||||
HTTPRequestDoc.Fields[31].Name = "stop-at-first-match"
|
||||
HTTPRequestDoc.Fields[31].Type = "bool"
|
||||
HTTPRequestDoc.Fields[31].Note = ""
|
||||
HTTPRequestDoc.Fields[31].Description = "SkipVariablesCheck skips the check for unresolved variables in request"
|
||||
HTTPRequestDoc.Fields[31].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request"
|
||||
HTTPRequestDoc.Fields[32].Name = "iterate-all"
|
||||
HTTPRequestDoc.Fields[31].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
|
||||
HTTPRequestDoc.Fields[31].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
|
||||
HTTPRequestDoc.Fields[32].Name = "skip-variables-check"
|
||||
HTTPRequestDoc.Fields[32].Type = "bool"
|
||||
HTTPRequestDoc.Fields[32].Note = ""
|
||||
HTTPRequestDoc.Fields[32].Description = "IterateAll iterates all the values extracted from internal extractors"
|
||||
HTTPRequestDoc.Fields[32].Comments[encoder.LineComment] = "IterateAll iterates all the values extracted from internal extractors"
|
||||
HTTPRequestDoc.Fields[33].Name = "digest-username"
|
||||
HTTPRequestDoc.Fields[33].Type = "string"
|
||||
HTTPRequestDoc.Fields[32].Description = "SkipVariablesCheck skips the check for unresolved variables in request"
|
||||
HTTPRequestDoc.Fields[32].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request"
|
||||
HTTPRequestDoc.Fields[33].Name = "iterate-all"
|
||||
HTTPRequestDoc.Fields[33].Type = "bool"
|
||||
HTTPRequestDoc.Fields[33].Note = ""
|
||||
HTTPRequestDoc.Fields[33].Description = "DigestAuthUsername specifies the username for digest authentication"
|
||||
HTTPRequestDoc.Fields[33].Comments[encoder.LineComment] = "DigestAuthUsername specifies the username for digest authentication"
|
||||
HTTPRequestDoc.Fields[34].Name = "digest-password"
|
||||
HTTPRequestDoc.Fields[33].Description = "IterateAll iterates all the values extracted from internal extractors"
|
||||
HTTPRequestDoc.Fields[33].Comments[encoder.LineComment] = "IterateAll iterates all the values extracted from internal extractors"
|
||||
HTTPRequestDoc.Fields[34].Name = "digest-username"
|
||||
HTTPRequestDoc.Fields[34].Type = "string"
|
||||
HTTPRequestDoc.Fields[34].Note = ""
|
||||
HTTPRequestDoc.Fields[34].Description = "DigestAuthPassword specifies the password for digest authentication"
|
||||
HTTPRequestDoc.Fields[34].Comments[encoder.LineComment] = "DigestAuthPassword specifies the password for digest authentication"
|
||||
HTTPRequestDoc.Fields[35].Name = "disable-path-automerge"
|
||||
HTTPRequestDoc.Fields[35].Type = "bool"
|
||||
HTTPRequestDoc.Fields[34].Description = "DigestAuthUsername specifies the username for digest authentication"
|
||||
HTTPRequestDoc.Fields[34].Comments[encoder.LineComment] = "DigestAuthUsername specifies the username for digest authentication"
|
||||
HTTPRequestDoc.Fields[35].Name = "digest-password"
|
||||
HTTPRequestDoc.Fields[35].Type = "string"
|
||||
HTTPRequestDoc.Fields[35].Note = ""
|
||||
HTTPRequestDoc.Fields[35].Description = "DisablePathAutomerge disables merging target url path with raw request path"
|
||||
HTTPRequestDoc.Fields[35].Comments[encoder.LineComment] = "DisablePathAutomerge disables merging target url path with raw request path"
|
||||
HTTPRequestDoc.Fields[36].Name = "pre-condition"
|
||||
HTTPRequestDoc.Fields[36].Type = "[]matchers.Matcher"
|
||||
HTTPRequestDoc.Fields[35].Description = "DigestAuthPassword specifies the password for digest authentication"
|
||||
HTTPRequestDoc.Fields[35].Comments[encoder.LineComment] = "DigestAuthPassword specifies the password for digest authentication"
|
||||
HTTPRequestDoc.Fields[36].Name = "disable-path-automerge"
|
||||
HTTPRequestDoc.Fields[36].Type = "bool"
|
||||
HTTPRequestDoc.Fields[36].Note = ""
|
||||
HTTPRequestDoc.Fields[36].Description = "Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not"
|
||||
HTTPRequestDoc.Fields[36].Comments[encoder.LineComment] = "Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not"
|
||||
HTTPRequestDoc.Fields[37].Name = "pre-condition-operator"
|
||||
HTTPRequestDoc.Fields[37].Type = "string"
|
||||
HTTPRequestDoc.Fields[36].Description = "DisablePathAutomerge disables merging target url path with raw request path"
|
||||
HTTPRequestDoc.Fields[36].Comments[encoder.LineComment] = "DisablePathAutomerge disables merging target url path with raw request path"
|
||||
HTTPRequestDoc.Fields[37].Name = "pre-condition"
|
||||
HTTPRequestDoc.Fields[37].Type = "[]matchers.Matcher"
|
||||
HTTPRequestDoc.Fields[37].Note = ""
|
||||
HTTPRequestDoc.Fields[37].Description = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR"
|
||||
HTTPRequestDoc.Fields[37].Comments[encoder.LineComment] = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR"
|
||||
HTTPRequestDoc.Fields[38].Name = "global-matchers"
|
||||
HTTPRequestDoc.Fields[38].Type = "bool"
|
||||
HTTPRequestDoc.Fields[37].Description = "Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not"
|
||||
HTTPRequestDoc.Fields[37].Comments[encoder.LineComment] = "Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not"
|
||||
HTTPRequestDoc.Fields[38].Name = "pre-condition-operator"
|
||||
HTTPRequestDoc.Fields[38].Type = "string"
|
||||
HTTPRequestDoc.Fields[38].Note = ""
|
||||
HTTPRequestDoc.Fields[38].Description = "GlobalMatchers marks matchers as static and applies globally to all result events from other templates"
|
||||
HTTPRequestDoc.Fields[38].Comments[encoder.LineComment] = "GlobalMatchers marks matchers as static and applies globally to all result events from other templates"
|
||||
HTTPRequestDoc.Fields[38].Description = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR"
|
||||
HTTPRequestDoc.Fields[38].Comments[encoder.LineComment] = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR"
|
||||
HTTPRequestDoc.Fields[39].Name = "global-matchers"
|
||||
HTTPRequestDoc.Fields[39].Type = "bool"
|
||||
HTTPRequestDoc.Fields[39].Note = ""
|
||||
HTTPRequestDoc.Fields[39].Description = "GlobalMatchers marks matchers as static and applies globally to all result events from other templates"
|
||||
HTTPRequestDoc.Fields[39].Comments[encoder.LineComment] = "GlobalMatchers marks matchers as static and applies globally to all result events from other templates"
|
||||
|
||||
GENERATORSAttackTypeHolderDoc.Type = "generators.AttackTypeHolder"
|
||||
GENERATORSAttackTypeHolderDoc.Comments[encoder.LineComment] = " AttackTypeHolder is used to hold internal type of the protocol"
|
||||
|
||||
@@ -212,6 +212,11 @@ type Options struct {
|
||||
// When enabled, each host gets its own rate limiter and global rate limit becomes unlimited.
|
||||
// Disabled by default.
|
||||
PerHostRateLimit bool
|
||||
// EnableHTTPCache enables HTTP caching (RFC 9111) for requests
|
||||
//
|
||||
// NOTE(dwisiswant0): this is experimental and might be enabled by default
|
||||
// in the future releases.
|
||||
EnableHTTPCache bool
|
||||
// LeaveDefaultPorts skips normalization of default ports
|
||||
LeaveDefaultPorts bool
|
||||
// AutomaticScan enables automatic tech based template execution
|
||||
@@ -578,6 +583,7 @@ func (options *Options) Copy() *Options {
|
||||
DisableHTTPProbe: options.DisableHTTPProbe,
|
||||
PreflightPortScan: options.PreflightPortScan,
|
||||
PerHostRateLimit: options.PerHostRateLimit,
|
||||
EnableHTTPCache: options.EnableHTTPCache,
|
||||
LeaveDefaultPorts: options.LeaveDefaultPorts,
|
||||
AutomaticScan: options.AutomaticScan,
|
||||
Silent: options.Silent,
|
||||
|
||||
Reference in new issue
Block a user