mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2026-11-08 10:10:18 -10:00
Compare commits
261
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b3706070ab | ||
|
|
755c9785b1 | ||
|
|
c73906b405 | ||
|
|
8891d70d42 | ||
|
|
ebfd2f2992 | ||
|
|
e74355344d | ||
|
|
b49beef554 | ||
|
|
cf3b5bf449 | ||
|
|
b63a23bd5c | ||
|
|
7fb1fe7bf2 | ||
|
|
3e93996471 | ||
|
|
8e535f625d | ||
|
|
93fb595fc1 | ||
|
|
df74ff3edf | ||
|
|
fe090d7c10 | ||
|
|
4cbc1bd75c | ||
|
|
09c47c32bd | ||
|
|
fb4fe015f0 | ||
|
|
924e2a7a25 | ||
|
|
3d60c9fdbe | ||
|
|
a6cddf86ec | ||
|
|
8dfd666223 | ||
|
|
56b6d42723 | ||
|
|
2c46f2c2ef | ||
|
|
0e498bcd76 | ||
|
|
f6af5216ba | ||
|
|
d9452011fc | ||
|
|
2b9c985818 | ||
|
|
2449d5628f | ||
|
|
e865ca496d | ||
|
|
6384399375 | ||
|
|
c994bba249 | ||
|
|
9eede47dbd | ||
|
|
7f56dc27c1 | ||
|
|
3edd5d956f | ||
|
|
3447f09c9f | ||
|
|
9ec2e995d0 | ||
|
|
f181a691b2 | ||
|
|
9102f333a9 | ||
|
|
3dab87bf77 | ||
|
|
7e151de8a3 | ||
|
|
1a8f4d40d6 | ||
|
|
7ea72b5268 | ||
|
|
29dfd671fc | ||
|
|
2978819d0e | ||
|
|
0aba8e5a65 | ||
|
|
78ccfc2912 | ||
|
|
c1878b6134 | ||
|
|
29977358d7 | ||
|
|
f63cad9651 | ||
|
|
528caa8831 | ||
|
|
c32cff8521 | ||
|
|
cf44afbb99 | ||
|
|
d5d8d50ab7 | ||
|
|
a754e7d223 | ||
|
|
a52e6753a2 | ||
|
|
ad1e6f8d75 | ||
|
|
aad0e51fef | ||
|
|
50b43f2015 | ||
|
|
146fb5fb34 | ||
|
|
3ea8dc29c3 | ||
|
|
4e8843a7cd | ||
|
|
761c7c0c29 | ||
|
|
232de93297 | ||
|
|
c746a8fdae | ||
|
|
9b37f1b441 | ||
|
|
8b8a3a11b6 | ||
|
|
e70044af4c | ||
|
|
d20240bf0c | ||
|
|
6730f5262f | ||
|
|
a42dd7a6d9 | ||
|
|
7877341a13 | ||
|
|
141f34a8ae | ||
|
|
8016d15fdc | ||
|
|
488d5886ea | ||
|
|
cf5557e0fd | ||
|
|
1a2363db8a | ||
|
|
3a4e436eed | ||
|
|
0f5d0d7976 | ||
|
|
954c825fab | ||
|
|
3eff4146e1 | ||
|
|
87d62d3a8f | ||
|
|
efcc8e95c3 | ||
|
|
96bfb2bac1 | ||
|
|
cc0f1e9606 | ||
|
|
0054e8b7ae | ||
|
|
aecef0ab75 | ||
|
|
0dde27f41d | ||
|
|
13cca2b2c8 | ||
|
|
7b0c6fb782 | ||
|
|
2829fd30bb | ||
|
|
53af79a392 | ||
|
|
90d51f733a | ||
|
|
d7da0e0914 | ||
|
|
5147c72a45 | ||
|
|
7e33712d08 | ||
|
|
3a44eaf7b5 | ||
|
|
fb93cbeeda | ||
|
|
69c831bad2 | ||
|
|
f544ea706c | ||
|
|
1093bbc62d | ||
|
|
133e8b98c0 | ||
|
|
ed1c5752f3 | ||
|
|
e60f4158ea | ||
|
|
984deac200 | ||
|
|
17360cb6e4 | ||
|
|
770f20eeab | ||
|
|
5b5d87f62a | ||
|
|
6c05580120 | ||
|
|
16154d1442 | ||
|
|
7fc4752a95 | ||
|
|
248bac75a0 | ||
|
|
6794b9cba0 | ||
|
|
8136d4f368 | ||
|
|
1a8124679e | ||
|
|
f3181b9a2a | ||
|
|
3c0b84b137 | ||
|
|
b625a659db | ||
|
|
dd8946d3f2 | ||
|
|
c814128ee2 | ||
|
|
6f59472f78 | ||
|
|
3be27b9dfd | ||
|
|
e535e0126f | ||
|
|
9d59fd0188 | ||
|
|
c4a63ec1de | ||
|
|
8c30b5d53c | ||
|
|
82144e542b | ||
|
|
3b5e3a4de9 | ||
|
|
b4551b901f | ||
|
|
ca1cbf2135 | ||
|
|
3384606dea | ||
|
|
d2ae3521cc | ||
|
|
f3298323f3 | ||
|
|
71293f2f81 | ||
|
|
a5b424bbfe | ||
|
|
966005ecb0 | ||
|
|
5cd344e7cd | ||
|
|
ae52ef04c9 | ||
|
|
b9ce0c2226 | ||
|
|
da316df556 | ||
|
|
ccb65a0016 | ||
|
|
5b92381494 | ||
|
|
e168f8dbfa | ||
|
|
f57bd8c8ee | ||
|
|
5557f4eae3 | ||
|
|
f6efa9da6c | ||
|
|
2db03a9335 | ||
|
|
5b7b83618c | ||
|
|
89cfb75bb6 | ||
|
|
f0429aa4b7 | ||
|
|
1684f4143e | ||
|
|
7823d46fdb | ||
|
|
c3750be380 | ||
|
|
361f877730 | ||
|
|
427007cf5d | ||
|
|
cd41cb9f4d | ||
|
|
295fb91123 | ||
|
|
19554a793d | ||
|
|
0967d42b40 | ||
|
|
e6c1a80498 | ||
|
|
a2f10442e7 | ||
|
|
231d838734 | ||
|
|
81a1159587 | ||
|
|
171db55891 | ||
|
|
59c6c855bd | ||
|
|
78c2712bd3 | ||
|
|
dcdc7ebc32 | ||
|
|
73e70ea49d | ||
|
|
0974d7fd04 | ||
|
|
635d28e2cb | ||
|
|
a2d686e75c | ||
|
|
3fd6e63e52 | ||
|
|
cc1d646347 | ||
|
|
bfef42f9e3 | ||
|
|
4e2af6bf67 | ||
|
|
2a80cd594d | ||
|
|
603a86b7e6 | ||
|
|
e0d3bb45ed | ||
|
|
8c560523e3 | ||
|
|
07590268c1 | ||
|
|
ae612572af | ||
|
|
67571b3cfe | ||
|
|
75016d1e96 | ||
|
|
6208dbe06a | ||
|
|
3597ab07f0 | ||
|
|
0f4f4b94e6 | ||
|
|
c903da3a0c | ||
|
|
86be8429ed | ||
|
|
7e04181391 | ||
|
|
6a6de384fc | ||
|
|
b529125031 | ||
|
|
3ef581c5e8 | ||
|
|
ca11a2fad6 | ||
|
|
7d450507f7 | ||
|
|
95a72cfd50 | ||
|
|
f8f89bb721 | ||
|
|
cb2d93174a | ||
|
|
61bd0828dc | ||
|
|
d44f07f648 | ||
|
|
93be3b8291 | ||
|
|
202524283b | ||
|
|
8ea5061f5e | ||
|
|
d2cf69aebb | ||
|
|
39e9286371 | ||
|
|
0ea42e5f66 | ||
|
|
f610ed4cab | ||
|
|
792998d8e2 | ||
|
|
c4fa2c74c1 | ||
|
|
d4f1a815ed | ||
|
|
a65841c034 | ||
|
|
876974f38b | ||
|
|
ca543d7885 | ||
|
|
fde6f72934 | ||
|
|
3af37362e3 | ||
|
|
99a9ce398d | ||
|
|
48af0b4f6c | ||
|
|
e1dfa1baa7 | ||
|
|
089e2a4ee0 | ||
|
|
46555bcd1e | ||
|
|
521a21c06a | ||
|
|
1acd40f97f | ||
|
|
09c2ca540a | ||
|
|
c863143771 | ||
|
|
5c8da8d88b | ||
|
|
4b22a3d53e | ||
|
|
94c77c1a28 | ||
|
|
ee1c847626 | ||
|
|
a21bfc4303 | ||
|
|
c487e59602 | ||
|
|
1f8dc4c358 | ||
|
|
608159bbbe | ||
|
|
b05359bc82 | ||
|
|
4916cf34f0 | ||
|
|
f460bf926d | ||
|
|
218a2f69a5 | ||
|
|
ff5734ba15 | ||
|
|
9c64a1cb9b | ||
|
|
1f0aef970c | ||
|
|
5b7debf349 | ||
|
|
e83382d4e4 | ||
|
|
b61321cd19 | ||
|
|
f20f95f67e | ||
|
|
efcef55681 | ||
|
|
5be258f948 | ||
|
|
6b358b39a3 | ||
|
|
b41f4d97d6 | ||
|
|
5c15c77777 | ||
|
|
ce57bdf4ab | ||
|
|
d55ab2f827 | ||
|
|
cf8d067fea | ||
|
|
4baf46f080 | ||
|
|
8304462420 | ||
|
|
1f538bcac6 | ||
|
|
937fa1252b | ||
|
|
99914e1a32 | ||
|
|
1a9a7563c0 | ||
|
|
248548e075 | ||
|
|
3eb3f66897 | ||
|
|
5f501da063 | ||
|
|
a0bd3b854e | ||
|
|
32845bccf2 |
No files matched your search
@@ -1,35 +0,0 @@
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(find:*)",
|
||||
"Bash(mkdir:*)",
|
||||
"Bash(cp:*)",
|
||||
"Bash(ls:*)",
|
||||
"Bash(make:*)",
|
||||
"Bash(go:*)",
|
||||
"Bash(golangci-lint:*)",
|
||||
"Bash(git merge:*)",
|
||||
"Bash(git add:*)",
|
||||
"Bash(git commit:*)",
|
||||
"Bash(git push:*)",
|
||||
"Bash(git pull:*)",
|
||||
"Bash(git fetch:*)",
|
||||
"Bash(git checkout:*)",
|
||||
"WebFetch(*)",
|
||||
"Write(*)",
|
||||
"WebSearch(*)",
|
||||
"MultiEdit(*)",
|
||||
"Edit(*)",
|
||||
"Bash(gh:*)",
|
||||
"Bash(grep:*)",
|
||||
"Bash(tree:*)",
|
||||
"Bash(./nuclei:*)",
|
||||
"WebFetch(domain:github.com)"
|
||||
],
|
||||
"deny": [
|
||||
"Bash(make run:*)",
|
||||
"Bash(./bin/nuclei:*)"
|
||||
],
|
||||
"defaultMode": "acceptEdits"
|
||||
}
|
||||
}
|
||||
File renamed without changes.
@@ -2,10 +2,6 @@ blank_issues_enabled: false
|
||||
|
||||
contact_links:
|
||||
|
||||
- name: 🐛 Report a Bug (Start with Discussion)
|
||||
url: https://github.com/orgs/projectdiscovery/discussions/new?category=q-a
|
||||
about: Start by reporting your issue in discussions for proper triage. Issues will be created after review to avoid duplicate/invalid reports.
|
||||
|
||||
- name: 💡 Request a Feature (Start with Discussion)
|
||||
url: https://github.com/orgs/projectdiscovery/discussions/new?category=ideas
|
||||
about: Share your feature idea in discussions first. This helps validate and refine the request before creating an issue.
|
||||
|
||||
@@ -2,6 +2,7 @@ addReviewers: true
|
||||
reviewers:
|
||||
- dogancanbakir
|
||||
- dwisiswant0
|
||||
- mzack9999
|
||||
|
||||
numberOfReviewers: 1
|
||||
skipKeywords:
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.actor == 'dependabot[bot]'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
token: ${{ secrets.DEPENDABOT_PAT }}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
- uses: projectdiscovery/actions/setup/go/compat-checks@v1
|
||||
with:
|
||||
release-test: true
|
||||
@@ -11,7 +11,7 @@ jobs:
|
||||
if: "${{ !endsWith(github.actor, '[bot]') }}"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
- uses: projectdiscovery/actions/setup/go@v1
|
||||
- uses: projectdiscovery/actions/setup/git@v1
|
||||
- run: make syntax-docs
|
||||
|
||||
@@ -28,9 +28,10 @@ jobs:
|
||||
LIST_FILE: "/tmp/targets-${{ matrix.targets }}.txt"
|
||||
PROFILE_MEM: "/tmp/nuclei-profile-${{ matrix.targets }}-targets"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
- uses: projectdiscovery/actions/setup/git@v1
|
||||
- uses: projectdiscovery/actions/setup/go@v1
|
||||
- uses: projectdiscovery/actions/cache/nuclei@v1
|
||||
- name: Generate list
|
||||
run: for i in {1..${{ matrix.targets }}}; do echo "https://honey.scanme.sh/?_=${i}" >> "${LIST_FILE}"; done
|
||||
# NOTE(dwisiswant0): use `-no-mhe` flag to get better samples.
|
||||
@@ -49,7 +50,7 @@ jobs:
|
||||
files: "${PGO_FILE}"
|
||||
message: "build: update PGO profile :robot:"
|
||||
- run: git push origin $GITHUB_REF
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: "pgo"
|
||||
path: "${{ env.PGO_FILE }}"
|
||||
@@ -16,11 +16,11 @@ jobs:
|
||||
env:
|
||||
OUTPUT: "/tmp/results.sarif"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
- uses: projectdiscovery/actions/setup/go@v1
|
||||
- run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
- run: govulncheck -scan package -format sarif ./... > $OUTPUT
|
||||
- uses: github/codeql-action/upload-sarif@v3
|
||||
- uses: github/codeql-action/upload-sarif@v4
|
||||
with:
|
||||
sarif_file: "${{ env.OUTPUT }}"
|
||||
category: "govulncheck"
|
||||
@@ -11,13 +11,15 @@ jobs:
|
||||
env:
|
||||
BENCH_OUT: "/tmp/bench.out"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
- uses: projectdiscovery/actions/setup/go@v1
|
||||
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
|
||||
- uses: projectdiscovery/actions/cache/nuclei@v1
|
||||
- run: make build-test
|
||||
- run: ./bin/nuclei.test -test.run - -test.bench=. -test.benchmem ./cmd/nuclei/ | tee $BENCH_OUT
|
||||
env:
|
||||
DISABLE_STDOUT: "1"
|
||||
- uses: actions/cache/restore@v4
|
||||
- uses: actions/cache/restore@v5
|
||||
with:
|
||||
path: ./cache
|
||||
key: ${{ runner.os }}-benchmark
|
||||
@@ -31,7 +33,7 @@ jobs:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
comment-on-alert: true
|
||||
summary-always: true
|
||||
- uses: actions/cache/save@v4
|
||||
- uses: actions/cache/save@v5
|
||||
if: github.event_name == 'push'
|
||||
with:
|
||||
path: ./cache
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
LIST_FILE: "/tmp/targets-${{ matrix.count }}.txt"
|
||||
PROFILE_MEM: "/tmp/nuclei-perf-test-${{ matrix.count }}"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
- uses: projectdiscovery/actions/setup/go@v1
|
||||
- run: make verify
|
||||
- name: Generate list
|
||||
|
||||
@@ -10,7 +10,7 @@ jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest-16-cores
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: projectdiscovery/actions/setup/go@v1
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/stale@v9
|
||||
- uses: actions/stale@v10
|
||||
with:
|
||||
days-before-stale: 90
|
||||
days-before-close: 7
|
||||
@@ -37,5 +37,4 @@ jobs:
|
||||
it, please comment or feel free to reopen it.
|
||||
close-issue-label: "Status: Abandoned"
|
||||
close-pr-label: "Status: Abandoned"
|
||||
exempt-issue-labels: "Status: Abandoned"
|
||||
exempt-pr-labels: "Status: Abandoned"
|
||||
exempt-issue-labels: "Type: Enhancement"
|
||||
@@ -22,8 +22,9 @@ jobs:
|
||||
if: "${{ !endsWith(github.actor, '[bot]') }}"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
- uses: projectdiscovery/actions/setup/go@v1
|
||||
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
|
||||
- uses: projectdiscovery/actions/golangci-lint/v2@v1
|
||||
|
||||
tests:
|
||||
@@ -35,8 +36,19 @@ jobs:
|
||||
os: [ubuntu-latest, windows-latest, macOS-latest]
|
||||
runs-on: "${{ matrix.os }}"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
- uses: projectdiscovery/actions/setup/go@v1
|
||||
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
|
||||
- uses: projectdiscovery/actions/cache/nuclei@v1
|
||||
- uses: projectdiscovery/actions/free-disk-space@v1
|
||||
with:
|
||||
llvm: 'false'
|
||||
php: 'false'
|
||||
mongodb: 'false'
|
||||
mysql: 'false'
|
||||
misc-packages: 'false'
|
||||
docker-images: 'false'
|
||||
tools-cache: 'false'
|
||||
- run: make vet
|
||||
- run: make build
|
||||
- run: make test
|
||||
@@ -52,8 +64,10 @@ jobs:
|
||||
needs: ["tests"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
- uses: projectdiscovery/actions/setup/go@v1
|
||||
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
|
||||
- uses: projectdiscovery/actions/cache/nuclei@v1
|
||||
- name: "Simple"
|
||||
run: go run .
|
||||
working-directory: examples/simple/
|
||||
@@ -74,9 +88,11 @@ jobs:
|
||||
os: [ubuntu-latest, windows-latest, macOS-latest]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
- uses: projectdiscovery/actions/setup/go@v1
|
||||
- uses: projectdiscovery/actions/cache/nuclei@v1
|
||||
- uses: projectdiscovery/actions/setup/python@v1
|
||||
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
|
||||
- run: bash run.sh "${{ matrix.os }}"
|
||||
env:
|
||||
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
|
||||
@@ -93,9 +109,11 @@ jobs:
|
||||
os: [ubuntu-latest, windows-latest, macOS-latest]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
- uses: projectdiscovery/actions/setup/go@v1
|
||||
- uses: projectdiscovery/actions/cache/nuclei@v1
|
||||
- uses: projectdiscovery/actions/setup/python@v1
|
||||
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
|
||||
- run: bash run.sh
|
||||
env:
|
||||
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
|
||||
@@ -106,8 +124,9 @@ jobs:
|
||||
needs: ["tests"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
- uses: projectdiscovery/actions/setup/go@v1
|
||||
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
|
||||
- run: make template-validate
|
||||
|
||||
codeql:
|
||||
@@ -119,19 +138,19 @@ jobs:
|
||||
contents: read
|
||||
security-events: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: github/codeql-action/init@v3
|
||||
- uses: actions/checkout@v6
|
||||
- uses: github/codeql-action/init@v4
|
||||
with:
|
||||
languages: 'go'
|
||||
- uses: github/codeql-action/autobuild@v3
|
||||
- uses: github/codeql-action/analyze@v3
|
||||
- uses: github/codeql-action/autobuild@v4
|
||||
- uses: github/codeql-action/analyze@v4
|
||||
|
||||
release:
|
||||
name: "Release test"
|
||||
needs: ["tests"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
- uses: projectdiscovery/actions/setup/go@v1
|
||||
- uses: projectdiscovery/actions/goreleaser@v1
|
||||
|
||||
@@ -143,7 +162,7 @@ jobs:
|
||||
TARGET_URL: "http://scanme.sh/a/?b=c"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
- run: make build
|
||||
- name: "Setup environment (push)"
|
||||
if: ${{ github.event_name == 'push' }}
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@
|
||||
.gitignore
|
||||
.idea
|
||||
.vscode
|
||||
|
||||
.claude/settings.local.json
|
||||
# Binaries
|
||||
/bin/*
|
||||
/bindgen
|
||||
|
||||
@@ -51,7 +51,7 @@ Each protocol (HTTP, DNS, Network, etc.) implements:
|
||||
### Template System
|
||||
- Templates are YAML files defining vulnerability detection logic
|
||||
- Compiled into executable requests with operators (matchers/extractors)
|
||||
- Support for workflows (multi-step template execution)
|
||||
- Support for workflows (multistep template execution)
|
||||
- Template clustering optimizes identical requests across multiple templates
|
||||
|
||||
### Key Execution Flow
|
||||
|
||||
@@ -22,7 +22,7 @@ When this flag is provided, nuclei will print the list of all helper functions a
|
||||
|
||||
- `-svd` flag
|
||||
|
||||
When this flag is provided, nuclei will print all `variables` pre and post execution of a request for a template. This is useful to understand what variables are available for a template and what values they have.
|
||||
When this flag is provided, nuclei will print all `variables` pre- and post-execution of a request for a template. This is useful to understand what variables are available for a template and what values they have.
|
||||
|
||||
- `-elog = errors.txt` flag
|
||||
|
||||
@@ -40,6 +40,8 @@ Nuclei was built with some environment variables in mind to help with debugging.
|
||||
| `SHOW_DSL_ERRORS=true` | Enables Printing DSL Errors (that are hidden by default) |
|
||||
| `HIDE_TEMPLATE_SIG_WARNING=true` | Hides Template Signature Verification Warnings |
|
||||
| `NUCLEI_LOG_ALL=true` | Log All Events that were skipped in verbose mode |
|
||||
| `NUCLEI_CONFIG_DIR` | Sets custom configuration directory path |
|
||||
| `NUCLEI_TEMPLATES_DIR` | Sets custom nuclei templates directory path |
|
||||
|
||||
|
||||
|
||||
@@ -145,7 +145,7 @@ type Exporter interface {
|
||||
}
|
||||
```
|
||||
|
||||
Exporters include `Elasticsearch`, `markdown`, `sarif` . Trackers include `GitHub` , `GitLab` and `Jira`.
|
||||
Exporters include `Elasticsearch`, `markdown`, `sarif` . Trackers include `GitHub`, `GitLab` and `Jira`.
|
||||
|
||||
Each exporter and trackers implement their own configuration in YAML format and are very modular in nature, so adding new ones is easy.
|
||||
|
||||
|
||||
@@ -75,7 +75,7 @@ syntax-docs: docgen
|
||||
syntax-docs:
|
||||
./bin/docgen SYNTAX-REFERENCE.md nuclei-jsonschema.json
|
||||
|
||||
test: GOFLAGS = -race -v
|
||||
test: GOFLAGS = -race -v -timeout 30m -count 1
|
||||
test:
|
||||
$(GOTEST) $(GOFLAGS) ./...
|
||||
|
||||
@@ -147,8 +147,6 @@ template-validate: build
|
||||
template-validate:
|
||||
./bin/nuclei -ut
|
||||
./bin/nuclei -validate \
|
||||
-et .github/ \
|
||||
-et helpers/payloads/ \
|
||||
-et http/technologies \
|
||||
-t dns \
|
||||
-t ssl \
|
||||
@@ -157,7 +155,5 @@ template-validate:
|
||||
-ept code
|
||||
./bin/nuclei -validate \
|
||||
-w workflows \
|
||||
-et .github/ \
|
||||
-et helpers/payloads/ \
|
||||
-et http/technologies \
|
||||
-ept code
|
||||
@@ -14,8 +14,6 @@
|
||||
|
||||
<p align="center">
|
||||
|
||||
<img src="https://img.shields.io/badge/go-1.22-%2300ADD8.svg?style=for-the-badge&logo=go&logoColor=white">
|
||||
|
||||
<a href="https://docs.projectdiscovery.io/tools/nuclei/overview?utm_source=github&utm_medium=web&utm_campaign=nuclei_readme"><img src="https://img.shields.io/badge/Documentation-%23000000.svg?style=for-the-badge&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJub25lIiBzdHJva2U9IiNmZmZmZmYiIHN0cm9rZS13aWR0aD0iMiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIiBjbGFzcz0ibHVjaWRlIGx1Y2lkZS1ib29rLW9wZW4iPjxwYXRoIGQ9Ik0xMiA3djE0Ii8+PHBhdGggZD0iTTMgMThhMSAxIDAgMCAxLTEtMVY0YTEgMSAwIDAgMSAxLTFoNWE0IDQgMCAwIDEgNCA0IDQgNCAwIDAgMSA0LTRoNWExIDEgMCAwIDEgMSAxdjEzYTEgMSAwIDAgMS0xIDFoLTZhMyAzIDAgMCAwLTMgMyAzIDMgMCAwIDAtMy0zeiIvPjwvc3ZnPg==&logoColor=white"></a>
|
||||
|
||||
<a href="https://github.com/projectdiscovery/nuclei-templates"><img src="https://img.shields.io/badge/Templates Library-%23000000.svg?style=for-the-badge&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIxNiIgaGVpZ2h0PSIxNiIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJub25lIiBzdHJva2U9IiNmZmZmZmYiIHN0cm9rZS13aWR0aD0iMS41IiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiIGNsYXNzPSJsdWNpZGUgbHVjaWRlLXNoaWVsZCI+PHBhdGggZD0iTTIwIDEzYzAgNS0zLjUgNy41LTcuNjYgOC45NWExIDEgMCAwIDEtLjY3LS4wMUM3LjUgMjAuNSA0IDE4IDQgMTNWNmExIDEgMCAwIDEgMS0xYzIgMCA0LjUtMS4yIDYuMjQtMi43MmExLjE3IDEuMTcgMCAwIDEgMS41MiAwQzE0LjUxIDMuODEgMTcgNSAxOSA1YTEgMSAwIDAgMSAxIDF6Ii8+PC9zdmc+&logoColor=white"></a>
|
||||
@@ -35,7 +33,7 @@
|
||||
- Reduce false positives by simulating real-world steps to verify a vulnerability.
|
||||
- Ultra-fast parallel scan processing and request clustering.
|
||||
- Integrate into CI/CD pipelines for vulnerability detection and regression testing.
|
||||
- Supports multiple protocols like TCP, DNS, HTTP, SSL, WHOIS JavaScript, Code and more.
|
||||
- Supports multiple protocols like TCP, DNS, HTTP, SSL, WHOIS, JavaScript, Code and more.
|
||||
- Integrate with Jira, Splunk, GitHub, Elastic, GitLab.
|
||||
|
||||
<br>
|
||||
@@ -85,7 +83,7 @@ _For security teams and enterprises, we provide a cloud-hosted service built on
|
||||
|
||||
- 50x faster scans
|
||||
- Large scale scanning with high accuracy
|
||||
- Integrations with cloud services (AWS, GCP, Azure, CloudFlare, Fastly, Terraform, Kubernetes)
|
||||
- Integrations with cloud services (AWS, GCP, Azure, Cloudflare, Fastly, Terraform, Kubernetes)
|
||||
- Jira, Slack, Linear, APIs and Webhooks
|
||||
- Executive and compliance reporting
|
||||
- Plus: Real-time scanning, SAML SSO, SOC 2 compliant platform (with EU and US hosting options), shared team workspaces, and more
|
||||
@@ -99,7 +97,7 @@ _For security teams and enterprises, we provide a cloud-hosted service built on
|
||||
|
||||
## Documentation
|
||||
|
||||
Browse the full Nuclei [**`documentation here`**](https://docs.projectdiscovery.io/tools/nuclei/running). If you’re new to Nuclei, check out our [**`foundational Youtube series`**](https://www.youtube.com/playlist?list=PLZRbR9aMzTTpItEdeNSulo8bYsvil80Rl).
|
||||
Browse the full Nuclei [**`documentation here`**](https://docs.projectdiscovery.io/tools/nuclei/running). If you’re new to Nuclei, check out our [**`foundational YouTube series`**](https://www.youtube.com/playlist?list=PLZRbR9aMzTTpItEdeNSulo8bYsvil80Rl).
|
||||
|
||||
<div align="center">
|
||||
|
||||
@@ -111,7 +109,7 @@ Browse the full Nuclei [**`documentation here`**](https://docs.projectdiscovery.
|
||||
|
||||
### Installation
|
||||
|
||||
`nuclei` requires **go1.23** to install successfully. Run the following command to get the repo:
|
||||
`nuclei` requires **go >= 1.24.2** to install successfully. Run the following command to get the repo:
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
@@ -142,7 +140,7 @@ TARGET:
|
||||
-u, -target string[] target URLs/hosts to scan
|
||||
-l, -list string path to file containing a list of target URLs/hosts to scan (one per line)
|
||||
-eh, -exclude-hosts string[] hosts to exclude to scan from the input list (ip, cidr, hostname)
|
||||
-resume string resume scan using resume.cfg (clustering will be disabled)
|
||||
-resume string resume scan from and save to specified file (clustering will be disabled)
|
||||
-sa, -scan-all-ips scan all the IP's associated with dns record
|
||||
-iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4)
|
||||
|
||||
@@ -163,7 +161,7 @@ TEMPLATES:
|
||||
-validate validate the passed templates to nuclei
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all available templates
|
||||
-tl list all templates matching current filters
|
||||
-tgl list all available tags
|
||||
-sign signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable
|
||||
-code enable loading code protocol-based templates
|
||||
@@ -286,6 +284,7 @@ RATE-LIMIT:
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
-timeout int time to wait in seconds before timeout (default 10)
|
||||
@@ -356,6 +355,7 @@ CLOUD:
|
||||
AUTHENTICATION:
|
||||
-sf, -secret-file string[] path to config file containing secrets for nuclei authenticated scan
|
||||
-ps, -prefetch-secrets prefetch secrets from the secrets file
|
||||
# NOTE: Headers in secrets files preserve exact casing (useful for case-sensitive APIs)
|
||||
|
||||
|
||||
EXAMPLES:
|
||||
|
||||
+4
-3
@@ -59,7 +59,7 @@ Nuclei使用零误报的定制模板向目标发送请求,同时可以对主
|
||||
|
||||
# 安装Nuclei
|
||||
|
||||
Nuclei需要 **go1.22** 才能安装成功。执行下列命令安装最新版本的Nuclei
|
||||
Nuclei需要 **go1.24.2** 才能安装成功。执行下列命令安装最新版本的Nuclei
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
@@ -119,7 +119,7 @@ Nuclei是一款注重于可配置性、可扩展性和易用性的基于模板
|
||||
目标:
|
||||
-u, -target string[] 指定扫描的目标URL/主机(多个目标则指定多个-u参数)
|
||||
-l, -list string 指定包含要扫描的目标URL/主机列表的文件路径(一行一个)
|
||||
-resume string 使用指定的resume.cfg文件恢复扫描(将禁用请求聚类)
|
||||
-resume string 从指定文件恢复扫描并保存到指定文件(将禁用请求聚类)
|
||||
-sa, -scan-all-ips 扫描由目标解析出来的所有IP(针对域名对应多个IP的情况)
|
||||
-iv, -ip-version string[] 要扫描的主机名的IP版本(4,6)-(默认为4)
|
||||
|
||||
@@ -238,6 +238,7 @@ UNCOVER引擎:
|
||||
-c, -concurrency int 并行执行的最大模板数量(默认:25)
|
||||
-hbs, -headless-bulk-size int 每个模板并行运行的无头主机最大数量(默认:10)
|
||||
-headc, -headless-concurrency int 并行指定无头主机最大数量(默认:10)
|
||||
-tlc, -template-loading-concurrency int 最大并发模板加载操作数(默认:50)
|
||||
|
||||
|
||||
优化:
|
||||
@@ -370,7 +371,7 @@ Nuclei允许您定制自己的测试方法,可以轻松的运行您的程序
|
||||
- 可以在几分钟处理上千台主机
|
||||
- 使用YAML语法定制自动化测试
|
||||
|
||||
欢迎查看我们其他的开源项目,可能有适合您的赏金猎人工作流:[github.com/projectdiscovery](http://github.com/projectdiscovery),我们还使用[Chaos绘制了每日的DNS数据](http://chaos.projectdiscovery.io)。
|
||||
欢迎查看我们其他的开源项目,可能有适合您的赏金猎人工作流:[github.com/projectdiscovery](https://github.com/projectdiscovery),我们还使用[Chaos绘制了每日的DNS数据](https://chaos.projectdiscovery.io)。
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
+3
-2
@@ -56,7 +56,7 @@ Tenemos un [repositorio dedicado](https://github.com/projectdiscovery/nuclei-tem
|
||||
|
||||
# Instalación de Nuclei
|
||||
|
||||
Nuclei requiere **go1.22** para instalarse correctamente. Ejecute el siguiente comando para instalar la última versión -
|
||||
Nuclei requiere **go1.24.2** para instalarse correctamente. Ejecute el siguiente comando para instalar la última versión -
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
@@ -118,7 +118,7 @@ TARGET:
|
||||
-u, -target string[] URLs/hosts a escanear
|
||||
-l, -list string ruta al archivo que contiene la lista de URLs/hosts a escanear (uno por línea)
|
||||
-eh, -exclude-hosts string[] hosts a excluir para escanear de la lista de entrada (ip, cidr, hostname)
|
||||
-resume string reanudar el escaneo usando resume.cfg (la clusterización quedará inhabilitada)
|
||||
-resume string reanudar el escaneo desde y guardar en el archivo especificado (la clusterización quedará inhabilitada)
|
||||
-sa, -scan-all-ips escanear todas las IP asociadas al registro dns
|
||||
-iv, -ip-version string[] versión IP a escanear del nombre de host (4,6) - (por defecto 4)
|
||||
|
||||
@@ -244,6 +244,7 @@ RATE-LIMIT:
|
||||
-headc, -headless-concurrency int número máximo de plantillas headless a ejecutar en paralelo (por defecto 10)
|
||||
-jsc, -js-concurrency int número máximo de entornos de ejecución de JavaScript a ejecutar en paralelo (por defecto 120)
|
||||
-pc, -payload-concurrency int concurrencia máxima de carga útil para cada plantilla (por defecto 25)
|
||||
-tlc, -template-loading-concurrency int número máximo de operaciones de carga de plantillas concurrentes (por defecto 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
-timeout int tiempo de espera en segundos (por defecto 10)
|
||||
|
||||
+4
-3
@@ -54,7 +54,7 @@ Kami memiliki [repositori khusus](https://github.com/projectdiscovery/nuclei-tem
|
||||
|
||||
# Instalasi Nuclei
|
||||
|
||||
Nuclei membutuhkan **go1.22** agar dapat diinstall. Jalankan perintah berikut untuk menginstal versi terbaru -
|
||||
Nuclei membutuhkan **go1.24.2** agar dapat diinstall. Jalankan perintah berikut untuk menginstal versi terbaru -
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
@@ -98,7 +98,7 @@ Flags:
|
||||
TARGET:
|
||||
-u, -target string[] target URLs/hosts to scan
|
||||
-l, -list string path to file containing a list of target URLs/hosts to scan (one per line)
|
||||
-resume string resume scan using resume.cfg (clustering will be disabled)
|
||||
-resume string resume scan from and save to specified file (clustering will be disabled)
|
||||
-sa, -scan-all-ips scan all the IP's associated with dns record
|
||||
-iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4)
|
||||
|
||||
@@ -211,6 +211,7 @@ RATE-LIMIT:
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
-timeout int time to wait in seconds before timeout (default 10)
|
||||
@@ -341,7 +342,7 @@ Nuclei memungkinkan Anda untuk menyesuaikan pendekatan pengujian Anda dengan ran
|
||||
- Dapat memproses ribuan host hanya dalam beberapa menit.
|
||||
- Mudah mengotomatiskan pendekatan pengujian khusus Anda dengan sintaks DSL berbasis YAML sederhana kami.
|
||||
|
||||
Silakan periksa proyek sumber terbuka kami yang lain yang mungkin cocok dengan alur kerja celah berhadiah Anda: [github.com/projectdiscovery](http://github.com/projectdiscovery), kami juga menyediakan [penyegaran data DNS di Chaos setiap hari](http://chaos.projectdiscovery.io).
|
||||
Silakan periksa proyek sumber terbuka kami yang lain yang mungkin cocok dengan alur kerja celah berhadiah Anda: [github.com/projectdiscovery](https://github.com/projectdiscovery), kami juga menyediakan [penyegaran data DNS di Chaos setiap hari](https://chaos.projectdiscovery.io).
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
+2
-2
@@ -53,7 +53,7 @@ Nucleiは、テンプレートに基づいてターゲット間でリクエス
|
||||
|
||||
# Nucleiのインストール
|
||||
|
||||
Nucleiを正常にインストールするには、**go1.22**が必要です。以下のコマンドを実行して最新バージョンをインストールしてください -
|
||||
Nucleiを正常にインストールするには、**go1.24.2**が必要です。以下のコマンドを実行して最新バージョンをインストールしてください -
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
@@ -113,7 +113,7 @@ Nucleiは、広範な設定可能性、大規模な拡張性、および使い
|
||||
ターゲット:
|
||||
-u, -target string[] スキャンする対象のURL/ホスト
|
||||
-l, -list string スキャンする対象のURL/ホストのリストが含まれているファイルへのパス(1行に1つ)
|
||||
-resume string resume.cfgを使用してスキャンを再開(クラスタリングは無効になります)
|
||||
-resume string 指定されたファイルからスキャンを再開し、指定されたファイルに保存(クラスタリングは無効になります)
|
||||
-sa, -scan-all-ips DNSレコードに関連付けられているすべてのIPをスキャン
|
||||
-iv, -ip-version string[] ホスト名のスキャンするIPバージョン(4,6)-(デフォルトは4)
|
||||
|
||||
|
||||
+4
-3
@@ -52,7 +52,7 @@ Nuclei는 템플릿을 기반으로 대상 간에 요청을 보내기 위해 사
|
||||
|
||||
# 설치
|
||||
|
||||
Nuclei를 성공적으로 설치하기 위해서 **go1.22**가 필요합니다. 다음 명령을 실행하여 최신 버전을 설치합니다.
|
||||
Nuclei를 성공적으로 설치하기 위해서 **go1.24.2**가 필요합니다. 다음 명령을 실행하여 최신 버전을 설치합니다.
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
@@ -96,7 +96,7 @@ Nuclei는 빠르고, 템플릿 기반의 취약점 스캐너로
|
||||
TARGET:
|
||||
-u, -target string[] 스캔할 대상 URL/호스트
|
||||
-l, -list string 스캔할 대상 URL/호스트 목록이 있는 파일 경로 (한 줄에 하나씩)
|
||||
-resume string resume.cfg를 사용하여 스캔 재개 (클러스터링은 비활성화됨)
|
||||
-resume string 지정된 파일에서 스캔을 재개하고 지정된 파일에 저장 (클러스터링은 비활성화됨)
|
||||
-sa, -scan-all-ips dns 레코드와 관련된 모든 IP 스캔
|
||||
-iv, -ip-version string[] 스캔할 호스트의 IP 버전 (4,6) - (기본값 4)
|
||||
|
||||
@@ -209,6 +209,7 @@ RATE-LIMIT:
|
||||
-c, -concurrency int 병렬로 실행할 최대 템플릿 수 (기본값 25)
|
||||
-hbs, -headless-bulk-size int 템플릿당 병렬로 분석할 최대 headless 호스트 수 (기본값 10)
|
||||
-headc, -headless-concurrency int 병렬로 실행할 최대 headless 템플릿 수 (기본값 10)
|
||||
-tlc, -template-loading-concurrency int 최대 동시 템플릿 로딩 작업 수 (기본값 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
-timeout int 타임아웃 전에 기다릴 초 수 (기본값 10)
|
||||
@@ -341,7 +342,7 @@ Nuclei를 사용하면 자체 검사 모음으로 테스트 접근 방식을 사
|
||||
- 몇 분 안에 수천 개의 호스트를 처리할 수 있음.
|
||||
- 간단한 YAML DSL로 사용자 지정 테스트 접근 방식을 쉽게 자동화할 수 있음.
|
||||
|
||||
버그 바운티 워크플로에 맞는 다른 오픈 소스 프로젝트를 확인할 수 있습니다.: [github.com/projectdiscovery](http://github.com/projectdiscovery), 또한, 우리는 매일 [Chaos에서 DNS 데이터를 갱신해 호스팅합니다](http://chaos.projectdiscovery.io).
|
||||
버그 바운티 워크플로에 맞는 다른 오픈 소스 프로젝트를 확인할 수 있습니다.: [github.com/projectdiscovery](https://github.com/projectdiscovery), 또한, 우리는 매일 [Chaos에서 DNS 데이터를 갱신해 호스팅합니다](https://chaos.projectdiscovery.io).
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
+3
-2
@@ -56,7 +56,7 @@ Temos um [repositório dedicado](https://github.com/projectdiscovery/nuclei-temp
|
||||
|
||||
# Instalação do Nuclei
|
||||
|
||||
O Nuclei requer **go1.22** para ser instalado corretamente. Execute o seguinte comando para instalar a versão mais recente:
|
||||
O Nuclei requer **go1.24.2** para ser instalado corretamente. Execute o seguinte comando para instalar a versão mais recente:
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
@@ -118,7 +118,7 @@ TARGET:
|
||||
-u, -target string[] URLs/hosts a serem escaneados
|
||||
-l, -list string caminho do arquivo contendo a lista de URLs/hosts a serem escaneados (um por linha)
|
||||
-eh, -exclude-hosts string[] hosts a serem excluídos do escaneamento na lista de entrada (ip, cidr, hostname)
|
||||
-resume string retomar o escaneamento usando resume.cfg (a clusterização será desabilitada)
|
||||
-resume string retomar o escaneamento a partir de e salvar no arquivo especificado (a clusterização será desabilitada)
|
||||
-sa, -scan-all-ips escanear todos os IPs associados ao registro DNS
|
||||
-iv, -ip-version string[] versão de IP a escanear do nome do host (4,6) - (padrão 4)
|
||||
|
||||
@@ -244,6 +244,7 @@ RATE-LIMIT:
|
||||
-headc, -headless-concurrency int número máximo de templates headless a serem executados em paralelo (padrão 10)
|
||||
-jsc, -js-concurrency int número máximo de ambientes de execução de JavaScript a serem executados em paralelo (padrão 120)
|
||||
-pc, -payload-concurrency int concorrência máxima de payload para cada template (padrão 25)
|
||||
-tlc, -template-loading-concurrency int número máximo de operações de carregamento de templates concorrentes (padrão 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
-timeout int tempo limite em segundos (padrão 10)
|
||||
|
||||
+2
-2
@@ -3218,7 +3218,7 @@ Inputs contains inputs for the network socket
|
||||
<div class="dt">
|
||||
|
||||
description: |
|
||||
Port is the port to send network requests to. this acts as default port but is overriden if target/input contains
|
||||
Port is the port to send network requests to. this acts as default port but is overridden if target/input contains
|
||||
non-http(s) ports like 80,8080,8081 etc
|
||||
|
||||
</div>
|
||||
@@ -4512,7 +4512,7 @@ permutations and combinations for all payloads.
|
||||
</div>
|
||||
<div class="dt">
|
||||
|
||||
Payload concurreny i.e threads for sending requests.
|
||||
Payload concurrency i.e threads for sending requests.
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -36,24 +36,24 @@
|
||||
{{binary}} -tags cve,dos,fuzz
|
||||
{{binary}} -tags cve -include-tags dos,fuzz
|
||||
{{binary}} -tags cve -exclude-tags cve2020
|
||||
{{binary}} -tags cve -exclude-templates cves/2020/
|
||||
{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml
|
||||
{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml -exclude-templates cves/2021/
|
||||
{{binary}} -t cves/
|
||||
{{binary}} -t cves/ -t exposures/
|
||||
{{binary}} -t cves/ -t exposures/ -tags config
|
||||
{{binary}} -t cves/ -t exposures/ -tags config,ssrf
|
||||
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical
|
||||
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam
|
||||
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli
|
||||
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2021/
|
||||
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2017/CVE-2017-7269.yaml
|
||||
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -include-templates cves/2017/CVE-2017-7269.yaml
|
||||
{{binary}} -tags cve -exclude-templates http/cves/2020/
|
||||
{{binary}} -tags cve -exclude-templates http/cves/2020/CVE-2020-9757.yaml
|
||||
{{binary}} -tags cve -exclude-templates http/cves/2020/CVE-2020-9757.yaml -exclude-templates http/cves/2021/
|
||||
{{binary}} -t http/cves/
|
||||
{{binary}} -t http/cves/ -t http/exposures/
|
||||
{{binary}} -t http/cves/ -t http/exposures/ -tags config
|
||||
{{binary}} -t http/cves/ -t http/exposures/ -tags config,ssrf
|
||||
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical
|
||||
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam
|
||||
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli
|
||||
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates http/cves/2021/
|
||||
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates http/cves/2017/CVE-2017-7269.yaml
|
||||
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -include-templates http/cves/2017/CVE-2017-7269.yaml
|
||||
|
||||
# Advanced Filtering
|
||||
{{binary}} -tags cve -author geeknik,pdteam -tc severity=='high'
|
||||
{{binary}} -tc contains(authors,'pdteam')
|
||||
{{binary}} -t cves/ -t exposures/ -tc contains(tags,'cve') -exclude-templates cves/2020/CVE-2020-9757.yaml
|
||||
{{binary}} -t http/cves/ -t http/exposures/ -tc contains(tags,'cve') -exclude-templates http/cves/2020/CVE-2020-9757.yaml
|
||||
{{binary}} -tc protocol=='dns'
|
||||
{{binary}} -tc contains(http_method,'GET')
|
||||
{{binary}} -tc len(body)>0
|
||||
@@ -65,7 +65,7 @@
|
||||
{{binary}} -w workflows
|
||||
{{binary}} -w workflows -author geeknik,pdteam
|
||||
{{binary}} -w workflows -severity high,critical
|
||||
{{binary}} -w workflows -author geeknik,pdteam -severity high,critical
|
||||
{{binary}} -w workflows -author geeknik,pdteam -severity high,critical
|
||||
|
||||
# Input Types
|
||||
# http protocol
|
||||
|
||||
@@ -24,6 +24,8 @@ var codeTestCases = []TestCaseInfo{
|
||||
{Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled},
|
||||
{Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() || isCodeDisabled() }},
|
||||
{Path: "protocols/code/pre-condition.yaml", TestCase: &codePreCondition{}, DisableOn: isCodeDisabled},
|
||||
{Path: "protocols/code/sh-virtual.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsLinux() || isCodeDisabled() }},
|
||||
{Path: "protocols/code/py-virtual.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsLinux() || isCodeDisabled() }},
|
||||
}
|
||||
|
||||
const (
|
||||
|
||||
@@ -62,9 +62,11 @@ var httpTestcases = []TestCaseInfo{
|
||||
{Path: "protocols/http/dsl-functions.yaml", TestCase: &httpDSLFunctions{}},
|
||||
{Path: "protocols/http/race-simple.yaml", TestCase: &httpRaceSimple{}},
|
||||
{Path: "protocols/http/race-multiple.yaml", TestCase: &httpRaceMultiple{}},
|
||||
{Path: "protocols/http/race-with-variables.yaml", TestCase: &httpRaceWithVariables{}},
|
||||
{Path: "protocols/http/stop-at-first-match.yaml", TestCase: &httpStopAtFirstMatch{}},
|
||||
{Path: "protocols/http/stop-at-first-match-with-extractors.yaml", TestCase: &httpStopAtFirstMatchWithExtractors{}},
|
||||
{Path: "protocols/http/variables.yaml", TestCase: &httpVariables{}},
|
||||
{Path: "protocols/http/variables-threads-previous.yaml", TestCase: &httpVariablesThreadsPrevious{}},
|
||||
{Path: "protocols/http/variable-dsl-function.yaml", TestCase: &httpVariableDSLFunction{}},
|
||||
{Path: "protocols/http/get-override-sni.yaml", TestCase: &httpSniAnnotation{}},
|
||||
{Path: "protocols/http/get-sni.yaml", TestCase: &customCLISNI{}},
|
||||
@@ -77,6 +79,7 @@ var httpTestcases = []TestCaseInfo{
|
||||
{Path: "protocols/http/cl-body-without-header.yaml", TestCase: &httpCLBodyWithoutHeader{}},
|
||||
{Path: "protocols/http/cl-body-with-header.yaml", TestCase: &httpCLBodyWithHeader{}},
|
||||
{Path: "protocols/http/cli-with-constants.yaml", TestCase: &ConstantWithCliVar{}},
|
||||
{Path: "protocols/http/constants-with-threads.yaml", TestCase: &constantsWithThreads{}},
|
||||
{Path: "protocols/http/matcher-status.yaml", TestCase: &matcherStatusTest{}},
|
||||
{Path: "protocols/http/disable-path-automerge.yaml", TestCase: &httpDisablePathAutomerge{}},
|
||||
{Path: "protocols/http/http-preprocessor.yaml", TestCase: &httpPreprocessor{}},
|
||||
@@ -1153,6 +1156,26 @@ func (h *httpRaceMultiple) Execute(filePath string) error {
|
||||
return expectResultsCount(results, 5)
|
||||
}
|
||||
|
||||
type httpRaceWithVariables struct{}
|
||||
|
||||
// Execute tests that variables and constants are properly resolved in race mode.
|
||||
func (h *httpRaceWithVariables) Execute(filePath string) error {
|
||||
router := httprouter.New()
|
||||
router.GET("/race", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
// Echo back the API key header so we can match on it
|
||||
_, _ = fmt.Fprint(w, r.Header.Get("X-API-Key"))
|
||||
})
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return expectResultsCount(results, 3)
|
||||
}
|
||||
|
||||
type httpStopAtFirstMatch struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
@@ -1220,6 +1243,30 @@ func (h *httpVariables) Execute(filePath string) error {
|
||||
return expectResultsCount(results, 0)
|
||||
}
|
||||
|
||||
type httpVariablesThreadsPrevious struct{}
|
||||
|
||||
// Execute tests that variables can reference data extracted from previous requests
|
||||
// when using threads mode (parallel execution).
|
||||
func (h *httpVariablesThreadsPrevious) Execute(filePath string) error {
|
||||
router := httprouter.New()
|
||||
router.GET("/login", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
_, _ = fmt.Fprint(w, "token=secret123")
|
||||
})
|
||||
router.GET("/api", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
// Echo back the Authorization header so we can match on it
|
||||
_, _ = fmt.Fprint(w, r.Header.Get("Authorization"))
|
||||
})
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
||||
type httpVariableDSLFunction struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
@@ -1466,6 +1513,26 @@ func (h *ConstantWithCliVar) Execute(filePath string) error {
|
||||
return expectResultsCount(got, 1)
|
||||
}
|
||||
|
||||
type constantsWithThreads struct{}
|
||||
|
||||
// Execute tests that constants are properly resolved when using threads mode.
|
||||
func (h *constantsWithThreads) Execute(filePath string) error {
|
||||
router := httprouter.New()
|
||||
router.GET("/api/:version", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
|
||||
// Echo back the API key header and version so we can match on them
|
||||
_, _ = fmt.Fprintf(w, "%s %s", r.Header.Get("X-API-Key"), p.ByName("version"))
|
||||
})
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
||||
type matcherStatusTest struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"os"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/kitabisa/go-ci"
|
||||
@@ -24,7 +25,7 @@ type TestCaseInfo struct {
|
||||
}
|
||||
|
||||
var (
|
||||
debug = os.Getenv("DEBUG") == "true"
|
||||
debug = isDebugMode()
|
||||
customTests = os.Getenv("TESTS")
|
||||
protocol = os.Getenv("PROTO")
|
||||
|
||||
@@ -46,6 +47,7 @@ var (
|
||||
"library": libraryTestcases,
|
||||
"templatesPath": templatesPathTestCases,
|
||||
"templatesDir": templatesDirTestCases,
|
||||
"env_vars": templatesDirEnvTestCases,
|
||||
"file": fileTestcases,
|
||||
"offlineHttp": offlineHttpTestcases,
|
||||
"customConfigDir": customConfigDirTestCases,
|
||||
@@ -59,6 +61,7 @@ var (
|
||||
"matcher-status": matcherStatusTestcases,
|
||||
"exporters": exportersTestCases,
|
||||
}
|
||||
|
||||
// flakyTests are run with a retry count of 3
|
||||
flakyTests = map[string]bool{
|
||||
"protocols/http/self-contained-file-input.yaml": true,
|
||||
@@ -89,11 +92,12 @@ func main() {
|
||||
}
|
||||
|
||||
// start fuzz playground server
|
||||
defer fuzzplayground.Cleanup()
|
||||
server := fuzzplayground.GetPlaygroundServer()
|
||||
defer func() {
|
||||
fuzzplayground.Cleanup()
|
||||
_ = server.Close()
|
||||
}()
|
||||
|
||||
go func() {
|
||||
if err := server.Start("localhost:8082"); err != nil {
|
||||
if !strings.Contains(err.Error(), "Server closed") {
|
||||
@@ -103,7 +107,6 @@ func main() {
|
||||
}()
|
||||
|
||||
customTestsList := normalizeSplit(customTests)
|
||||
|
||||
failedTestTemplatePaths := runTests(customTestsList)
|
||||
|
||||
if len(failedTestTemplatePaths) > 0 {
|
||||
@@ -130,6 +133,27 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// isDebugMode checks if debug mode is enabled via any of the supported debug
|
||||
// environment variables.
|
||||
func isDebugMode() bool {
|
||||
debugEnvVars := []string{
|
||||
"DEBUG",
|
||||
"ACTIONS_RUNNER_DEBUG", // GitHub Actions runner debug
|
||||
// Add more debug environment variables here as needed
|
||||
}
|
||||
|
||||
truthyValues := []string{"true", "1", "yes", "on", "enabled"}
|
||||
|
||||
for _, envVar := range debugEnvVars {
|
||||
envValue := strings.ToLower(strings.TrimSpace(os.Getenv(envVar)))
|
||||
if slices.Contains(truthyValues, envValue) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// execute a testcase with retry and consider best of N
|
||||
// intended for flaky tests like interactsh
|
||||
func executeWithRetry(testCase testutils.TestCase, templatePath string, retryCount int) (string, error) {
|
||||
|
||||
@@ -15,13 +15,18 @@ var jsTestcases = []TestCaseInfo{
|
||||
{Path: "protocols/javascript/ssh-server-fingerprint.yaml", TestCase: &javascriptSSHServerFingerprint{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
|
||||
{Path: "protocols/javascript/net-multi-step.yaml", TestCase: &networkMultiStep{}},
|
||||
{Path: "protocols/javascript/net-https.yaml", TestCase: &javascriptNetHttps{}},
|
||||
{Path: "protocols/javascript/oracle-auth-test.yaml", TestCase: &javascriptOracleAuthTest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
|
||||
{Path: "protocols/javascript/vnc-pass-brute.yaml", TestCase: &javascriptVncPassBrute{}},
|
||||
{Path: "protocols/javascript/multi-ports.yaml", TestCase: &javascriptMultiPortsSSH{}},
|
||||
}
|
||||
|
||||
var (
|
||||
redisResource *dockertest.Resource
|
||||
sshResource *dockertest.Resource
|
||||
pool *dockertest.Pool
|
||||
defaultRetry = 3
|
||||
redisResource *dockertest.Resource
|
||||
sshResource *dockertest.Resource
|
||||
oracleResource *dockertest.Resource
|
||||
vncResource *dockertest.Resource
|
||||
pool *dockertest.Pool
|
||||
defaultRetry = 3
|
||||
)
|
||||
|
||||
type javascriptNetHttps struct{}
|
||||
@@ -98,6 +103,82 @@ func (j *javascriptSSHServerFingerprint) Execute(filePath string) error {
|
||||
return multierr.Combine(errs...)
|
||||
}
|
||||
|
||||
type javascriptOracleAuthTest struct{}
|
||||
|
||||
func (j *javascriptOracleAuthTest) Execute(filePath string) error {
|
||||
if oracleResource == nil || pool == nil {
|
||||
// skip test as oracle is not running
|
||||
return nil
|
||||
}
|
||||
tempPort := oracleResource.GetPort("1521/tcp")
|
||||
finalURL := "localhost:" + tempPort
|
||||
defer purge(oracleResource)
|
||||
|
||||
errs := []error{}
|
||||
for i := 0; i < defaultRetry; i++ {
|
||||
results := []string{}
|
||||
var err error
|
||||
_ = pool.Retry(func() error {
|
||||
//let ssh server start
|
||||
time.Sleep(3 * time.Second)
|
||||
results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := expectResultsCount(results, 1); err == nil {
|
||||
return nil
|
||||
} else {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return multierr.Combine(errs...)
|
||||
}
|
||||
|
||||
type javascriptVncPassBrute struct{}
|
||||
|
||||
func (j *javascriptVncPassBrute) Execute(filePath string) error {
|
||||
if vncResource == nil || pool == nil {
|
||||
// skip test as vnc is not running
|
||||
return nil
|
||||
}
|
||||
tempPort := vncResource.GetPort("5900/tcp")
|
||||
finalURL := "localhost:" + tempPort
|
||||
defer purge(vncResource)
|
||||
errs := []error{}
|
||||
for i := 0; i < defaultRetry; i++ {
|
||||
results := []string{}
|
||||
var err error
|
||||
_ = pool.Retry(func() error {
|
||||
//let ssh server start
|
||||
time.Sleep(3 * time.Second)
|
||||
results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := expectResultsCount(results, 1); err == nil {
|
||||
return nil
|
||||
} else {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return multierr.Combine(errs...)
|
||||
}
|
||||
|
||||
type javascriptMultiPortsSSH struct{}
|
||||
|
||||
func (j *javascriptMultiPortsSSH) Execute(filePath string) error {
|
||||
// use scanme.sh as target to ensure we match on the 2nd default port 22
|
||||
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
||||
// purge any given resource if it is not nil
|
||||
func purge(resource *dockertest.Resource) {
|
||||
if resource != nil && pool != nil {
|
||||
@@ -163,4 +244,41 @@ func init() {
|
||||
if err := sshResource.Expire(30); err != nil {
|
||||
log.Printf("Could not expire resource: %s", err)
|
||||
}
|
||||
|
||||
// setup a temporary oracle instance
|
||||
oracleResource, err = pool.RunWithOptions(&dockertest.RunOptions{
|
||||
Repository: "gvenzl/oracle-xe",
|
||||
Tag: "latest",
|
||||
Env: []string{
|
||||
"ORACLE_PASSWORD=mysecret",
|
||||
},
|
||||
Platform: "linux/amd64",
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Could not start Oracle resource: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
// by default expire after 30 sec
|
||||
if err := oracleResource.Expire(30); err != nil {
|
||||
log.Printf("Could not expire Oracle resource: %s", err)
|
||||
}
|
||||
|
||||
// setup a temporary vnc server
|
||||
vncResource, err = pool.RunWithOptions(&dockertest.RunOptions{
|
||||
Repository: "dorowu/ubuntu-desktop-lxde-vnc",
|
||||
Tag: "latest",
|
||||
Env: []string{
|
||||
"VNC_PASSWORD=mysecret",
|
||||
},
|
||||
Platform: "linux/amd64",
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Could not start resource: %s", err)
|
||||
return
|
||||
}
|
||||
// by default expire after 30 sec
|
||||
if err := vncResource.Expire(30); err != nil {
|
||||
log.Printf("Could not expire resource: %s", err)
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/logrusorgru/aurora"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
|
||||
@@ -70,6 +71,7 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error)
|
||||
|
||||
defaultOpts := types.DefaultOptions()
|
||||
defaultOpts.ExecutionId = "test"
|
||||
defaultOpts.Logger = gologger.DefaultLogger
|
||||
|
||||
mockProgress := &testutils.MockProgressClient{}
|
||||
reportingClient, err := reporting.New(&reporting.Options{ExecutionId: defaultOpts.ExecutionId}, "", false)
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
osutils "github.com/projectdiscovery/utils/os"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
)
|
||||
|
||||
// isNotLinux returns true if not running on Linux (used to skip tests on non-Linux OS)
|
||||
var isNotLinux = func() bool { return !osutils.IsLinux() }
|
||||
|
||||
var templatesDirEnvTestCases = []TestCaseInfo{
|
||||
{Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templatesDirEnvBasicTest{}, DisableOn: isNotLinux},
|
||||
{Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templatesDirEnvAbsolutePathTest{}, DisableOn: isNotLinux},
|
||||
{Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templatesDirEnvRelativePathTest{}, DisableOn: isNotLinux},
|
||||
{Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templatesDirEnvPrecedenceTest{}, DisableOn: isNotLinux},
|
||||
{Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templatesDirEnvCustomTemplatesTest{}, DisableOn: isNotLinux},
|
||||
}
|
||||
|
||||
// copyTemplateToDir copies a template file to a destination directory, preserving the directory structure
|
||||
func copyTemplateToDir(templatePath, destDir string) error {
|
||||
// Read the template file
|
||||
templateData, err := os.ReadFile(templatePath)
|
||||
if err != nil {
|
||||
return errkit.Wrap(err, "failed to read template file")
|
||||
}
|
||||
|
||||
// Create the destination path preserving directory structure
|
||||
destPath := filepath.Join(destDir, templatePath)
|
||||
destDirPath := filepath.Dir(destPath)
|
||||
|
||||
// Create the destination directory if it doesn't exist
|
||||
if err := os.MkdirAll(destDirPath, 0755); err != nil {
|
||||
return errkit.Wrap(err, "failed to create destination directory")
|
||||
}
|
||||
|
||||
// Write the template file
|
||||
if err := os.WriteFile(destPath, templateData, 0644); err != nil {
|
||||
return errkit.Wrap(err, "failed to write template file")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// templatesDirEnvBasicTest tests basic functionality of NUCLEI_TEMPLATES_DIR
|
||||
type templatesDirEnvBasicTest struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *templatesDirEnvBasicTest) Execute(filePath string) error {
|
||||
tempdir, err := os.MkdirTemp("", "nuclei-templates-dir-env-*")
|
||||
if err != nil {
|
||||
return errkit.Wrap(err, "failed to create temp dir")
|
||||
}
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tempdir)
|
||||
}()
|
||||
|
||||
// Copy template to temp directory
|
||||
if err := copyTemplateToDir(filePath, tempdir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set NUCLEI_TEMPLATES_DIR and run nuclei
|
||||
envVars := []string{"NUCLEI_TEMPLATES_DIR=" + tempdir}
|
||||
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, envVars, "-t", filePath, "-u", "8x8exch02.8x8.com")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
||||
// templatesDirEnvAbsolutePathTest tests that absolute paths work correctly
|
||||
type templatesDirEnvAbsolutePathTest struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *templatesDirEnvAbsolutePathTest) Execute(filePath string) error {
|
||||
tempdir, err := os.MkdirTemp("", "nuclei-templates-dir-env-abs-*")
|
||||
if err != nil {
|
||||
return errkit.Wrap(err, "failed to create temp dir")
|
||||
}
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tempdir)
|
||||
}()
|
||||
|
||||
// Get absolute path
|
||||
absTempDir, err := filepath.Abs(tempdir)
|
||||
if err != nil {
|
||||
return errkit.Wrap(err, "failed to get absolute path")
|
||||
}
|
||||
|
||||
// Copy template to temp directory
|
||||
if err := copyTemplateToDir(filePath, absTempDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set NUCLEI_TEMPLATES_DIR with absolute path and run nuclei
|
||||
envVars := []string{"NUCLEI_TEMPLATES_DIR=" + absTempDir}
|
||||
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, envVars, "-t", filePath, "-u", "8x8exch02.8x8.com")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
||||
// templatesDirEnvRelativePathTest tests that relative paths are resolved correctly
|
||||
type templatesDirEnvRelativePathTest struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *templatesDirEnvRelativePathTest) Execute(filePath string) error {
|
||||
// Create temp directory in current working directory
|
||||
tempdir, err := os.MkdirTemp(".", "nuclei-templates-dir-env-rel-*")
|
||||
if err != nil {
|
||||
return errkit.Wrap(err, "failed to create temp dir")
|
||||
}
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tempdir)
|
||||
}()
|
||||
|
||||
// Get relative path (just the directory name)
|
||||
relPath := filepath.Base(tempdir)
|
||||
|
||||
// Copy template to temp directory
|
||||
if err := copyTemplateToDir(filePath, tempdir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set NUCLEI_TEMPLATES_DIR with relative path and run nuclei
|
||||
// Note: The implementation should convert relative paths to absolute
|
||||
envVars := []string{"NUCLEI_TEMPLATES_DIR=" + relPath}
|
||||
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, envVars, "-t", filePath, "-u", "8x8exch02.8x8.com")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
||||
// templatesDirEnvPrecedenceTest tests that -ud flag takes precedence over NUCLEI_TEMPLATES_DIR
|
||||
type templatesDirEnvPrecedenceTest struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *templatesDirEnvPrecedenceTest) Execute(filePath string) error {
|
||||
// Create two temp directories
|
||||
envTempDir, err := os.MkdirTemp("", "nuclei-templates-dir-env-*")
|
||||
if err != nil {
|
||||
return errkit.Wrap(err, "failed to create env temp dir")
|
||||
}
|
||||
defer func() {
|
||||
_ = os.RemoveAll(envTempDir)
|
||||
}()
|
||||
|
||||
flagTempDir, err := os.MkdirTemp("", "nuclei-templates-dir-flag-*")
|
||||
if err != nil {
|
||||
return errkit.Wrap(err, "failed to create flag temp dir")
|
||||
}
|
||||
defer func() {
|
||||
_ = os.RemoveAll(flagTempDir)
|
||||
}()
|
||||
|
||||
// Copy template to flag temp directory (this should be used due to precedence)
|
||||
if err := copyTemplateToDir(filePath, flagTempDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set NUCLEI_TEMPLATES_DIR to envTempDir (should be ignored due to -ud flag)
|
||||
envVars := []string{"NUCLEI_TEMPLATES_DIR=" + envTempDir}
|
||||
// Use -ud flag which should take precedence
|
||||
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, envVars, "-t", filePath, "-u", "8x8exch02.8x8.com", "-ud", flagTempDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
||||
// templatesDirEnvCustomTemplatesTest tests that custom template subdirectories are correctly set
|
||||
type templatesDirEnvCustomTemplatesTest struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *templatesDirEnvCustomTemplatesTest) Execute(filePath string) error {
|
||||
tempdir, err := os.MkdirTemp("", "nuclei-templates-dir-custom-*")
|
||||
if err != nil {
|
||||
return errkit.Wrap(err, "failed to create temp dir")
|
||||
}
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tempdir)
|
||||
}()
|
||||
|
||||
// Create custom template subdirectories structure
|
||||
customDirs := []string{"github", "s3", "gitlab", "azure"}
|
||||
for _, dir := range customDirs {
|
||||
customDirPath := filepath.Join(tempdir, dir)
|
||||
if err := os.MkdirAll(customDirPath, 0755); err != nil {
|
||||
return errkit.Wrap(err, "failed to create custom template directory")
|
||||
}
|
||||
}
|
||||
|
||||
// Copy template to temp directory
|
||||
if err := copyTemplateToDir(filePath, tempdir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set NUCLEI_TEMPLATES_DIR and run nuclei
|
||||
envVars := []string{"NUCLEI_TEMPLATES_DIR=" + tempdir}
|
||||
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, envVars, "-t", filePath, "-u", "8x8exch02.8x8.com")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
+53
-5
@@ -20,6 +20,7 @@ import (
|
||||
_ "github.com/projectdiscovery/utils/pprof"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
"github.com/rs/xid"
|
||||
"gopkg.in/yaml.v2"
|
||||
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/projectdiscovery/gologger/levels"
|
||||
@@ -193,8 +194,11 @@ func main() {
|
||||
})
|
||||
}
|
||||
|
||||
// Setup graceful exits
|
||||
// Setup filename for graceful exits
|
||||
resumeFileName := types.DefaultResumeFilePath()
|
||||
if options.Resume == "" {
|
||||
resumeFileName = options.Resume
|
||||
}
|
||||
c := make(chan os.Signal, 1)
|
||||
signal.Notify(c, os.Interrupt)
|
||||
go func() {
|
||||
@@ -254,7 +258,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.StringSliceVarP(&options.Targets, "target", "u", nil, "target URLs/hosts to scan", goflags.CommaSeparatedStringSliceOptions),
|
||||
flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
|
||||
flagSet.StringSliceVarP(&options.ExcludeTargets, "exclude-hosts", "eh", nil, "hosts to exclude to scan from the input list (ip, cidr, hostname)", goflags.FileCommaSeparatedStringSliceOptions),
|
||||
flagSet.StringVar(&options.Resume, "resume", "", "resume scan using resume.cfg (clustering will be disabled)"),
|
||||
flagSet.StringVar(&options.Resume, "resume", "", "resume scan from and save to specified file (clustering will be disabled)"),
|
||||
flagSet.BoolVarP(&options.ScanAllIPs, "scan-all-ips", "sa", false, "scan all the IP's associated with dns record"),
|
||||
flagSet.StringSliceVarP(&options.IPVersion, "ip-version", "iv", nil, "IP version to scan of hostname (4,6) - (default 4)", goflags.CommaSeparatedStringSliceOptions),
|
||||
)
|
||||
@@ -263,6 +267,8 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.StringVarP(&options.InputFileMode, "input-mode", "im", "list", fmt.Sprintf("mode of input file (%v)", provider.SupportedInputFormats())),
|
||||
flagSet.BoolVarP(&options.FormatUseRequiredOnly, "required-only", "ro", false, "use only required fields in input format when generating requests"),
|
||||
flagSet.BoolVarP(&options.SkipFormatValidation, "skip-format-validation", "sfv", false, "skip format validation (like missing vars) when parsing input file"),
|
||||
flagSet.BoolVarP(&options.VarsTextTemplating, "vars-text-templating", "vtt", false, "enable text templating for vars in input file (only for yaml input mode)"),
|
||||
flagSet.StringSliceVarP(&options.VarsFilePaths, "var-file-paths", "vfp", nil, "list of yaml file contained vars to inject into yaml input", goflags.CommaSeparatedStringSliceOptions),
|
||||
)
|
||||
|
||||
flagSet.CreateGroup("templates", "Templates",
|
||||
@@ -277,7 +283,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
|
||||
flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
|
||||
flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
|
||||
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
|
||||
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all templates matching current filters"),
|
||||
flagSet.BoolVar(&options.TagList, "tgl", false, "list all available tags"),
|
||||
flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"cloud.projectdiscovery.io"}, "allowed domain list to load remote templates from"),
|
||||
flagSet.BoolVar(&options.SignTemplates, "sign", false, "signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable"),
|
||||
@@ -408,6 +414,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.IntVarP(&options.JsConcurrency, "js-concurrency", "jsc", 120, "maximum number of javascript runtimes to be executed in parallel"),
|
||||
flagSet.IntVarP(&options.PayloadConcurrency, "payload-concurrency", "pc", 25, "max payload concurrency for each template"),
|
||||
flagSet.IntVarP(&options.ProbeConcurrency, "probe-concurrency", "prc", 50, "http probe concurrency with httpx"),
|
||||
flagSet.IntVarP(&options.TemplateLoadingConcurrency, "template-loading-concurrency", "tlc", types.DefaultTemplateLoadingConcurrency, "maximum number of concurrent template loading operations"),
|
||||
)
|
||||
flagSet.CreateGroup("optimization", "Optimizations",
|
||||
flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait in seconds before timeout"),
|
||||
@@ -571,6 +578,7 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
|
||||
config.DefaultConfig.SetConfigDir(customConfigDir)
|
||||
readFlagsConfig(flagSet)
|
||||
}
|
||||
|
||||
if cfgFile != "" {
|
||||
if !fileutil.FileExists(cfgFile) {
|
||||
options.Logger.Fatal().Msgf("given config file '%s' does not exist", cfgFile)
|
||||
@@ -579,9 +587,49 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
|
||||
if err := flagSet.MergeConfigFile(cfgFile); err != nil {
|
||||
options.Logger.Fatal().Msgf("Could not read config: %s\n", err)
|
||||
}
|
||||
|
||||
if !options.Vars.IsEmpty() {
|
||||
// Maybe we should add vars to the config file as well even if they are set via flags?
|
||||
file, err := os.Open(cfgFile)
|
||||
if err != nil {
|
||||
gologger.Fatal().Msgf("Could not open config file: %s\n", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
}()
|
||||
data := make(map[string]interface{})
|
||||
err = yaml.NewDecoder(file).Decode(&data)
|
||||
if err != nil {
|
||||
gologger.Fatal().Msgf("Could not decode config file: %s\n", err)
|
||||
}
|
||||
|
||||
variables := data["var"]
|
||||
if variables != nil {
|
||||
if varSlice, ok := variables.([]interface{}); ok {
|
||||
for _, value := range varSlice {
|
||||
if strVal, ok := value.(string); ok {
|
||||
err = options.Vars.Set(strVal)
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("Could not set variable from config file: %s\n", err)
|
||||
}
|
||||
} else {
|
||||
gologger.Warning().Msgf("Skipping non-string variable in config: %#v", value)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
gologger.Warning().Msgf("No 'var' section found in config file: %s", cfgFile)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
if options.NewTemplatesDirectory != "" {
|
||||
config.DefaultConfig.SetTemplatesDir(options.NewTemplatesDirectory)
|
||||
|
||||
templatesDir := options.NewTemplatesDirectory
|
||||
if templatesDir == "" {
|
||||
templatesDir = os.Getenv(config.NucleiTemplatesDirEnv)
|
||||
}
|
||||
if templatesDir != "" {
|
||||
config.DefaultConfig.SetTemplatesDir(templatesDir)
|
||||
}
|
||||
|
||||
defaultProfilesPath := filepath.Join(config.DefaultConfig.GetTemplateDir(), "profiles")
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
package main_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"runtime"
|
||||
"runtime/pprof"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -48,6 +52,31 @@ func TestMain(m *testing.M) {
|
||||
os.Exit(exitCode)
|
||||
}
|
||||
|
||||
// getUniqFilename generates a unique filename by appending .N if file exists
|
||||
// Similar to wget's behavior: file.cpu.prof, file.cpu.1.prof, file.cpu.2.prof, etc.
|
||||
func getUniqFilename(basePath string) string {
|
||||
if _, err := os.Stat(basePath); os.IsNotExist(err) {
|
||||
return basePath
|
||||
}
|
||||
|
||||
lastDot := strings.LastIndex(basePath, ".")
|
||||
var name, ext string
|
||||
if lastDot != -1 {
|
||||
name = basePath[:lastDot]
|
||||
ext = basePath[lastDot:]
|
||||
} else {
|
||||
name = basePath
|
||||
ext = ""
|
||||
}
|
||||
|
||||
for i := 1; ; i++ {
|
||||
newPath := fmt.Sprintf("%s.%d%s", name, i, ext)
|
||||
if _, err := os.Stat(newPath); os.IsNotExist(err) {
|
||||
return newPath
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getDefaultOptions() *types.Options {
|
||||
return &types.Options{
|
||||
RemoteTemplateDomainList: []string{"cloud.projectdiscovery.io"},
|
||||
@@ -106,24 +135,55 @@ func runEnumBenchmark(b *testing.B, options *types.Options) {
|
||||
}
|
||||
defer nucleiRunner.Close()
|
||||
|
||||
b.ResetTimer()
|
||||
benchNameSlug := strings.ReplaceAll(b.Name(), "/", "-")
|
||||
|
||||
// Start CPU profiling
|
||||
cpuProfileBase := fmt.Sprintf("%s.cpu.prof", benchNameSlug)
|
||||
cpuProfilePath := getUniqFilename(cpuProfileBase)
|
||||
cpuProfile, err := os.Create(cpuProfilePath)
|
||||
if err != nil {
|
||||
b.Fatalf("failed to create CPU profile: %s", err)
|
||||
}
|
||||
defer func() { _ = cpuProfile.Close() }()
|
||||
|
||||
if err := pprof.StartCPUProfile(cpuProfile); err != nil {
|
||||
b.Fatalf("failed to start CPU profile: %s", err)
|
||||
}
|
||||
defer pprof.StopCPUProfile()
|
||||
|
||||
b.ReportAllocs()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
for b.Loop() {
|
||||
if err := nucleiRunner.RunEnumeration(); err != nil {
|
||||
b.Fatalf("%s failed: %s", b.Name(), err)
|
||||
}
|
||||
}
|
||||
|
||||
b.StopTimer()
|
||||
|
||||
// Write heap profile
|
||||
heapProfileBase := fmt.Sprintf("%s.heap.prof", benchNameSlug)
|
||||
heapProfilePath := getUniqFilename(heapProfileBase)
|
||||
heapProfile, err := os.Create(heapProfilePath)
|
||||
if err != nil {
|
||||
b.Fatalf("failed to create heap profile: %s", err)
|
||||
}
|
||||
defer func() { _ = heapProfile.Close() }()
|
||||
|
||||
runtime.GC() // Force GC before heap profile
|
||||
if err := pprof.WriteHeapProfile(heapProfile); err != nil {
|
||||
b.Fatalf("failed to write heap profile: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRunEnumeration(b *testing.B) {
|
||||
// Default case: run enumeration with default options == all nuclei-templates
|
||||
// b.Run("Default", func(b *testing.B) {
|
||||
// options := getDefaultOptions()
|
||||
// options.Targets = []string{targetURL}
|
||||
b.Run("Default", func(b *testing.B) {
|
||||
options := getDefaultOptions()
|
||||
options.Targets = []string{targetURL}
|
||||
|
||||
// runEnumBenchmark(b, options)
|
||||
// })
|
||||
runEnumBenchmark(b, options)
|
||||
})
|
||||
|
||||
// Case: https://github.com/projectdiscovery/nuclei/pull/6258
|
||||
b.Run("Multiproto", func(b *testing.B) {
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
id: basic-dns-a-example
|
||||
|
||||
info:
|
||||
name: Test DNS A Query Template
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
dns:
|
||||
- name: "{{FQDN}}"
|
||||
type: SRV
|
||||
class: inet
|
||||
recursion: true
|
||||
retries: 3
|
||||
matchers:
|
||||
- type: word
|
||||
part: all
|
||||
words:
|
||||
- "SRV"
|
||||
@@ -1,11 +1,13 @@
|
||||
module github.com/projectdiscovery/nuclei/v3
|
||||
|
||||
go 1.24.1
|
||||
go 1.24.2
|
||||
|
||||
toolchain go1.24.4
|
||||
|
||||
require (
|
||||
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
|
||||
github.com/andygrunwald/go-jira v1.16.0
|
||||
github.com/antchfx/htmlquery v1.3.4
|
||||
github.com/andygrunwald/go-jira v1.16.1
|
||||
github.com/antchfx/htmlquery v1.3.5
|
||||
github.com/bluele/gcache v0.0.2
|
||||
github.com/go-playground/validator/v10 v10.26.0
|
||||
github.com/go-rod/rod v0.116.2
|
||||
@@ -16,16 +18,16 @@ require (
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/julienschmidt/httprouter v1.3.0
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible
|
||||
github.com/miekg/dns v1.1.66
|
||||
github.com/miekg/dns v1.1.68
|
||||
github.com/olekukonko/tablewriter v1.0.8
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/projectdiscovery/clistats v0.1.1
|
||||
github.com/projectdiscovery/fastdialer v0.4.7
|
||||
github.com/projectdiscovery/hmap v0.0.93
|
||||
github.com/projectdiscovery/fastdialer v0.4.20
|
||||
github.com/projectdiscovery/hmap v0.0.98
|
||||
github.com/projectdiscovery/interactsh v1.2.4
|
||||
github.com/projectdiscovery/rawhttp v0.1.90
|
||||
github.com/projectdiscovery/retryabledns v1.0.106
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.121
|
||||
github.com/projectdiscovery/retryabledns v1.0.111
|
||||
github.com/projectdiscovery/retryablehttp-go v1.1.1
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.6
|
||||
github.com/remeh/sizedwaitgroup v1.0.0
|
||||
github.com/rs/xid v1.6.0
|
||||
@@ -35,26 +37,28 @@ require (
|
||||
github.com/spf13/cast v1.9.2
|
||||
github.com/syndtr/goleveldb v1.0.0
|
||||
github.com/valyala/fasttemplate v1.2.2
|
||||
github.com/weppos/publicsuffix-go v0.50.0
|
||||
github.com/weppos/publicsuffix-go v0.50.1
|
||||
go.uber.org/multierr v1.11.0
|
||||
golang.org/x/net v0.43.0
|
||||
golang.org/x/net v0.47.0
|
||||
golang.org/x/oauth2 v0.30.0
|
||||
golang.org/x/text v0.28.0
|
||||
golang.org/x/text v0.31.0
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
)
|
||||
|
||||
require (
|
||||
code.gitea.io/sdk/gitea v0.21.0
|
||||
carvel.dev/ytt v0.52.0
|
||||
code.gitea.io/sdk/gitea v0.17.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0
|
||||
github.com/DataDog/gostackparse v0.7.0
|
||||
github.com/Masterminds/semver/v3 v3.4.0
|
||||
github.com/Masterminds/semver/v3 v3.2.1
|
||||
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057
|
||||
github.com/Mzack9999/goja v0.0.0-20250507184235-e46100e9c697
|
||||
github.com/Mzack9999/goja_nodejs v0.0.0-20250507184139-66bcbf65c883
|
||||
github.com/alexsnet/go-vnc v0.1.0
|
||||
github.com/alitto/pond v1.9.2
|
||||
github.com/antchfx/xmlquery v1.4.4
|
||||
github.com/antchfx/xpath v1.3.4
|
||||
github.com/antchfx/xpath v1.3.5
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.5
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.17
|
||||
@@ -71,7 +75,7 @@ require (
|
||||
github.com/getkin/kin-openapi v0.132.0
|
||||
github.com/go-git/go-git/v5 v5.16.2
|
||||
github.com/go-ldap/ldap/v3 v3.4.11
|
||||
github.com/go-pg/pg v8.0.7+incompatible
|
||||
github.com/go-pg/pg/v10 v10.15.0
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/goccy/go-json v0.10.5
|
||||
github.com/google/uuid v1.6.0
|
||||
@@ -83,34 +87,36 @@ require (
|
||||
github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/mattn/go-sqlite3 v1.14.28
|
||||
github.com/mholt/archives v0.1.3
|
||||
github.com/maypok86/otter/v2 v2.2.1
|
||||
github.com/mholt/archives v0.1.5
|
||||
github.com/microsoft/go-mssqldb v1.9.2
|
||||
github.com/ory/dockertest/v3 v3.12.0
|
||||
github.com/praetorian-inc/fingerprintx v1.1.15
|
||||
github.com/projectdiscovery/dsl v0.5.1
|
||||
github.com/projectdiscovery/dsl v0.8.8
|
||||
github.com/projectdiscovery/fasttemplate v0.0.2
|
||||
github.com/projectdiscovery/gcache v0.0.0-20241015120333-12546c6e3f4c
|
||||
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
|
||||
github.com/projectdiscovery/goflags v0.1.74
|
||||
github.com/projectdiscovery/gologger v1.1.54
|
||||
github.com/projectdiscovery/gologger v1.1.64
|
||||
github.com/projectdiscovery/gostruct v0.0.2
|
||||
github.com/projectdiscovery/gozero v0.1.0
|
||||
github.com/projectdiscovery/httpx v1.7.1-0.20250901093029-3909667cbaa6
|
||||
github.com/projectdiscovery/mapcidr v1.1.34
|
||||
github.com/projectdiscovery/gozero v0.1.1-0.20251027191944-a4ea43320b81
|
||||
github.com/projectdiscovery/httpx v1.7.4
|
||||
github.com/projectdiscovery/mapcidr v1.1.97
|
||||
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
|
||||
github.com/projectdiscovery/networkpolicy v0.1.21
|
||||
github.com/projectdiscovery/networkpolicy v0.1.32
|
||||
github.com/projectdiscovery/ratelimit v0.0.82
|
||||
github.com/projectdiscovery/rdap v0.9.0
|
||||
github.com/projectdiscovery/sarif v0.0.1
|
||||
github.com/projectdiscovery/tlsx v1.2.1
|
||||
github.com/projectdiscovery/uncover v1.1.0
|
||||
github.com/projectdiscovery/useragent v0.0.101
|
||||
github.com/projectdiscovery/utils v0.4.24-0.20250823123502-bd7f2849ddb4
|
||||
github.com/projectdiscovery/wappalyzergo v0.2.43
|
||||
github.com/projectdiscovery/tlsx v1.2.2
|
||||
github.com/projectdiscovery/uncover v1.2.0
|
||||
github.com/projectdiscovery/useragent v0.0.105
|
||||
github.com/projectdiscovery/utils v0.7.3
|
||||
github.com/projectdiscovery/wappalyzergo v0.2.59
|
||||
github.com/redis/go-redis/v9 v9.11.0
|
||||
github.com/seh-msft/burpxml v1.0.1
|
||||
github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466
|
||||
github.com/stretchr/testify v1.11.0
|
||||
github.com/sijms/go-ora/v2 v2.9.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
|
||||
github.com/testcontainers/testcontainers-go v0.38.0
|
||||
github.com/testcontainers/testcontainers-go/modules/mongodb v0.37.0
|
||||
@@ -118,7 +124,7 @@ require (
|
||||
github.com/zmap/zgrab2 v0.1.8
|
||||
gitlab.com/gitlab-org/api/client-go v0.130.1
|
||||
go.mongodb.org/mongo-driver v1.17.4
|
||||
golang.org/x/term v0.34.0
|
||||
golang.org/x/term v0.37.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
moul.io/http2curl v1.0.0
|
||||
)
|
||||
@@ -128,25 +134,25 @@ require (
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect
|
||||
github.com/42wim/httpsig v1.2.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
|
||||
github.com/BurntSushi/toml v1.3.2 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
|
||||
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
|
||||
github.com/ProtonMail/go-crypto v1.1.6 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.10.3 // indirect
|
||||
github.com/STARRY-S/zip v0.2.1 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.11.0 // indirect
|
||||
github.com/STARRY-S/zip v0.2.3 // indirect
|
||||
github.com/VividCortex/ewma v1.2.0 // indirect
|
||||
github.com/akrylysov/pogreb v0.10.2 // indirect
|
||||
github.com/alecthomas/chroma/v2 v2.14.0 // indirect
|
||||
github.com/alecthomas/kingpin/v2 v2.4.0 // indirect
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
|
||||
github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect
|
||||
@@ -168,12 +174,15 @@ require (
|
||||
github.com/bits-and-blooms/bitset v1.13.0 // indirect
|
||||
github.com/bits-and-blooms/bloom/v3 v3.5.0 // indirect
|
||||
github.com/bodgit/plumbing v1.3.0 // indirect
|
||||
github.com/bodgit/sevenzip v1.6.0 // indirect
|
||||
github.com/bodgit/sevenzip v1.6.1 // indirect
|
||||
github.com/bodgit/windows v1.0.1 // indirect
|
||||
github.com/brianvoe/gofakeit/v7 v7.2.1 // indirect
|
||||
github.com/buger/jsonparser v1.1.1 // indirect
|
||||
github.com/bytedance/sonic/loader v0.3.0 // indirect
|
||||
github.com/caddyserver/certmagic v0.19.2 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/censys/censys-sdk-go v0.19.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
|
||||
github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 // indirect
|
||||
@@ -185,14 +194,14 @@ require (
|
||||
github.com/cloudflare/cfssl v1.6.4 // indirect
|
||||
github.com/cloudflare/circl v1.6.1 // indirect
|
||||
github.com/cloudwego/base64x v0.1.5 // indirect
|
||||
github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
|
||||
github.com/cnf/structhash v0.0.0-20250313080605-df4c6cc74a9a // indirect
|
||||
github.com/containerd/continuity v0.4.5 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
github.com/containerd/platforms v0.2.1 // indirect
|
||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.4.1 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.5.1 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/davidmz/go-pageant v1.0.2 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
@@ -204,12 +213,13 @@ require (
|
||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
|
||||
github.com/ebitengine/purego v0.8.4 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/ericlagergren/decimal v0.0.0-20240411145413-00de7ca16731 // indirect
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/felixge/fgprof v0.9.5 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
|
||||
github.com/gaissmai/bart v0.24.0 // indirect
|
||||
github.com/gaissmai/bart v0.26.0 // indirect
|
||||
github.com/geoffgarside/ber v1.1.0 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/gin-gonic/gin v1.9.1 // indirect
|
||||
@@ -222,6 +232,7 @@ require (
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.21.0 // indirect
|
||||
github.com/go-openapi/swag v0.23.0 // indirect
|
||||
github.com/go-pg/zerochecker v0.2.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
|
||||
@@ -237,15 +248,16 @@ require (
|
||||
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
|
||||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
|
||||
github.com/gorilla/css v1.0.1 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/gosimple/slug v1.15.0 // indirect
|
||||
github.com/gosimple/unidecode v1.0.1 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/hashicorp/go-version v1.7.0 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect
|
||||
github.com/hdm/jarm-go v0.0.7 // indirect
|
||||
github.com/iangcarroll/cookiemonster v1.6.0 // indirect
|
||||
github.com/imdario/mergo v0.3.16 // indirect
|
||||
github.com/itchyny/timefmt-go v0.1.6 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
@@ -255,9 +267,10 @@ require (
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/k14s/starlark-go v0.0.0-20200720175618-3a5c849cc368 // indirect
|
||||
github.com/kataras/jwt v0.1.10 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
|
||||
github.com/klauspost/pgzip v1.2.6 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
@@ -265,8 +278,8 @@ require (
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/libdns/libdns v0.2.1 // indirect
|
||||
github.com/logrusorgru/aurora/v4 v4.0.0 // indirect
|
||||
github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/lor00x/goldap v0.0.0-20240304151906-8d785c64d1c8 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20250821153705-5981dea3221d // indirect
|
||||
github.com/mackerelio/go-osstat v0.2.4 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
@@ -277,7 +290,7 @@ require (
|
||||
github.com/mholt/acmez v1.2.0 // indirect
|
||||
github.com/microcosm-cc/bluemonday v1.0.27 // indirect
|
||||
github.com/mikelolasagasti/xz v1.0.1 // indirect
|
||||
github.com/minio/minlz v1.0.0 // indirect
|
||||
github.com/minio/minlz v1.0.1 // indirect
|
||||
github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
@@ -294,25 +307,25 @@ require (
|
||||
github.com/morikuni/aec v1.0.0 // indirect
|
||||
github.com/muesli/reflow v0.3.0 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.1.0 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.2.1 // indirect
|
||||
github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect
|
||||
github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect
|
||||
github.com/olekukonko/errors v1.1.0 // indirect
|
||||
github.com/olekukonko/ll v0.0.9 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/opencontainers/runc v1.2.3 // indirect
|
||||
github.com/opencontainers/runc v1.2.8 // indirect
|
||||
github.com/openrdap/rdap v0.9.1 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
|
||||
github.com/perimeterx/marshmallow v1.1.5 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.21 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||
github.com/pjbgf/sha1cd v0.3.2 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/projectdiscovery/asnmap v1.1.1 // indirect
|
||||
github.com/projectdiscovery/blackrock v0.0.1 // indirect
|
||||
github.com/projectdiscovery/cdncheck v1.1.33 // indirect
|
||||
github.com/projectdiscovery/cdncheck v1.2.14 // indirect
|
||||
github.com/projectdiscovery/freeport v0.0.7 // indirect
|
||||
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
|
||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
|
||||
@@ -324,22 +337,29 @@ require (
|
||||
github.com/shoenig/go-m1cpu v0.1.6 // indirect
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
github.com/skeema/knownhosts v1.3.1 // indirect
|
||||
github.com/sorairolake/lzip-go v0.3.5 // indirect
|
||||
github.com/sorairolake/lzip-go v0.3.8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/tidwall/btree v1.7.0 // indirect
|
||||
github.com/tidwall/buntdb v1.3.1 // indirect
|
||||
github.com/tidwall/gjson v1.18.0 // indirect
|
||||
github.com/tidwall/grect v0.1.4 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/match v1.2.0 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
github.com/tidwall/rtred v0.1.2 // indirect
|
||||
github.com/tidwall/tinyqueue v0.1.1 // indirect
|
||||
github.com/tim-ywliu/nested-logrus-formatter v1.3.2 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.15 // indirect
|
||||
github.com/tklauser/numcpus v0.10.0 // indirect
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.11 // indirect
|
||||
github.com/ulikunitz/xz v0.5.12 // indirect
|
||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/vmihailenco/bufpool v0.1.11 // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect
|
||||
github.com/vmihailenco/tagparser v0.1.2 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/vulncheck-oss/go-exploit v1.51.0 // indirect
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
@@ -352,18 +372,18 @@ require (
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
github.com/ysmood/fetchup v0.2.3 // indirect
|
||||
github.com/ysmood/got v0.40.0 // indirect
|
||||
github.com/yuin/goldmark v1.7.8 // indirect
|
||||
github.com/yuin/goldmark v1.7.13 // indirect
|
||||
github.com/yuin/goldmark-emoji v1.0.5 // indirect
|
||||
github.com/zcalusic/sysinfo v1.0.2 // indirect
|
||||
github.com/zeebo/blake3 v0.2.3 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect
|
||||
go.opentelemetry.io/otel v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.38.0 // indirect
|
||||
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
||||
golang.org/x/arch v0.3.0 // indirect
|
||||
golang.org/x/sync v0.16.0 // indirect
|
||||
golang.org/x/sync v0.18.0 // indirect
|
||||
gopkg.in/djherbis/times.v1 v1.3.0 // indirect
|
||||
mellium.im/sasl v0.3.2 // indirect
|
||||
)
|
||||
@@ -387,12 +407,12 @@ require (
|
||||
go.etcd.io/bbolt v1.4.0 // indirect
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
goftp.io/server/v2 v2.0.1 // indirect
|
||||
golang.org/x/crypto v0.41.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b
|
||||
golang.org/x/mod v0.26.0 // indirect
|
||||
golang.org/x/sys v0.35.0 // indirect
|
||||
golang.org/x/time v0.11.0 // indirect
|
||||
golang.org/x/tools v0.35.0
|
||||
golang.org/x/crypto v0.45.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20250911091902-df9299821621
|
||||
golang.org/x/mod v0.29.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
golang.org/x/tools v0.38.0
|
||||
google.golang.org/protobuf v1.36.6 // indirect
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
|
||||
gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
|
||||
@@ -406,10 +426,3 @@ require (
|
||||
|
||||
// https://go.dev/ref/mod#go-mod-file-retract
|
||||
retract v3.2.0 // retract due to broken js protocol issue
|
||||
|
||||
// Fix genproto version conflicts
|
||||
replace (
|
||||
google.golang.org/genproto => google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142
|
||||
google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142
|
||||
google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
id: fuzz-body
|
||||
|
||||
info:
|
||||
name: fuzzing error sqli payloads in http req body
|
||||
author: pdteam
|
||||
severity: info
|
||||
description: |
|
||||
This template attempts to find SQL injection vulnerabilities by fuzzing http body
|
||||
It automatically handles and parses json,xml,multipart form and x-www-form-urlencoded data
|
||||
and performs fuzzing on the value of every key
|
||||
|
||||
http:
|
||||
- pre-condition:
|
||||
- type: dsl
|
||||
dsl:
|
||||
- method != "GET"
|
||||
- method != "HEAD"
|
||||
condition: and
|
||||
|
||||
payloads:
|
||||
injection:
|
||||
- "'"
|
||||
- "\""
|
||||
- ";"
|
||||
|
||||
fuzzing:
|
||||
- part: body
|
||||
type: postfix
|
||||
mode: single
|
||||
fuzz:
|
||||
- '{{injection}}'
|
||||
|
||||
stop-at-first-match: true
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "unrecognized token:"
|
||||
- "null"
|
||||
@@ -0,0 +1,27 @@
|
||||
id: signed-python-code-in-virtual-env
|
||||
|
||||
info:
|
||||
name: signed-python-code-in-virtual-env
|
||||
author: pdteam
|
||||
severity: info
|
||||
tags: code
|
||||
description: |
|
||||
signed python code in virtual environment
|
||||
|
||||
code:
|
||||
- engine:
|
||||
- sh
|
||||
- bash
|
||||
sandbox:
|
||||
working-dir: /tmp
|
||||
image: python:3.14
|
||||
source: |
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
print("hello from python virtual code")
|
||||
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "hello from python virtual code"
|
||||
@@ -0,0 +1,25 @@
|
||||
id: signed-code-in-virtual-env
|
||||
|
||||
info:
|
||||
name: signed-code-in-virtual-env
|
||||
author: pdteam
|
||||
severity: info
|
||||
tags: code
|
||||
description: |
|
||||
signed code in virtual environment
|
||||
|
||||
code:
|
||||
- engine:
|
||||
- sh
|
||||
- bash
|
||||
sandbox:
|
||||
working-dir: /tmp
|
||||
image: ubuntu:latest
|
||||
source: |
|
||||
echo "hello from sh virtual code"
|
||||
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "hello from sh virtual code"
|
||||
# digest: 4a0a00473045022100a2a71c423b72e600ac2d78209482b48591924157d49e2dfb7767445b0073e92b022009a6365e84247e268c7ffcb2f9ed424529dada6f087d5f21400ab0b2a822ca56:4a3eb6b4988d95847d4203be25ed1d46
|
||||
@@ -0,0 +1,27 @@
|
||||
id: constants-with-threads
|
||||
|
||||
info:
|
||||
name: Constants with Threads
|
||||
author: pdteam
|
||||
severity: info
|
||||
description: |
|
||||
Test that constants are properly resolved when using threads mode.
|
||||
|
||||
constants:
|
||||
api_key: "supersecretkey123"
|
||||
api_version: "v2"
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}/api/{{api_version}}"
|
||||
threads: 5
|
||||
headers:
|
||||
X-API-Key: "{{api_key}}"
|
||||
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "supersecretkey123"
|
||||
- "v2"
|
||||
condition: and
|
||||
@@ -0,0 +1,30 @@
|
||||
id: race-with-variables
|
||||
|
||||
info:
|
||||
name: Race Condition with Variables
|
||||
author: pdteam
|
||||
severity: info
|
||||
description: |
|
||||
Test that variables and constants are properly resolved in race mode.
|
||||
|
||||
variables:
|
||||
random_id: "{{rand_base(8)}}"
|
||||
|
||||
constants:
|
||||
api_key: "racekey123"
|
||||
|
||||
http:
|
||||
- raw:
|
||||
- |
|
||||
GET /race HTTP/1.1
|
||||
Host: {{Hostname}}
|
||||
X-Request-Id: {{random_id}}
|
||||
X-API-Key: {{api_key}}
|
||||
|
||||
race: true
|
||||
race_count: 3
|
||||
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "racekey123"
|
||||
@@ -0,0 +1,38 @@
|
||||
id: variables-threads-previous
|
||||
|
||||
info:
|
||||
name: Variables with Threads and Previous Request Data
|
||||
author: pdteam
|
||||
severity: info
|
||||
description: |
|
||||
Test that variables can reference data extracted from previous requests
|
||||
when using threads mode (parallel execution).
|
||||
|
||||
variables:
|
||||
auth_header: "Bearer {{extracted_token}}"
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}/login"
|
||||
|
||||
extractors:
|
||||
- type: regex
|
||||
name: extracted_token
|
||||
part: body
|
||||
regex:
|
||||
- 'token=([a-z0-9]+)'
|
||||
group: 1
|
||||
internal: true
|
||||
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}/api"
|
||||
threads: 5
|
||||
headers:
|
||||
Authorization: "{{auth_header}}"
|
||||
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "Bearer secret123"
|
||||
@@ -0,0 +1,28 @@
|
||||
id: multi-ports
|
||||
|
||||
info:
|
||||
name: Multi Ports - Detection
|
||||
author: pdteam
|
||||
severity: info
|
||||
description: |
|
||||
Multi Ports template for testing
|
||||
metadata:
|
||||
max-request: 1
|
||||
tags: js,detect,multi-ports,enum,network
|
||||
|
||||
javascript:
|
||||
- pre-condition: |
|
||||
isPortOpen(Host,Port);
|
||||
code: |
|
||||
var m = require("nuclei/ssh");
|
||||
var c = m.SSHClient();
|
||||
var response = c.ConnectSSHInfoMode(Host, Port);
|
||||
Export(response);
|
||||
args:
|
||||
Host: "{{Host}}"
|
||||
Port: "2222,22" # Port 22 should match
|
||||
|
||||
extractors:
|
||||
- type: json
|
||||
json:
|
||||
- '.UserAuth'
|
||||
@@ -0,0 +1,38 @@
|
||||
id: vnc-password-test
|
||||
|
||||
info:
|
||||
name: VNC Password Authentication Test
|
||||
author: pdteam
|
||||
severity: high
|
||||
description: |
|
||||
Tests VNC authentication with correct and incorrect passwords.
|
||||
metadata:
|
||||
shodan-query: product:"vnc"
|
||||
tags: js,network,vnc,authentication
|
||||
|
||||
javascript:
|
||||
- pre-condition: |
|
||||
isPortOpen(Host,Port)
|
||||
|
||||
code: |
|
||||
let vnc = require('nuclei/vnc');
|
||||
let client = new vnc.VNCClient();
|
||||
client.Connect(Host, Port, Password);
|
||||
|
||||
args:
|
||||
Host: "{{Host}}"
|
||||
Port: "5900"
|
||||
Password: "{{passwords}}"
|
||||
payloads:
|
||||
passwords:
|
||||
- ""
|
||||
- root
|
||||
- password
|
||||
- admin
|
||||
- mysecret
|
||||
stop-at-first-match: true
|
||||
|
||||
matchers:
|
||||
- type: dsl
|
||||
dsl:
|
||||
- "success == true"
|
||||
@@ -1,3 +1,3 @@
|
||||
## keys
|
||||
|
||||
the keys stored here especially `ci-private-key.pem` and `ci.crt` are used in integration tests to test template signing and verfication functionality introduced in nuclei v3
|
||||
the keys stored here especially `ci-private-key.pem` and `ci.crt` are used in integration tests to test template signing and verification functionality introduced in nuclei v3
|
||||
@@ -104,7 +104,7 @@ func NewUploadWriter(ctx context.Context, logger *gologger.Logger, creds *pdcpau
|
||||
// SetScanID sets the scan id for the upload writer
|
||||
func (u *UploadWriter) SetScanID(id string) error {
|
||||
if !xidRegex.MatchString(id) {
|
||||
return fmt.Errorf("invalid scan id provided")
|
||||
gologger.Warning().Msgf("invalid asset id provided (unknown xid format): %s", id)
|
||||
}
|
||||
u.scanID = id
|
||||
return nil
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/authprovider/authx"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
@@ -68,7 +69,7 @@ func GetLazyAuthFetchCallback(opts *AuthLazyFetchOptions) authx.LazyFetchSecret
|
||||
return func(d *authx.Dynamic) error {
|
||||
tmpls := opts.TemplateStore.LoadTemplates([]string{d.TemplatePath})
|
||||
if len(tmpls) == 0 {
|
||||
return fmt.Errorf("no templates found for path: %s", d.TemplatePath)
|
||||
return fmt.Errorf("%w for path: %s", disk.ErrNoTemplatesFound, d.TemplatePath)
|
||||
}
|
||||
if len(tmpls) > 1 {
|
||||
return fmt.Errorf("multiple templates found for path: %s", d.TemplatePath)
|
||||
|
||||
+22
-20
@@ -125,14 +125,6 @@ func New(options *types.Options) (*Runner, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// if template list or template display is enabled, enable all templates
|
||||
if options.TemplateList || options.TemplateDisplay {
|
||||
options.EnableCodeTemplates = true
|
||||
options.EnableFileTemplates = true
|
||||
options.EnableSelfContainedTemplates = true
|
||||
options.EnableGlobalMatchersTemplates = true
|
||||
}
|
||||
|
||||
// check for custom template updates and update if available
|
||||
ctm, err := customtemplates.NewCustomTemplatesManager(options)
|
||||
if err != nil {
|
||||
@@ -254,8 +246,23 @@ func New(options *types.Options) (*Runner, error) {
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "nuclei-tmp-*")
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not create temporary directory")
|
||||
}
|
||||
runner.tmpDir = tmpDir
|
||||
|
||||
// Cleanup tmpDir only if initialization fails
|
||||
// On successful initialization, Close() method will handle cleanup
|
||||
cleanupOnError := true
|
||||
defer func() {
|
||||
if cleanupOnError && runner.tmpDir != "" {
|
||||
_ = os.RemoveAll(runner.tmpDir)
|
||||
}
|
||||
}()
|
||||
|
||||
// create the input provider and load the inputs
|
||||
inputProvider, err := provider.NewInputProvider(provider.InputOptions{Options: options})
|
||||
inputProvider, err := provider.NewInputProvider(provider.InputOptions{Options: options, TempDir: runner.tmpDir})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not create input provider")
|
||||
}
|
||||
@@ -384,16 +391,10 @@ func New(options *types.Options) (*Runner, error) {
|
||||
if options.RateLimit > 0 && options.RateLimitDuration == 0 {
|
||||
options.RateLimitDuration = time.Second
|
||||
}
|
||||
if options.RateLimit == 0 && options.RateLimitDuration == 0 {
|
||||
runner.rateLimiter = ratelimit.NewUnlimited(context.Background())
|
||||
} else {
|
||||
runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimit), options.RateLimitDuration)
|
||||
}
|
||||
|
||||
if tmpDir, err := os.MkdirTemp("", "nuclei-tmp-*"); err == nil {
|
||||
runner.tmpDir = tmpDir
|
||||
}
|
||||
runner.rateLimiter = utils.GetRateLimiter(context.Background(), options.RateLimit, options.RateLimitDuration)
|
||||
|
||||
// Initialization successful, disable cleanup on error
|
||||
cleanupOnError = false
|
||||
return runner, nil
|
||||
}
|
||||
|
||||
@@ -576,7 +577,9 @@ func (r *Runner) RunEnumeration() error {
|
||||
}
|
||||
|
||||
if len(r.options.SecretsFile) > 0 && !r.options.Validate {
|
||||
authTmplStore, err := GetAuthTmplStore(r.options, r.catalog, executorOpts)
|
||||
// Clone options so GetAuthTmplStore can modify them without affecting the original
|
||||
authOptions := r.options.Copy()
|
||||
authTmplStore, err := GetAuthTmplStore(authOptions, r.catalog, executorOpts)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to load dynamic auth templates")
|
||||
}
|
||||
@@ -659,7 +662,6 @@ func (r *Runner) RunEnumeration() error {
|
||||
}
|
||||
store.Load()
|
||||
// TODO: remove below functions after v3 or update warning messages
|
||||
disk.PrintDeprecatedPathsMsgIfApplicable(r.options.Silent)
|
||||
templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
|
||||
|
||||
// add the hosts from the metadata queries of loaded templates into input provider
|
||||
|
||||
@@ -44,7 +44,13 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) {
|
||||
config.DefaultConfig.TemplateVersion,
|
||||
config.DefaultConfig.TemplatesDirectory,
|
||||
)
|
||||
for _, tpl := range store.Templates() {
|
||||
// order templates alphabetically by path
|
||||
templates := store.Templates()
|
||||
sort.Slice(templates, func(i, j int) bool {
|
||||
return templates[i].Path < templates[j].Path
|
||||
})
|
||||
|
||||
for _, tpl := range templates {
|
||||
if hasExtraFlags(r.options) {
|
||||
if r.options.TemplateDisplay {
|
||||
colorize := !r.options.NoColor
|
||||
|
||||
+1
-1
@@ -37,7 +37,7 @@ import nuclei "github.com/projectdiscovery/nuclei/v3/lib"
|
||||
|
||||
## Advanced Example of using Nuclei Library/SDK
|
||||
|
||||
For Various use cases like batching etc you might want to run nuclei in goroutines this can be done by using `nuclei.NewThreadSafeNucleiEngine`
|
||||
For Various use cases like batching etc. you might want to run nuclei in goroutines this can be done by using `nuclei.NewThreadSafeNucleiEngine`
|
||||
|
||||
```go
|
||||
// create nuclei engine with options
|
||||
|
||||
+22
-13
@@ -3,11 +3,12 @@ package nuclei
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
|
||||
@@ -18,7 +19,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
pkgtypes "github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
)
|
||||
@@ -181,7 +181,7 @@ func WithGlobalRateLimitCtx(ctx context.Context, maxTokens int, duration time.Du
|
||||
return func(e *NucleiEngine) error {
|
||||
e.opts.RateLimit = maxTokens
|
||||
e.opts.RateLimitDuration = duration
|
||||
e.rateLimiter = ratelimit.New(ctx, uint(e.opts.RateLimit), e.opts.RateLimitDuration)
|
||||
e.rateLimiter = utils.GetRateLimiter(ctx, e.opts.RateLimit, e.opts.RateLimitDuration)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -195,8 +195,10 @@ type HeadlessOpts struct {
|
||||
}
|
||||
|
||||
// EnableHeadless allows execution of headless templates
|
||||
// *Use With Caution*: Enabling headless mode may open up attack surface due to browser usage
|
||||
// and can be prone to exploitation by custom unverified templates if not properly configured
|
||||
//
|
||||
// Warning: enabling headless mode may open up attack surface due to browser
|
||||
// usage and can be prone to exploitation by custom unverified templates if not
|
||||
// properly configured.
|
||||
func EnableHeadlessWithOpts(hopts *HeadlessOpts) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
e.opts.Headless = true
|
||||
@@ -206,14 +208,6 @@ func EnableHeadlessWithOpts(hopts *HeadlessOpts) NucleiSDKOptions {
|
||||
e.opts.ShowBrowser = hopts.ShowBrowser
|
||||
e.opts.UseInstalledChrome = hopts.UseChrome
|
||||
}
|
||||
if engine.MustDisableSandbox() {
|
||||
e.Logger.Warning().Msgf("The current platform and privileged user will run the browser without sandbox")
|
||||
}
|
||||
browser, err := engine.New(e.opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
e.browserInstance = browser
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -559,3 +553,18 @@ func WithOptions(opts *pkgtypes.Options) NucleiSDKOptions {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithTemporaryDirectory allows setting a parent directory for SDK-managed temporary files.
|
||||
// A temporary directory will be created inside the provided directory and cleaned up on engine close.
|
||||
// If not set, a temporary directory will be automatically created in the system temp location.
|
||||
// The parent directory is assumed to exist.
|
||||
func WithTemporaryDirectory(parentDir string) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
tmpDir, err := os.MkdirTemp(parentDir, "nuclei-tmp-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
e.tmpDir = tmpDir
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
//go:build !race
|
||||
// +build !race
|
||||
|
||||
package nuclei_test
|
||||
|
||||
|
||||
+8
-10
@@ -12,7 +12,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
"github.com/rs/xid"
|
||||
)
|
||||
@@ -53,11 +53,7 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
|
||||
if opts.RateLimit > 0 && opts.RateLimitDuration == 0 {
|
||||
opts.RateLimitDuration = time.Second
|
||||
}
|
||||
if opts.RateLimit == 0 && opts.RateLimitDuration == 0 {
|
||||
u.executerOpts.RateLimiter = ratelimit.NewUnlimited(ctx)
|
||||
} else {
|
||||
u.executerOpts.RateLimiter = ratelimit.New(ctx, uint(opts.RateLimit), opts.RateLimitDuration)
|
||||
}
|
||||
u.executerOpts.RateLimiter = utils.GetRateLimiter(ctx, opts.RateLimit, opts.RateLimitDuration)
|
||||
u.engine = core.New(opts)
|
||||
u.engine.SetExecuterOptions(u.executerOpts)
|
||||
return u, nil
|
||||
@@ -91,10 +87,12 @@ type ThreadSafeNucleiEngine struct {
|
||||
func NewThreadSafeNucleiEngineCtx(ctx context.Context, opts ...NucleiSDKOptions) (*ThreadSafeNucleiEngine, error) {
|
||||
defaultOptions := types.DefaultOptions()
|
||||
defaultOptions.ExecutionId = xid.New().String()
|
||||
// default options
|
||||
|
||||
e := &NucleiEngine{
|
||||
opts: defaultOptions,
|
||||
mode: threadSafe,
|
||||
opts: defaultOptions,
|
||||
mode: threadSafe,
|
||||
ctx: ctx,
|
||||
Logger: defaultOptions.Logger,
|
||||
}
|
||||
for _, option := range opts {
|
||||
if err := option(e); err != nil {
|
||||
@@ -124,7 +122,7 @@ func (e *ThreadSafeNucleiEngine) GlobalResultCallback(callback func(event *outpu
|
||||
}
|
||||
|
||||
// ExecuteNucleiWithOptsCtx executes templates on targets and calls callback on each result(only if results are found)
|
||||
// This method can be called concurrently and it will use some global resources but can be runned parallelly
|
||||
// This method can be called concurrently and it will use some global resources but can be run parallelly
|
||||
// by invoking this method with different options and targets
|
||||
// Note: Not all options are thread-safe. this method will throw error if you try to use non-thread-safe options
|
||||
func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, targets []string, opts ...NucleiSDKOptions) error {
|
||||
|
||||
+12
-3
@@ -5,6 +5,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
@@ -92,6 +93,9 @@ type NucleiEngine struct {
|
||||
|
||||
// Logger instance for the engine
|
||||
Logger *gologger.Logger
|
||||
|
||||
// Temporary directory for SDK-managed template files
|
||||
tmpDir string
|
||||
}
|
||||
|
||||
// LoadAllTemplates loads all nuclei template based on given options
|
||||
@@ -231,6 +235,9 @@ func (e *NucleiEngine) closeInternal() {
|
||||
if e.httpxClient != nil {
|
||||
_ = e.httpxClient.Close()
|
||||
}
|
||||
if e.tmpDir != "" {
|
||||
_ = os.RemoveAll(e.tmpDir)
|
||||
}
|
||||
}
|
||||
|
||||
// Close all resources used by nuclei engine
|
||||
@@ -313,10 +320,12 @@ func NewNucleiEngineCtx(ctx context.Context, options ...NucleiSDKOptions) (*Nucl
|
||||
// default options
|
||||
defaultOptions := types.DefaultOptions()
|
||||
defaultOptions.ExecutionId = xid.New().String()
|
||||
|
||||
e := &NucleiEngine{
|
||||
opts: defaultOptions,
|
||||
mode: singleInstance,
|
||||
ctx: ctx,
|
||||
opts: defaultOptions,
|
||||
mode: singleInstance,
|
||||
ctx: ctx,
|
||||
Logger: defaultOptions.Logger,
|
||||
}
|
||||
for _, option := range options {
|
||||
if err := option(e); err != nil {
|
||||
|
||||
+38
-19
@@ -3,6 +3,7 @@ package nuclei
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -12,7 +13,6 @@ import (
|
||||
|
||||
"github.com/logrusorgru/aurora"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/gologger/levels"
|
||||
"github.com/projectdiscovery/httpx/common/httpx"
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/runner"
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
|
||||
@@ -96,13 +97,11 @@ func (e *NucleiEngine) applyRequiredDefaults(ctx context.Context) {
|
||||
|
||||
// init
|
||||
func (e *NucleiEngine) init(ctx context.Context) error {
|
||||
// Set a default logger if one isn't provided in the options
|
||||
if e.opts.Logger != nil {
|
||||
// Update logger ref (if it was changed by [WithLogger])
|
||||
// (Logger is already initialized)
|
||||
if e.opts.Logger != e.Logger {
|
||||
e.Logger = e.opts.Logger
|
||||
} else {
|
||||
e.opts.Logger = &gologger.Logger{}
|
||||
}
|
||||
e.Logger = e.opts.Logger
|
||||
|
||||
if e.opts.Verbose {
|
||||
e.Logger.SetMaxLevel(levels.LevelVerbose)
|
||||
@@ -166,24 +165,44 @@ func (e *NucleiEngine) init(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if e.opts.Headless {
|
||||
if engine.MustDisableSandbox() {
|
||||
e.Logger.Warning().Msgf("The current platform and privileged user will run the browser without sandbox")
|
||||
}
|
||||
browser, err := engine.New(e.opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
e.browserInstance = browser
|
||||
}
|
||||
|
||||
if e.catalog == nil {
|
||||
e.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory)
|
||||
}
|
||||
|
||||
if e.tmpDir == "" {
|
||||
tmpDir, err := os.MkdirTemp("", "nuclei-tmp-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
e.tmpDir = tmpDir
|
||||
}
|
||||
|
||||
e.executerOpts = &protocols.ExecutorOptions{
|
||||
Output: e.customWriter,
|
||||
Options: e.opts,
|
||||
Progress: e.customProgress,
|
||||
Catalog: e.catalog,
|
||||
IssuesClient: e.rc,
|
||||
RateLimiter: e.rateLimiter,
|
||||
Interactsh: e.interactshClient,
|
||||
Colorizer: aurora.NewAurora(true),
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
Browser: e.browserInstance,
|
||||
Parser: e.parser,
|
||||
InputHelper: input.NewHelper(),
|
||||
Logger: e.opts.Logger,
|
||||
Output: e.customWriter,
|
||||
Options: e.opts,
|
||||
Progress: e.customProgress,
|
||||
Catalog: e.catalog,
|
||||
IssuesClient: e.rc,
|
||||
RateLimiter: e.rateLimiter,
|
||||
Interactsh: e.interactshClient,
|
||||
Colorizer: aurora.NewAurora(true),
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
Browser: e.browserInstance,
|
||||
Parser: e.parser,
|
||||
InputHelper: input.NewHelper(),
|
||||
TemporaryDirectory: e.tmpDir,
|
||||
Logger: e.opts.Logger,
|
||||
}
|
||||
if e.opts.ShouldUseHostError() && e.hostErrCache != nil {
|
||||
e.executerOpts.HostErrorsCache = e.hostErrCache
|
||||
|
||||
@@ -35,3 +35,23 @@ func TestContextCancelNucleiEngine(t *testing.T) {
|
||||
}
|
||||
defer ne.Close()
|
||||
}
|
||||
|
||||
func TestHeadlessOptionInitialization(t *testing.T) {
|
||||
ne, err := nuclei.NewNucleiEngineCtx(
|
||||
context.Background(),
|
||||
nuclei.EnableHeadlessWithOpts(&nuclei.HeadlessOpts{
|
||||
PageTimeout: 20,
|
||||
ShowBrowser: false,
|
||||
UseChrome: false,
|
||||
HeadlessOptions: []string{},
|
||||
}),
|
||||
)
|
||||
|
||||
require.NoError(t, err, "could not create nuclei engine with headless options")
|
||||
require.NotNil(t, ne, "nuclei engine should not be nil")
|
||||
|
||||
// Verify logger is initialized
|
||||
require.NotNil(t, ne.Logger, "logger should be initialized")
|
||||
|
||||
defer ne.Close()
|
||||
}
|
||||
@@ -59,6 +59,11 @@
|
||||
"title": "engine",
|
||||
"description": "Engine"
|
||||
},
|
||||
"sandbox": {
|
||||
"$ref": "#/$defs/code.Sandbox",
|
||||
"title": "sandbox",
|
||||
"description": "Sandbox"
|
||||
},
|
||||
"pre-condition": {
|
||||
"type": "string",
|
||||
"title": "pre-condition for the request",
|
||||
@@ -86,6 +91,22 @@
|
||||
"additionalProperties": false,
|
||||
"type": "object"
|
||||
},
|
||||
"code.Sandbox": {
|
||||
"properties": {
|
||||
"working-dir": {
|
||||
"type": "string",
|
||||
"title": "working-dir",
|
||||
"description": "Working directory"
|
||||
},
|
||||
"image": {
|
||||
"type": "string",
|
||||
"title": "image",
|
||||
"description": "Image"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"type": "object"
|
||||
},
|
||||
"dns.DNSRequestTypeHolder": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
|
||||
@@ -3,7 +3,7 @@ package authx
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
|
||||
@@ -30,8 +30,8 @@ type Dynamic struct {
|
||||
Input string `json:"input" yaml:"input"` // (optional) target for the dynamic secret
|
||||
Extracted map[string]interface{} `json:"-" yaml:"-"` // extracted values from the dynamic secret
|
||||
fetchCallback LazyFetchSecret `json:"-" yaml:"-"`
|
||||
m *sync.Mutex `json:"-" yaml:"-"` // mutex for lazy fetch
|
||||
fetched bool `json:"-" yaml:"-"` // flag to check if the secret has been fetched
|
||||
fetched *atomic.Bool `json:"-" yaml:"-"` // atomic flag to check if the secret has been fetched
|
||||
fetching *atomic.Bool `json:"-" yaml:"-"` // atomic flag to prevent recursive fetch calls
|
||||
error error `json:"-" yaml:"-"` // error if any
|
||||
}
|
||||
|
||||
@@ -70,7 +70,8 @@ func (d *Dynamic) UnmarshalJSON(data []byte) error {
|
||||
|
||||
// Validate validates the dynamic secret
|
||||
func (d *Dynamic) Validate() error {
|
||||
d.m = &sync.Mutex{}
|
||||
d.fetched = &atomic.Bool{}
|
||||
d.fetching = &atomic.Bool{}
|
||||
if d.TemplatePath == "" {
|
||||
return errkit.New(" template-path is required for dynamic secret")
|
||||
}
|
||||
@@ -97,9 +98,7 @@ func (d *Dynamic) Validate() error {
|
||||
func (d *Dynamic) SetLazyFetchCallback(callback LazyFetchSecret) {
|
||||
d.fetchCallback = func(d *Dynamic) error {
|
||||
err := callback(d)
|
||||
d.fetched = true
|
||||
if err != nil {
|
||||
d.error = err
|
||||
return err
|
||||
}
|
||||
if len(d.Extracted) == 0 {
|
||||
@@ -184,9 +183,15 @@ func (d *Dynamic) applyValuesToSecret(secret *Secret) error {
|
||||
|
||||
// GetStrategy returns the auth strategies for the dynamic secret
|
||||
func (d *Dynamic) GetStrategies() []AuthStrategy {
|
||||
if !d.fetched {
|
||||
if d.fetched.Load() {
|
||||
if d.error != nil {
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
// Try to fetch if not already fetched
|
||||
_ = d.Fetch(true)
|
||||
}
|
||||
|
||||
if d.error != nil {
|
||||
return nil
|
||||
}
|
||||
@@ -203,12 +208,23 @@ func (d *Dynamic) GetStrategies() []AuthStrategy {
|
||||
// Fetch fetches the dynamic secret
|
||||
// if isFatal is true, it will stop the execution if the secret could not be fetched
|
||||
func (d *Dynamic) Fetch(isFatal bool) error {
|
||||
d.m.Lock()
|
||||
defer d.m.Unlock()
|
||||
if d.fetched {
|
||||
return nil
|
||||
if d.fetched.Load() {
|
||||
return d.error
|
||||
}
|
||||
|
||||
// Try to set fetching flag atomically
|
||||
if !d.fetching.CompareAndSwap(false, true) {
|
||||
// Already fetching, return current error
|
||||
return d.error
|
||||
}
|
||||
|
||||
// We're the only one fetching, call the callback
|
||||
d.error = d.fetchCallback(d)
|
||||
|
||||
// Mark as fetched and clear fetching flag
|
||||
d.fetched.Store(true)
|
||||
d.fetching.Store(false)
|
||||
|
||||
if d.error != nil && isFatal {
|
||||
gologger.Fatal().Msgf("Could not fetch dynamic secret: %s\n", d.error)
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ type Secret struct {
|
||||
Type string `json:"type" yaml:"type"`
|
||||
Domains []string `json:"domains" yaml:"domains"`
|
||||
DomainsRegex []string `json:"domains-regex" yaml:"domains-regex"`
|
||||
Headers []KV `json:"headers" yaml:"headers"`
|
||||
Headers []KV `json:"headers" yaml:"headers"` // Headers preserve exact casing (useful for case-sensitive APIs)
|
||||
Cookies []Cookie `json:"cookies" yaml:"cookies"`
|
||||
Params []KV `json:"params" yaml:"params"`
|
||||
Username string `json:"username" yaml:"username"` // can be either email or username
|
||||
@@ -148,7 +148,7 @@ func (s *Secret) Validate() error {
|
||||
}
|
||||
|
||||
type KV struct {
|
||||
Key string `json:"key" yaml:"key"`
|
||||
Key string `json:"key" yaml:"key"` // Header key (preserves exact casing)
|
||||
Value string `json:"value" yaml:"value"`
|
||||
}
|
||||
|
||||
|
||||
@@ -21,15 +21,19 @@ func NewHeadersAuthStrategy(data *Secret) *HeadersAuthStrategy {
|
||||
}
|
||||
|
||||
// Apply applies the headers auth strategy to the request
|
||||
// NOTE: This preserves exact header casing (e.g., barAuthToken stays as barAuthToken)
|
||||
// This is useful for APIs that require case-sensitive header names
|
||||
func (s *HeadersAuthStrategy) Apply(req *http.Request) {
|
||||
for _, header := range s.Data.Headers {
|
||||
req.Header.Set(header.Key, header.Value)
|
||||
req.Header[header.Key] = []string{header.Value}
|
||||
}
|
||||
}
|
||||
|
||||
// ApplyOnRR applies the headers auth strategy to the retryable request
|
||||
// NOTE: This preserves exact header casing (e.g., barAuthToken stays as barAuthToken)
|
||||
// This is useful for APIs that require case-sensitive header names
|
||||
func (s *HeadersAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
|
||||
for _, header := range s.Data.Headers {
|
||||
req.Header.Set(header.Key, header.Value)
|
||||
req.Header[header.Key] = []string{header.Value}
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,8 @@ info:
|
||||
# static secrets
|
||||
static:
|
||||
# for header based auth session
|
||||
# NOTE: Headers preserve exact casing (e.g., x-pdcp-key stays as x-pdcp-key)
|
||||
# This is useful for APIs that require case-sensitive header names
|
||||
- type: header
|
||||
domains:
|
||||
- api.projectdiscovery.io
|
||||
@@ -20,6 +22,8 @@ static:
|
||||
headers:
|
||||
- key: x-pdcp-key
|
||||
value: <api-key-here>
|
||||
- key: barAuthToken
|
||||
value: <auth-token-here>
|
||||
|
||||
# for query based auth session
|
||||
- type: Query
|
||||
|
||||
@@ -31,7 +31,7 @@ const (
|
||||
CLIConfigFileName = "config.yaml"
|
||||
ReportingConfigFilename = "reporting-config.yaml"
|
||||
// Version is the current version of nuclei
|
||||
Version = `v3.4.10`
|
||||
Version = `v3.6.1`
|
||||
// Directory Names of custom templates
|
||||
CustomS3TemplatesDirName = "s3"
|
||||
CustomGitHubTemplatesDirName = "github"
|
||||
@@ -40,6 +40,7 @@ const (
|
||||
BinaryName = "nuclei"
|
||||
FallbackConfigFolderName = ".nuclei-config"
|
||||
NucleiConfigDirEnv = "NUCLEI_CONFIG_DIR"
|
||||
NucleiTemplatesDirEnv = "NUCLEI_TEMPLATES_DIR"
|
||||
)
|
||||
|
||||
// IsOutdatedVersion compares two versions and returns true
|
||||
|
||||
@@ -2,7 +2,6 @@ package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"runtime/debug"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"gopkg.in/yaml.v2"
|
||||
@@ -18,7 +17,7 @@ type IgnoreFile struct {
|
||||
func ReadIgnoreFile() IgnoreFile {
|
||||
file, err := os.Open(DefaultConfig.GetIgnoreFilePath())
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n%s\n", err, string(debug.Stack()))
|
||||
gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err)
|
||||
return IgnoreFile{}
|
||||
}
|
||||
defer func() {
|
||||
|
||||
@@ -200,8 +200,8 @@ func (c *Config) GetNewAdditions() []string {
|
||||
if err != nil {
|
||||
return arr
|
||||
}
|
||||
for _, v := range strings.Fields(string(bin)) {
|
||||
if IsTemplate(v) {
|
||||
for v := range strings.FieldsSeq(string(bin)) {
|
||||
if IsTemplateWithRoot(v, c.TemplatesDirectory) {
|
||||
arr = append(arr, v)
|
||||
}
|
||||
}
|
||||
@@ -350,7 +350,7 @@ func (c *Config) IsDebugArgEnabled(arg string) bool {
|
||||
|
||||
// parseDebugArgs from string
|
||||
func (c *Config) parseDebugArgs(data string) {
|
||||
// use space as seperator instead of commas
|
||||
// use space as separator instead of commas
|
||||
tmp := strings.Fields(data)
|
||||
for _, v := range tmp {
|
||||
key := v
|
||||
|
||||
@@ -12,7 +12,10 @@ import (
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
)
|
||||
|
||||
var knownConfigFiles = []string{"cves.json", "contributors.json", "TEMPLATES-STATS.json"}
|
||||
var (
|
||||
knownConfigFiles = []string{"cves.json", "contributors.json", "TEMPLATES-STATS.json"}
|
||||
knownMiscDirectories = []string{".git", ".github", "helpers"}
|
||||
)
|
||||
|
||||
// TemplateFormat
|
||||
type TemplateFormat uint8
|
||||
@@ -23,6 +26,25 @@ const (
|
||||
Unknown
|
||||
)
|
||||
|
||||
// GetKnownConfigFiles returns known config files.
|
||||
func GetKnownConfigFiles() []string {
|
||||
return knownConfigFiles
|
||||
}
|
||||
|
||||
// GetKnownMiscDirectories returns known misc directories with trailing slashes.
|
||||
//
|
||||
// The trailing slash ensures that directory matching is explicit and avoids
|
||||
// falsely match files with similar names (e.g. "helpers" matching
|
||||
// "some-helpers.yaml"), since [IsTemplate] checks against normalized full paths.
|
||||
func GetKnownMiscDirectories() []string {
|
||||
trailedSlashDirs := make([]string, 0, len(knownMiscDirectories))
|
||||
for _, dir := range knownMiscDirectories {
|
||||
trailedSlashDirs = append(trailedSlashDirs, dir+string(os.PathSeparator))
|
||||
}
|
||||
|
||||
return trailedSlashDirs
|
||||
}
|
||||
|
||||
// GetTemplateFormatFromExt returns template format
|
||||
func GetTemplateFormatFromExt(filePath string) TemplateFormat {
|
||||
fileExt := strings.ToLower(filepath.Ext(filePath))
|
||||
@@ -41,13 +63,54 @@ func GetSupportTemplateFileExtensions() []string {
|
||||
return []string{extensions.YAML, extensions.JSON}
|
||||
}
|
||||
|
||||
// IsTemplate is a callback function used by goflags to decide if given file should be read
|
||||
// if it is not a nuclei-template file only then file is read
|
||||
func IsTemplate(filename string) bool {
|
||||
if stringsutil.ContainsAny(filename, knownConfigFiles...) {
|
||||
// IsTemplate returns true if the file is a template based on its path.
|
||||
// It used by goflags and other places to filter out non-template files.
|
||||
func IsTemplate(fpath string) bool {
|
||||
return IsTemplateWithRoot(fpath, "")
|
||||
}
|
||||
|
||||
// IsTemplateWithRoot returns true if the file is a template based on its path
|
||||
// and root directory. If rootDir is provided, it checks for excluded
|
||||
// directories relative to the root.
|
||||
func IsTemplateWithRoot(fpath, rootDir string) bool {
|
||||
fpath = filepath.FromSlash(fpath)
|
||||
fname := filepath.Base(fpath)
|
||||
fext := strings.ToLower(filepath.Ext(fpath))
|
||||
|
||||
if stringsutil.ContainsAny(fname, GetKnownConfigFiles()...) {
|
||||
return false
|
||||
}
|
||||
return stringsutil.EqualFoldAny(filepath.Ext(filename), GetSupportTemplateFileExtensions()...)
|
||||
|
||||
var pathToCheck string
|
||||
if rootDir != "" {
|
||||
if filepath.IsAbs(fpath) {
|
||||
rel, err := filepath.Rel(rootDir, fpath)
|
||||
if err == nil && !strings.HasPrefix(rel, "..") {
|
||||
pathToCheck = rel
|
||||
} else {
|
||||
pathToCheck = fpath
|
||||
}
|
||||
} else {
|
||||
pathToCheck = fpath
|
||||
}
|
||||
} else {
|
||||
pathToCheck = fpath
|
||||
}
|
||||
|
||||
// Only check components if pathToCheck is NOT absolute
|
||||
// This avoids false positives on parent directories for absolute paths
|
||||
if !filepath.IsAbs(pathToCheck) {
|
||||
parts := strings.Split(pathToCheck, string(os.PathSeparator))
|
||||
for _, p := range parts {
|
||||
for _, excluded := range knownMiscDirectories {
|
||||
if strings.EqualFold(p, excluded) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return stringsutil.EqualFoldAny(fext, GetSupportTemplateFileExtensions()...)
|
||||
}
|
||||
|
||||
type template struct {
|
||||
@@ -91,11 +154,21 @@ func GetNucleiTemplatesIndex() (map[string]string, error) {
|
||||
if err == nil {
|
||||
for _, v := range records {
|
||||
if len(v) >= 2 {
|
||||
index[v[0]] = v[1]
|
||||
templateID := v[0]
|
||||
templatePath := v[1]
|
||||
// Normalize path for consistent comparison (handles Windows path issues)
|
||||
normalizedPath := filepath.Clean(templatePath)
|
||||
// Validate that the file actually exists (prevents stale entries from deleted files on Windows)
|
||||
if fileutil.FileExists(normalizedPath) {
|
||||
index[templateID] = normalizedPath
|
||||
}
|
||||
}
|
||||
}
|
||||
// Close file handle before returning
|
||||
_ = f.Close()
|
||||
return index, nil
|
||||
}
|
||||
_ = f.Close()
|
||||
}
|
||||
DefaultConfig.Logger.Error().Msgf("failed to read index file creating new one: %v", err)
|
||||
}
|
||||
@@ -111,16 +184,18 @@ func GetNucleiTemplatesIndex() (map[string]string, error) {
|
||||
DefaultConfig.Logger.Verbose().Msgf("failed to walk path=%v err=%v", path, err)
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() || !IsTemplate(path) || stringsutil.ContainsAny(path, ignoreDirs...) {
|
||||
if d.IsDir() || !IsTemplateWithRoot(path, DefaultConfig.TemplatesDirectory) || stringsutil.ContainsAny(path, ignoreDirs...) {
|
||||
return nil
|
||||
}
|
||||
// Normalize path for consistent comparison (handles Windows path issues)
|
||||
normalizedPath := filepath.Clean(path)
|
||||
// get template id from file
|
||||
id, err := getTemplateID(path)
|
||||
id, err := getTemplateID(normalizedPath)
|
||||
if err != nil || id == "" {
|
||||
DefaultConfig.Logger.Verbose().Msgf("failed to get template id from file=%v got id=%v err=%v", path, id, err)
|
||||
DefaultConfig.Logger.Verbose().Msgf("failed to get template id from file=%v got id=%v err=%v", normalizedPath, id, err)
|
||||
return nil
|
||||
}
|
||||
index[id] = path
|
||||
index[id] = normalizedPath
|
||||
return nil
|
||||
})
|
||||
return index, err
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
osutils "github.com/projectdiscovery/utils/os"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func toAbs(p string) string {
|
||||
if osutils.IsWindows() {
|
||||
// Infer the drive letter from the Windows system path
|
||||
// SystemRoot or WINDIR typically points to something like "C:\Windows"
|
||||
systemRoot := os.Getenv("SystemRoot")
|
||||
if systemRoot == "" {
|
||||
systemRoot = os.Getenv("WINDIR")
|
||||
}
|
||||
// Extract volume name (e.g., "C:") from the system path
|
||||
volumeName := filepath.VolumeName(systemRoot)
|
||||
if volumeName == "" {
|
||||
// Fallback to C: if we can't determine the volume
|
||||
volumeName = "C:"
|
||||
}
|
||||
return filepath.FromSlash(volumeName + p)
|
||||
}
|
||||
return filepath.FromSlash(p)
|
||||
}
|
||||
|
||||
func TestIsTemplate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fpath string
|
||||
rootDir string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "valid template",
|
||||
fpath: "dns/cname.yaml",
|
||||
rootDir: "",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "excluded directory relative",
|
||||
fpath: "helpers/data.txt",
|
||||
rootDir: "",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "excluded directory .git",
|
||||
fpath: ".git/config",
|
||||
rootDir: "",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "absolute path with excluded parent dir (bug fix)",
|
||||
fpath: toAbs("/path/to/somewhere/that/has/helpers/dir/nuclei/nuclei-templates/dns/cname.yaml"),
|
||||
rootDir: toAbs("/path/to/somewhere/that/has/helpers/dir/nuclei/nuclei-templates"),
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "absolute path with excluded dir inside root",
|
||||
fpath: toAbs("/path/to/somewhere/that/has/helpers/dir/nuclei/nuclei-templates/helpers/data.txt"),
|
||||
rootDir: toAbs("/path/to/somewhere/that/has/helpers/dir/nuclei/nuclei-templates"),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "absolute path without root (skip check)",
|
||||
fpath: toAbs("/opt/helpers/foo.yaml"),
|
||||
rootDir: "",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "valid template with different extension",
|
||||
fpath: "dns/cname.json",
|
||||
rootDir: "",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "invalid extension",
|
||||
fpath: "dns/cname.txt",
|
||||
rootDir: "",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "excluded config file",
|
||||
fpath: "cves.json",
|
||||
rootDir: "",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := IsTemplateWithRoot(tt.fpath, tt.rootDir)
|
||||
require.Equal(t, tt.want, got, "IsTemplateWithRoot(%q, %q)", tt.fpath, tt.rootDir)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -38,13 +38,7 @@ func NewFSCatalog(fs fs.FS, directory string) *DiskCatalog {
|
||||
// It is used to read template and payload files based on catalog responses.
|
||||
func (d *DiskCatalog) OpenFile(filename string) (io.ReadCloser, error) {
|
||||
if d.templatesFS == nil {
|
||||
file, err := os.Open(filename)
|
||||
if err != nil {
|
||||
if file, errx := os.Open(BackwardsCompatiblePaths(d.templatesDirectory, filename)); errx == nil {
|
||||
return file, nil
|
||||
}
|
||||
}
|
||||
return file, err
|
||||
return os.Open(filename)
|
||||
}
|
||||
|
||||
return d.templatesFS.Open(filename)
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
package disk
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
ErrNoTemplatesFound = errors.New("no templates found")
|
||||
)
|
||||
+48
-91
@@ -1,6 +1,7 @@
|
||||
package disk
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -65,41 +66,34 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri
|
||||
// or folders provided as in.
|
||||
func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) {
|
||||
processed := make(map[string]struct{})
|
||||
// Template input includes a wildcard
|
||||
if strings.Contains(target, "*") {
|
||||
matches, findErr := c.findGlobPathMatches(target, processed)
|
||||
if findErr != nil {
|
||||
return nil, errors.Wrap(findErr, "could not find glob matches")
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
return nil, errors.Errorf("no templates found for path")
|
||||
}
|
||||
return matches, nil
|
||||
}
|
||||
|
||||
// try to handle deprecated template paths
|
||||
absPath := target
|
||||
if c.templatesFS == nil {
|
||||
absPath = BackwardsCompatiblePaths(c.templatesDirectory, target)
|
||||
if absPath != target && strings.TrimPrefix(absPath, c.templatesDirectory+string(filepath.Separator)) != target {
|
||||
if config.DefaultConfig.LogAllEvents {
|
||||
config.DefaultConfig.Logger.Print().Msgf("[%v] requested Template path %s is deprecated, please update to %s\n", aurora.Yellow("WRN").String(), target, absPath)
|
||||
}
|
||||
deprecatedPathsCounter++
|
||||
}
|
||||
|
||||
var err error
|
||||
absPath, err = c.convertPathToAbsolute(absPath)
|
||||
target, err = c.convertPathToAbsolute(target)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "could not find template file")
|
||||
}
|
||||
}
|
||||
|
||||
// Template input is either a file or a directory
|
||||
match, file, err := c.findFileMatches(absPath, processed)
|
||||
if strings.Contains(target, "*") {
|
||||
globMatches, err := c.findGlobPathMatches(target, processed)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not globbing path")
|
||||
}
|
||||
|
||||
if len(globMatches) > 0 {
|
||||
return globMatches, nil
|
||||
} else {
|
||||
return globMatches, fmt.Errorf("%w in path %q", ErrNoTemplatesFound, target)
|
||||
}
|
||||
}
|
||||
|
||||
// `target` is either a file or a directory
|
||||
match, file, err := c.findFileMatches(target, processed)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not find file")
|
||||
}
|
||||
|
||||
if file {
|
||||
if match != "" {
|
||||
return []string{match}, nil
|
||||
@@ -109,13 +103,15 @@ func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) {
|
||||
|
||||
// Recursively walk down the Templates directory and run all
|
||||
// the template file checks
|
||||
matches, err := c.findDirectoryMatches(absPath, processed)
|
||||
matches, err := c.findDirectoryMatches(target, processed)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not find directory matches")
|
||||
}
|
||||
|
||||
if len(matches) == 0 {
|
||||
return nil, errors.Errorf("no templates found in path %s", absPath)
|
||||
return nil, fmt.Errorf("%w in path %q", ErrNoTemplatesFound, target)
|
||||
}
|
||||
|
||||
return matches, nil
|
||||
}
|
||||
|
||||
@@ -135,72 +131,29 @@ func (c *DiskCatalog) convertPathToAbsolute(t string) (string, error) {
|
||||
|
||||
// findGlobPathMatches returns the matched files from a glob path
|
||||
func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) {
|
||||
// to support globbing on old paths we use brute force to find matches with exit on first match
|
||||
// trim templateDir if any
|
||||
relPath := strings.TrimPrefix(absPath, c.templatesDirectory)
|
||||
// trim leading slash if any
|
||||
relPath = strings.TrimPrefix(relPath, string(os.PathSeparator))
|
||||
|
||||
OldPathsResolver := func(inputGlob string) []string {
|
||||
templateDir := c.templatesDirectory
|
||||
if c.templatesDirectory == "" {
|
||||
templateDir = "./"
|
||||
}
|
||||
|
||||
if c.templatesFS == nil {
|
||||
matches, _ := fs.Glob(os.DirFS(filepath.Join(templateDir, "http")), inputGlob)
|
||||
if len(matches) != 0 {
|
||||
return matches
|
||||
}
|
||||
|
||||
// condition to support network cve related globs
|
||||
matches, _ = fs.Glob(os.DirFS(filepath.Join(templateDir, "network")), inputGlob)
|
||||
return matches
|
||||
} else {
|
||||
sub, err := fs.Sub(c.templatesFS, filepath.Join(templateDir, "http"))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
matches, _ := fs.Glob(sub, inputGlob)
|
||||
if len(matches) != 0 {
|
||||
return matches
|
||||
}
|
||||
|
||||
// condition to support network cve related globs
|
||||
sub, err = fs.Sub(c.templatesFS, filepath.Join(templateDir, "network"))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
matches, _ = fs.Glob(sub, inputGlob)
|
||||
return matches
|
||||
}
|
||||
}
|
||||
|
||||
var matched []string
|
||||
var matches []string
|
||||
if c.templatesFS == nil {
|
||||
var err error
|
||||
matches, err = filepath.Glob(relPath)
|
||||
if len(matches) != 0 {
|
||||
matched = append(matched, matches...)
|
||||
} else {
|
||||
matched = append(matched, OldPathsResolver(relPath)...)
|
||||
}
|
||||
if err != nil && len(matched) == 0 {
|
||||
return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
|
||||
}
|
||||
if c.templatesFS != nil {
|
||||
// fs.FS always uses forward slashes
|
||||
relPath = strings.TrimPrefix(relPath, "/")
|
||||
} else {
|
||||
var err error
|
||||
matches, err = fs.Glob(c.templatesFS, relPath)
|
||||
if len(matches) != 0 {
|
||||
matched = append(matched, matches...)
|
||||
} else {
|
||||
matched = append(matched, OldPathsResolver(relPath)...)
|
||||
}
|
||||
if err != nil && len(matched) == 0 {
|
||||
return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
|
||||
}
|
||||
relPath = strings.TrimPrefix(relPath, string(os.PathSeparator))
|
||||
}
|
||||
|
||||
var err error
|
||||
var matches []string
|
||||
|
||||
if c.templatesFS != nil {
|
||||
matches, err = fs.Glob(c.templatesFS, relPath)
|
||||
} else {
|
||||
matches, err = filepath.Glob(absPath)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
results := make([]string, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
if _, ok := processed[match]; !ok {
|
||||
@@ -208,6 +161,7 @@ func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]s
|
||||
results = append(results, match)
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
@@ -257,7 +211,7 @@ func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown {
|
||||
if !d.IsDir() && config.IsTemplateWithRoot(path, absPath) {
|
||||
if _, ok := processed[path]; !ok {
|
||||
results = append(results, path)
|
||||
processed[path] = struct{}{}
|
||||
@@ -281,7 +235,7 @@ func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown {
|
||||
if !d.IsDir() && config.IsTemplateWithRoot(path, absPath) {
|
||||
if _, ok := processed[path]; !ok {
|
||||
results = append(results, path)
|
||||
processed[path] = struct{}{}
|
||||
@@ -294,8 +248,11 @@ func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]
|
||||
return results, err
|
||||
}
|
||||
|
||||
// PrintDeprecatedPathsMsgIfApplicable prints a warning message if any deprecated paths are found
|
||||
// Unless mode is silent warning message is printed
|
||||
// PrintDeprecatedPathsMsgIfApplicable prints a warning message if any
|
||||
// deprecated paths are found. Unless mode is silent warning message is printed.
|
||||
//
|
||||
// Deprecated: No longer used since the official Nuclei Templates repository
|
||||
// have restructured this a long time ago.
|
||||
func PrintDeprecatedPathsMsgIfApplicable(isSilent bool) {
|
||||
if !updateutils.IsOutdated("v9.4.3", config.DefaultConfig.TemplateVersion) {
|
||||
return
|
||||
|
||||
@@ -72,6 +72,9 @@ func (c *DiskCatalog) tryResolve(fullPath string) (string, error) {
|
||||
|
||||
// BackwardsCompatiblePaths returns new paths for all old/legacy template paths
|
||||
// Note: this is a temporary function and will be removed in the future release
|
||||
//
|
||||
// Deprecated: No longer used since the official Nuclei Templates repository
|
||||
// have restructured this a long time ago.
|
||||
func BackwardsCompatiblePaths(templateDir string, oldPath string) string {
|
||||
// TODO: remove this function in the future release
|
||||
// 1. all http related paths are now moved at path /http
|
||||
|
||||
@@ -0,0 +1,344 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
)
|
||||
|
||||
// Filter represents filtering criteria for template metadata.
|
||||
//
|
||||
// Inclusion fields (e.g., Authors, Tags, IDs, Severities, ProtocolTypes) use
|
||||
// AND logic across different filter types and OR logic within each type.
|
||||
// Exclusion fields (e.g., ExcludeTags, ExcludeIDs, ExcludeSeverities,
|
||||
// ExcludeProtocolTypes) take precedence over inclusion fields. Additionally,
|
||||
// IncludeTemplates and IncludeTags can force inclusion of templates even if
|
||||
// they match exclusion criteria.
|
||||
type Filter struct {
|
||||
// Authors to include.
|
||||
Authors []string
|
||||
|
||||
// Tags to include.
|
||||
Tags []string
|
||||
|
||||
// ExcludeTags to exclude (takes precedence over Tags).
|
||||
ExcludeTags []string
|
||||
|
||||
// IncludeTags to force include even if excluded.
|
||||
IncludeTags []string
|
||||
|
||||
// IDs to include (supports wildcards, OR logic).
|
||||
IDs []string
|
||||
|
||||
// ExcludeIDs to exclude (supports wildcards).
|
||||
ExcludeIDs []string
|
||||
|
||||
// IncludeTemplates paths to force include even if excluded.
|
||||
IncludeTemplates []string
|
||||
|
||||
// ExcludeTemplates paths to exclude.
|
||||
ExcludeTemplates []string
|
||||
|
||||
// Severities to include.
|
||||
Severities []severity.Severity
|
||||
|
||||
// ExcludeSeverities to exclude.
|
||||
ExcludeSeverities []severity.Severity
|
||||
|
||||
// ProtocolTypes to include.
|
||||
ProtocolTypes []types.ProtocolType
|
||||
|
||||
// ExcludeProtocolTypes to exclude.
|
||||
ExcludeProtocolTypes []types.ProtocolType
|
||||
}
|
||||
|
||||
// Matches checks if metadata matches the filter criteria.
|
||||
func (f *Filter) Matches(m *Metadata) bool {
|
||||
if f.isForcedInclude(m) {
|
||||
return true
|
||||
}
|
||||
|
||||
if f.isExcluded(m) {
|
||||
return false
|
||||
}
|
||||
|
||||
if !f.matchesIncludes(m) {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// isForcedInclude checks if template is forced to be included.
|
||||
func (f *Filter) isForcedInclude(m *Metadata) bool {
|
||||
if len(f.IncludeTemplates) > 0 {
|
||||
for _, includePath := range f.IncludeTemplates {
|
||||
if matchesPath(m.FilePath, includePath) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(f.IncludeTags) > 0 {
|
||||
if slices.ContainsFunc(f.IncludeTags, m.HasTag) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// isExcluded checks if template should be excluded.
|
||||
func (f *Filter) isExcluded(m *Metadata) bool {
|
||||
if len(f.ExcludeTemplates) > 0 {
|
||||
for _, excludePath := range f.ExcludeTemplates {
|
||||
if matchesPath(m.FilePath, excludePath) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(f.ExcludeTags) > 0 {
|
||||
if slices.ContainsFunc(f.ExcludeTags, m.HasTag) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if len(f.ExcludeIDs) > 0 {
|
||||
for _, excludeID := range f.ExcludeIDs {
|
||||
if matchesID(m.ID, excludeID) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(f.ExcludeSeverities) > 0 {
|
||||
if slices.ContainsFunc(f.ExcludeSeverities, m.MatchesSeverity) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if len(f.ExcludeProtocolTypes) > 0 {
|
||||
if slices.ContainsFunc(f.ExcludeProtocolTypes, m.MatchesProtocol) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// matchesIncludes checks if metadata matches include filters.
|
||||
//
|
||||
// Returns true if no include filters are specified, or if all specified filter
|
||||
// types match.
|
||||
func (f *Filter) matchesIncludes(m *Metadata) bool {
|
||||
if len(f.Authors) > 0 {
|
||||
if !slices.ContainsFunc(f.Authors, m.HasAuthor) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if len(f.Tags) > 0 {
|
||||
if !slices.ContainsFunc(f.Tags, m.HasTag) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if len(f.IDs) > 0 {
|
||||
matched := false
|
||||
for _, id := range f.IDs {
|
||||
if matchesID(m.ID, id) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if len(f.Severities) > 0 {
|
||||
if !slices.ContainsFunc(f.Severities, m.MatchesSeverity) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if len(f.ProtocolTypes) > 0 {
|
||||
if !slices.ContainsFunc(f.ProtocolTypes, m.MatchesProtocol) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// matchesID checks if template ID matches pattern (supports wildcards).
|
||||
func matchesID(templateID, pattern string) bool {
|
||||
// Convert to lowercase for case-insensitive matching
|
||||
templateID = strings.ToLower(templateID)
|
||||
pattern = strings.ToLower(pattern)
|
||||
|
||||
if templateID == pattern {
|
||||
return true
|
||||
}
|
||||
|
||||
matched, _ := filepath.Match(pattern, templateID)
|
||||
|
||||
return matched
|
||||
}
|
||||
|
||||
// matchesPath checks if template path matches pattern.
|
||||
func matchesPath(templatePath, pattern string) bool {
|
||||
templatePath = filepath.Clean(templatePath)
|
||||
pattern = filepath.Clean(pattern)
|
||||
|
||||
if templatePath == pattern {
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasPrefix(templatePath, pattern+string(filepath.Separator)) {
|
||||
return true
|
||||
}
|
||||
|
||||
matched, _ := filepath.Match(pattern, templatePath)
|
||||
|
||||
return matched
|
||||
}
|
||||
|
||||
// FilterFunc is a function that filters metadata.
|
||||
type FilterFunc func(*Metadata) bool
|
||||
|
||||
// UnmarshalFilter creates a Filter from nuclei options.
|
||||
func UnmarshalFilter(
|
||||
authors, tags, excludeTags, includeTags []string,
|
||||
ids, excludeIDs []string,
|
||||
includeTemplates, excludeTemplates []string,
|
||||
severities, excludeSeverities []string,
|
||||
protocolTypes, excludeProtocolTypes []string,
|
||||
) (*Filter, error) {
|
||||
filter := &Filter{
|
||||
Authors: authors,
|
||||
Tags: tags,
|
||||
ExcludeTags: excludeTags,
|
||||
IncludeTags: includeTags,
|
||||
IDs: ids,
|
||||
ExcludeIDs: excludeIDs,
|
||||
IncludeTemplates: includeTemplates,
|
||||
ExcludeTemplates: excludeTemplates,
|
||||
}
|
||||
|
||||
for _, sev := range severities {
|
||||
holder := &severity.Holder{}
|
||||
if err := holder.UnmarshalYAML(func(v interface{}) error {
|
||||
*v.(*string) = sev
|
||||
return nil
|
||||
}); err == nil {
|
||||
filter.Severities = append(filter.Severities, holder.Severity)
|
||||
}
|
||||
}
|
||||
|
||||
for _, sev := range excludeSeverities {
|
||||
holder := &severity.Holder{}
|
||||
if err := holder.UnmarshalYAML(func(v interface{}) error {
|
||||
*v.(*string) = sev
|
||||
return nil
|
||||
}); err == nil {
|
||||
filter.ExcludeSeverities = append(filter.ExcludeSeverities, holder.Severity)
|
||||
}
|
||||
}
|
||||
|
||||
for _, pt := range protocolTypes {
|
||||
holder := &types.TypeHolder{}
|
||||
if err := holder.UnmarshalYAML(func(v interface{}) error {
|
||||
*v.(*string) = pt
|
||||
return nil
|
||||
}); err == nil && holder.ProtocolType != types.InvalidProtocol {
|
||||
filter.ProtocolTypes = append(filter.ProtocolTypes, holder.ProtocolType)
|
||||
}
|
||||
}
|
||||
|
||||
for _, pt := range excludeProtocolTypes {
|
||||
holder := &types.TypeHolder{}
|
||||
if err := holder.UnmarshalYAML(func(v interface{}) error {
|
||||
*v.(*string) = pt
|
||||
return nil
|
||||
}); err == nil && holder.ProtocolType != types.InvalidProtocol {
|
||||
filter.ExcludeProtocolTypes = append(filter.ExcludeProtocolTypes, holder.ProtocolType)
|
||||
}
|
||||
}
|
||||
|
||||
return filter, nil
|
||||
}
|
||||
|
||||
// UnmarshalFilterFunc creates a FilterFunc from filter criteria.
|
||||
func UnmarshalFilterFunc(filter *Filter) FilterFunc {
|
||||
if filter == nil {
|
||||
return func(*Metadata) bool { return true }
|
||||
}
|
||||
|
||||
return filter.Matches
|
||||
}
|
||||
|
||||
// IsEmpty returns true if filter has no criteria set.
|
||||
func (f *Filter) IsEmpty() bool {
|
||||
return len(f.Authors) == 0 &&
|
||||
len(f.Tags) == 0 &&
|
||||
len(f.ExcludeTags) == 0 &&
|
||||
len(f.IncludeTags) == 0 &&
|
||||
len(f.IDs) == 0 &&
|
||||
len(f.ExcludeIDs) == 0 &&
|
||||
len(f.IncludeTemplates) == 0 &&
|
||||
len(f.ExcludeTemplates) == 0 &&
|
||||
len(f.Severities) == 0 &&
|
||||
len(f.ExcludeSeverities) == 0 &&
|
||||
len(f.ProtocolTypes) == 0 &&
|
||||
len(f.ExcludeProtocolTypes) == 0
|
||||
}
|
||||
|
||||
// String returns a human-readable representation of the filter.
|
||||
func (f *Filter) String() string {
|
||||
var parts []string
|
||||
|
||||
if len(f.Authors) > 0 {
|
||||
parts = append(parts, "authors="+strings.Join(f.Authors, ","))
|
||||
}
|
||||
|
||||
if len(f.Tags) > 0 {
|
||||
parts = append(parts, "tags="+strings.Join(f.Tags, ","))
|
||||
}
|
||||
|
||||
if len(f.ExcludeTags) > 0 {
|
||||
parts = append(parts, "exclude-tags="+strings.Join(f.ExcludeTags, ","))
|
||||
}
|
||||
|
||||
if len(f.IDs) > 0 {
|
||||
parts = append(parts, "ids="+strings.Join(f.IDs, ","))
|
||||
}
|
||||
|
||||
if len(f.Severities) > 0 {
|
||||
sevs := make([]string, len(f.Severities))
|
||||
for i, s := range f.Severities {
|
||||
sevs[i] = s.String()
|
||||
}
|
||||
|
||||
parts = append(parts, "severities="+strings.Join(sevs, ","))
|
||||
}
|
||||
|
||||
if len(f.ProtocolTypes) > 0 {
|
||||
pts := make([]string, len(f.ProtocolTypes))
|
||||
for i, p := range f.ProtocolTypes {
|
||||
pts[i] = p.String()
|
||||
}
|
||||
|
||||
parts = append(parts, "types="+strings.Join(pts, ","))
|
||||
}
|
||||
|
||||
if len(parts) == 0 {
|
||||
return "filter=<nil>"
|
||||
}
|
||||
|
||||
return "filter(" + strings.Join(parts, ", ") + ")"
|
||||
}
|
||||
@@ -0,0 +1,407 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFilterMatches(t *testing.T) {
|
||||
metadata := &Metadata{
|
||||
ID: "test-template-1",
|
||||
FilePath: "/templates/cves/2021/CVE-2021-1234.yaml",
|
||||
Name: "Test CVE Template",
|
||||
Authors: []string{"pdteam", "geeknik"},
|
||||
Tags: []string{"cve", "rce", "apache"},
|
||||
Severity: "critical",
|
||||
ProtocolType: "http",
|
||||
}
|
||||
|
||||
t.Run("Empty filter matches all", func(t *testing.T) {
|
||||
filter := &Filter{}
|
||||
require.True(t, filter.Matches(metadata))
|
||||
require.True(t, filter.IsEmpty())
|
||||
})
|
||||
|
||||
t.Run("Author filter - match", func(t *testing.T) {
|
||||
filter := &Filter{Authors: []string{"pdteam"}}
|
||||
require.True(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Author filter - no match", func(t *testing.T) {
|
||||
filter := &Filter{Authors: []string{"unknown"}}
|
||||
require.False(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Multiple authors - OR logic", func(t *testing.T) {
|
||||
filter := &Filter{Authors: []string{"unknown", "geeknik"}}
|
||||
require.True(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Tag filter - match", func(t *testing.T) {
|
||||
filter := &Filter{Tags: []string{"cve"}}
|
||||
require.True(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Tag filter - no match", func(t *testing.T) {
|
||||
filter := &Filter{Tags: []string{"xss"}}
|
||||
require.False(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Exclude tags - match", func(t *testing.T) {
|
||||
filter := &Filter{ExcludeTags: []string{"rce"}}
|
||||
require.False(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Include tags overrides exclude", func(t *testing.T) {
|
||||
filter := &Filter{
|
||||
ExcludeTags: []string{"rce"},
|
||||
IncludeTags: []string{"cve"},
|
||||
}
|
||||
require.True(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("ID filter - exact match", func(t *testing.T) {
|
||||
filter := &Filter{IDs: []string{"test-template-1"}}
|
||||
require.True(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("ID filter - wildcard match", func(t *testing.T) {
|
||||
filter := &Filter{IDs: []string{"test-*"}}
|
||||
require.True(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("ID filter - no match", func(t *testing.T) {
|
||||
filter := &Filter{IDs: []string{"other-*"}}
|
||||
require.False(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Exclude ID - exact match", func(t *testing.T) {
|
||||
filter := &Filter{ExcludeIDs: []string{"test-template-1"}}
|
||||
require.False(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Exclude ID - wildcard match", func(t *testing.T) {
|
||||
filter := &Filter{ExcludeIDs: []string{"test-*"}}
|
||||
require.False(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Severity filter - match", func(t *testing.T) {
|
||||
filter := &Filter{Severities: []severity.Severity{severity.Critical}}
|
||||
require.True(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Severity filter - no match", func(t *testing.T) {
|
||||
filter := &Filter{Severities: []severity.Severity{severity.High, severity.Medium}}
|
||||
require.False(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Exclude severity - match", func(t *testing.T) {
|
||||
filter := &Filter{ExcludeSeverities: []severity.Severity{severity.Critical}}
|
||||
require.False(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Protocol type filter - match", func(t *testing.T) {
|
||||
filter := &Filter{ProtocolTypes: []types.ProtocolType{types.HTTPProtocol}}
|
||||
require.True(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Protocol type filter - no match", func(t *testing.T) {
|
||||
filter := &Filter{ProtocolTypes: []types.ProtocolType{types.DNSProtocol}}
|
||||
require.False(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Exclude protocol type - match", func(t *testing.T) {
|
||||
filter := &Filter{ExcludeProtocolTypes: []types.ProtocolType{types.HTTPProtocol}}
|
||||
require.False(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Include templates - path match", func(t *testing.T) {
|
||||
filter := &Filter{
|
||||
ExcludeTags: []string{"cve"},
|
||||
IncludeTemplates: []string{"/templates/cves/"},
|
||||
}
|
||||
require.True(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Exclude templates - path match", func(t *testing.T) {
|
||||
filter := &Filter{
|
||||
ExcludeTemplates: []string{"/templates/cves/"},
|
||||
}
|
||||
require.False(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Complex filter - all match", func(t *testing.T) {
|
||||
filter := &Filter{
|
||||
Authors: []string{"pdteam"},
|
||||
Tags: []string{"cve"},
|
||||
Severities: []severity.Severity{severity.Critical},
|
||||
ProtocolTypes: []types.ProtocolType{types.HTTPProtocol},
|
||||
}
|
||||
require.True(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Complex filter - AND logic across types", func(t *testing.T) {
|
||||
filter := &Filter{
|
||||
Authors: []string{"pdteam"}, // matches
|
||||
Tags: []string{"xss"}, // doesn't match
|
||||
Severities: []severity.Severity{severity.Critical}, // matches
|
||||
}
|
||||
// With AND logic across filter types, doesn't match because tags don't match
|
||||
// even though author and severity match
|
||||
require.False(t, filter.Matches(metadata))
|
||||
})
|
||||
|
||||
t.Run("Complex filter - no match at all", func(t *testing.T) {
|
||||
filter := &Filter{
|
||||
Authors: []string{"unknown"}, // doesn't match
|
||||
Tags: []string{"xss"}, // doesn't match
|
||||
Severities: []severity.Severity{severity.Low}, // doesn't match
|
||||
}
|
||||
require.False(t, filter.Matches(metadata))
|
||||
})
|
||||
}
|
||||
|
||||
func TestMatchesPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
pattern string
|
||||
expected bool
|
||||
}{
|
||||
{"exact match", "/templates/cves/2021/test.yaml", "/templates/cves/2021/test.yaml", true},
|
||||
{"directory prefix", "/templates/cves/2021/test.yaml", "/templates/cves", true},
|
||||
{"directory with slash", "/templates/cves/2021/test.yaml", "/templates/cves/", true},
|
||||
{"no match", "/templates/cves/2021/test.yaml", "/templates/exploits", false},
|
||||
{"wildcard match", "/templates/cves/2021/test.yaml", "/templates/*/2021/*.yaml", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := matchesPath(tt.path, tt.pattern)
|
||||
require.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
id string
|
||||
pattern string
|
||||
expected bool
|
||||
}{
|
||||
{"exact match", "CVE-2021-1234", "CVE-2021-1234", true},
|
||||
{"wildcard prefix", "CVE-2021-1234", "CVE-*", true},
|
||||
{"wildcard suffix", "CVE-2021-1234", "*-1234", true},
|
||||
{"wildcard middle", "CVE-2021-1234", "CVE-*-1234", true},
|
||||
{"no match", "CVE-2021-1234", "CVE-2022-*", false},
|
||||
{"partial no match", "CVE-2021-1234", "CVE-2021-12", false},
|
||||
{"case insensitive exact", "cve-2021-1234", "CVE-2021-1234", true},
|
||||
{"case insensitive wildcard", "CVE-2021-1234", "cve-*", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := matchesID(tt.id, tt.pattern)
|
||||
require.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnmarshalFilter(t *testing.T) {
|
||||
filter, err := UnmarshalFilter(
|
||||
[]string{"author1", "author2"},
|
||||
[]string{"tag1", "tag2"},
|
||||
[]string{"exclude-tag"},
|
||||
[]string{"include-tag"},
|
||||
[]string{"id1", "id2*"},
|
||||
[]string{"exclude-id*"},
|
||||
[]string{"/include/path"},
|
||||
[]string{"/exclude/path"},
|
||||
[]string{"critical", "high"},
|
||||
[]string{"info"},
|
||||
[]string{"http", "dns"},
|
||||
[]string{"file"},
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, filter)
|
||||
|
||||
require.Equal(t, []string{"author1", "author2"}, filter.Authors)
|
||||
require.Equal(t, []string{"tag1", "tag2"}, filter.Tags)
|
||||
require.Equal(t, []string{"exclude-tag"}, filter.ExcludeTags)
|
||||
require.Equal(t, []string{"include-tag"}, filter.IncludeTags)
|
||||
require.Equal(t, []string{"id1", "id2*"}, filter.IDs)
|
||||
require.Equal(t, []string{"exclude-id*"}, filter.ExcludeIDs)
|
||||
require.Equal(t, []string{"/include/path"}, filter.IncludeTemplates)
|
||||
require.Equal(t, []string{"/exclude/path"}, filter.ExcludeTemplates)
|
||||
|
||||
require.Len(t, filter.Severities, 2)
|
||||
require.Contains(t, filter.Severities, severity.Critical)
|
||||
require.Contains(t, filter.Severities, severity.High)
|
||||
|
||||
require.Len(t, filter.ExcludeSeverities, 1)
|
||||
require.Contains(t, filter.ExcludeSeverities, severity.Info)
|
||||
|
||||
require.Len(t, filter.ProtocolTypes, 2)
|
||||
require.Contains(t, filter.ProtocolTypes, types.HTTPProtocol)
|
||||
require.Contains(t, filter.ProtocolTypes, types.DNSProtocol)
|
||||
|
||||
require.Len(t, filter.ExcludeProtocolTypes, 1)
|
||||
require.Contains(t, filter.ExcludeProtocolTypes, types.FileProtocol)
|
||||
}
|
||||
|
||||
func TestIndexFilter(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
idx, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create test templates and metadata
|
||||
templates := []struct {
|
||||
id string
|
||||
path string
|
||||
authors []string
|
||||
tags []string
|
||||
severity string
|
||||
protocol string
|
||||
}{
|
||||
{"cve-2021-1", "/templates/cves/CVE-2021-1.yaml", []string{"pdteam"}, []string{"cve", "rce"}, "critical", "http"},
|
||||
{"cve-2021-2", "/templates/cves/CVE-2021-2.yaml", []string{"pdteam"}, []string{"cve", "xss"}, "high", "http"},
|
||||
{"exploit-1", "/templates/exploits/exploit-1.yaml", []string{"geeknik"}, []string{"exploit"}, "medium", "dns"},
|
||||
{"info-1", "/templates/info/info-1.yaml", []string{"author1"}, []string{"info"}, "info", "http"},
|
||||
}
|
||||
|
||||
for _, tmpl := range templates {
|
||||
tmpFile := filepath.Join(tmpDir, filepath.Base(tmpl.path))
|
||||
err := os.WriteFile(tmpFile, []byte("id: "+tmpl.id), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
metadata := &Metadata{
|
||||
ID: tmpl.id,
|
||||
FilePath: tmpFile,
|
||||
Authors: tmpl.authors,
|
||||
Tags: tmpl.tags,
|
||||
Severity: tmpl.severity,
|
||||
ProtocolType: tmpl.protocol,
|
||||
}
|
||||
idx.Set(tmpl.path, metadata)
|
||||
}
|
||||
|
||||
t.Run("No filter returns all", func(t *testing.T) {
|
||||
results := idx.Filter(nil)
|
||||
require.Len(t, results, 4)
|
||||
})
|
||||
|
||||
t.Run("Filter by author", func(t *testing.T) {
|
||||
filter := &Filter{Authors: []string{"pdteam"}}
|
||||
results := idx.Filter(filter)
|
||||
require.Len(t, results, 2)
|
||||
})
|
||||
|
||||
t.Run("Filter by tag", func(t *testing.T) {
|
||||
filter := &Filter{Tags: []string{"cve"}}
|
||||
results := idx.Filter(filter)
|
||||
require.Len(t, results, 2)
|
||||
})
|
||||
|
||||
t.Run("Filter by severity", func(t *testing.T) {
|
||||
filter := &Filter{Severities: []severity.Severity{severity.Critical}}
|
||||
results := idx.Filter(filter)
|
||||
require.Len(t, results, 1)
|
||||
})
|
||||
|
||||
t.Run("Filter by protocol type", func(t *testing.T) {
|
||||
filter := &Filter{ProtocolTypes: []types.ProtocolType{types.HTTPProtocol}}
|
||||
results := idx.Filter(filter)
|
||||
require.Len(t, results, 3)
|
||||
})
|
||||
|
||||
t.Run("Exclude by severity", func(t *testing.T) {
|
||||
filter := &Filter{ExcludeSeverities: []severity.Severity{severity.Info}}
|
||||
results := idx.Filter(filter)
|
||||
require.Len(t, results, 3)
|
||||
})
|
||||
|
||||
t.Run("Exclude by tag", func(t *testing.T) {
|
||||
filter := &Filter{ExcludeTags: []string{"info"}}
|
||||
results := idx.Filter(filter)
|
||||
require.Len(t, results, 3)
|
||||
})
|
||||
|
||||
t.Run("Complex filter", func(t *testing.T) {
|
||||
filter := &Filter{
|
||||
Tags: []string{"cve"},
|
||||
Severities: []severity.Severity{severity.Critical, severity.High},
|
||||
ExcludeSeverities: []severity.Severity{severity.Info},
|
||||
}
|
||||
results := idx.Filter(filter)
|
||||
require.Len(t, results, 2)
|
||||
})
|
||||
|
||||
t.Run("Count with filter", func(t *testing.T) {
|
||||
filter := &Filter{Tags: []string{"cve"}}
|
||||
count := idx.Count(filter)
|
||||
require.Equal(t, 2, count)
|
||||
})
|
||||
|
||||
t.Run("Count without filter", func(t *testing.T) {
|
||||
count := idx.Count(nil)
|
||||
require.Equal(t, 4, count)
|
||||
})
|
||||
}
|
||||
|
||||
func TestIndexFilterFunc(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
idx, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add test metadata
|
||||
for i := 0; i < 5; i++ {
|
||||
metadata := &Metadata{
|
||||
ID: "test-" + string(rune('a'+i)),
|
||||
FilePath: "/tmp/test.yaml",
|
||||
Severity: "high",
|
||||
}
|
||||
if i%2 == 0 {
|
||||
metadata.Tags = []string{"even"}
|
||||
} else {
|
||||
metadata.Tags = []string{"odd"}
|
||||
}
|
||||
idx.Set("/tmp/test-"+string(rune('a'+i))+".yaml", metadata)
|
||||
}
|
||||
|
||||
t.Run("Custom filter function", func(t *testing.T) {
|
||||
results := idx.FilterFunc(func(m *Metadata) bool {
|
||||
return m.HasTag("even")
|
||||
})
|
||||
require.Len(t, results, 3) // 0, 2, 4
|
||||
})
|
||||
|
||||
t.Run("Nil filter function returns all", func(t *testing.T) {
|
||||
results := idx.FilterFunc(nil)
|
||||
require.Len(t, results, 5)
|
||||
})
|
||||
}
|
||||
|
||||
func TestFilterString(t *testing.T) {
|
||||
filter := &Filter{
|
||||
Authors: []string{"author1", "author2"},
|
||||
Tags: []string{"tag1"},
|
||||
Severities: []severity.Severity{severity.Critical, severity.High},
|
||||
ProtocolTypes: []types.ProtocolType{types.HTTPProtocol},
|
||||
}
|
||||
|
||||
str := filter.String()
|
||||
require.Contains(t, str, "authors=")
|
||||
require.Contains(t, str, "tags=")
|
||||
require.Contains(t, str, "severities=")
|
||||
require.Contains(t, str, "types=")
|
||||
|
||||
emptyFilter := &Filter{}
|
||||
require.Equal(t, "filter=<nil>", emptyFilter.String())
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"encoding/gob"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/maypok86/otter/v2"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
folderutil "github.com/projectdiscovery/utils/folder"
|
||||
)
|
||||
|
||||
const (
|
||||
// IndexFileName is the name of the persistent cache file.
|
||||
IndexFileName = "index.gob"
|
||||
|
||||
// IndexVersion is the schema version for cache invalidation on breaking
|
||||
// changes.
|
||||
IndexVersion = 1
|
||||
|
||||
// DefaultMaxSize is the default maximum number of templates to cache.
|
||||
DefaultMaxSize = 50000
|
||||
|
||||
// DefaultMaxWeight is the default maximum weight of the cache.
|
||||
DefaultMaxWeight = DefaultMaxSize * 800 // ~40MB assuming ~800B/entry
|
||||
)
|
||||
|
||||
// Index represents a cache for template metadata.
|
||||
type Index struct {
|
||||
cache *otter.Cache[string, *Metadata]
|
||||
cacheFile string
|
||||
mu sync.RWMutex
|
||||
version int
|
||||
}
|
||||
|
||||
// cacheSnapshot represents the serialized cache structure.
|
||||
type cacheSnapshot struct {
|
||||
Version int `gob:"version"`
|
||||
Data map[string]*Metadata `gob:"data"`
|
||||
}
|
||||
|
||||
// NewIndex creates a new template metadata cache with the given options.
|
||||
func NewIndex(cacheDir string) (*Index, error) {
|
||||
if cacheDir == "" {
|
||||
cacheDir = folderutil.AppCacheDirOrDefault(".nuclei-cache", config.BinaryName)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(cacheDir, 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cacheFile := filepath.Join(cacheDir, IndexFileName)
|
||||
|
||||
// NOTE(dwisiswant0): Build cache with adaptive sizing based on memory cost.
|
||||
opts := &otter.Options[string, *Metadata]{
|
||||
MaximumWeight: uint64(DefaultMaxWeight),
|
||||
Weigher: func(key string, value *Metadata) uint32 {
|
||||
if value == nil {
|
||||
return uint32(len(key))
|
||||
}
|
||||
|
||||
weight := len(key)
|
||||
weight += len(value.ID)
|
||||
weight += len(value.FilePath)
|
||||
weight += 24 // ModTime is time.Time (24B)
|
||||
weight += len(value.Name)
|
||||
weight += len(value.Severity)
|
||||
weight += len(value.ProtocolType)
|
||||
weight += len(value.TemplateVerifier)
|
||||
|
||||
for _, author := range value.Authors {
|
||||
weight += len(author)
|
||||
}
|
||||
for _, tag := range value.Tags {
|
||||
weight += len(tag)
|
||||
}
|
||||
|
||||
return uint32(weight)
|
||||
},
|
||||
}
|
||||
|
||||
cache, err := otter.New(opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c := &Index{
|
||||
cache: cache,
|
||||
cacheFile: cacheFile,
|
||||
version: IndexVersion,
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// NewDefaultIndex creates a index with default settings in the default cache
|
||||
// directory.
|
||||
func NewDefaultIndex() (*Index, error) {
|
||||
return NewIndex("")
|
||||
}
|
||||
|
||||
// Get retrieves metadata for a template path, validating freshness via mtime.
|
||||
func (i *Index) Get(path string) (*Metadata, bool) {
|
||||
i.mu.RLock()
|
||||
defer i.mu.RUnlock()
|
||||
|
||||
metadata, found := i.cache.GetIfPresent(path)
|
||||
if !found {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
if !metadata.IsValid() {
|
||||
go i.Delete(path)
|
||||
|
||||
return nil, false
|
||||
}
|
||||
|
||||
return metadata, true
|
||||
}
|
||||
|
||||
// Set stores metadata for a template path.
|
||||
//
|
||||
// The caller is responsible for ensuring the metadata is valid and contains
|
||||
// the correct checksum before calling this method.
|
||||
// Use [SetFromTemplate] for automatic extraction and checksum computation.
|
||||
//
|
||||
// Returns the metadata and whether it was successfully cached (false if evicted).
|
||||
func (i *Index) Set(path string, metadata *Metadata) (*Metadata, bool) {
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
|
||||
return i.cache.Set(path, metadata)
|
||||
}
|
||||
|
||||
// SetFromTemplate extracts metadata from a parsed template and stores it.
|
||||
//
|
||||
// Returns the metadata and whether it was successfully cached. The metadata is
|
||||
// always returned (even on checksum failure) for immediate filtering use.
|
||||
// Returns false if the metadata was not cached (e.g., set, evicted).
|
||||
func (i *Index) SetFromTemplate(path string, tpl *templates.Template) (*Metadata, bool) {
|
||||
metadata := NewMetadataFromTemplate(path, tpl)
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return metadata, false
|
||||
}
|
||||
metadata.ModTime = info.ModTime()
|
||||
|
||||
if i.cache == nil {
|
||||
return metadata, false
|
||||
}
|
||||
|
||||
return i.Set(path, metadata)
|
||||
}
|
||||
|
||||
// Has checks if metadata exists for a path without validation.
|
||||
func (i *Index) Has(path string) bool {
|
||||
i.mu.RLock()
|
||||
defer i.mu.RUnlock()
|
||||
|
||||
_, found := i.cache.GetIfPresent(path)
|
||||
|
||||
return found
|
||||
}
|
||||
|
||||
// Delete removes metadata for a path.
|
||||
func (i *Index) Delete(path string) {
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
|
||||
i.cache.Invalidate(path)
|
||||
}
|
||||
|
||||
// Size returns the number of cached entries.
|
||||
func (i *Index) Size() int {
|
||||
i.mu.RLock()
|
||||
defer i.mu.RUnlock()
|
||||
|
||||
return i.cache.EstimatedSize()
|
||||
}
|
||||
|
||||
// Clear removes all cached entries.
|
||||
func (i *Index) Clear() {
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
|
||||
i.cache.InvalidateAll()
|
||||
}
|
||||
|
||||
// Save persists the cache to disk using gob encoding.
|
||||
func (i *Index) Save() error {
|
||||
i.mu.RLock()
|
||||
defer i.mu.RUnlock()
|
||||
|
||||
snapshot := &cacheSnapshot{
|
||||
Version: i.version,
|
||||
Data: make(map[string]*Metadata),
|
||||
}
|
||||
|
||||
maps.Insert(snapshot.Data, i.cache.All())
|
||||
|
||||
// NOTE(dwisiswant0): write to temp for atomic op.
|
||||
tmpFile := i.cacheFile + ".tmp"
|
||||
file, err := os.Create(tmpFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
encoder := gob.NewEncoder(file)
|
||||
if err := encoder.Encode(snapshot); err != nil {
|
||||
_ = file.Close()
|
||||
_ = os.Remove(tmpFile)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
if err := file.Close(); err != nil {
|
||||
_ = os.Remove(tmpFile)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.Rename(tmpFile, i.cacheFile); err != nil {
|
||||
_ = os.Remove(tmpFile)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Load loads the cache from disk using gob decoding.
|
||||
func (i *Index) Load() error {
|
||||
file, err := os.Open(i.cacheFile)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
var snapshot cacheSnapshot
|
||||
|
||||
decoder := gob.NewDecoder(file)
|
||||
if err := decoder.Decode(&snapshot); err != nil {
|
||||
_ = file.Close()
|
||||
_ = os.Remove(i.cacheFile)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if snapshot.Version != i.version {
|
||||
_ = file.Close()
|
||||
_ = os.Remove(i.cacheFile)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
|
||||
for key, value := range snapshot.Data {
|
||||
i.cache.Set(key, value)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Filter returns all template paths that match the given filter criteria.
|
||||
func (i *Index) Filter(filter *Filter) []string {
|
||||
if filter == nil || filter.IsEmpty() {
|
||||
return i.All()
|
||||
}
|
||||
|
||||
i.mu.RLock()
|
||||
defer i.mu.RUnlock()
|
||||
|
||||
var matched []string
|
||||
for path, metadata := range i.cache.All() {
|
||||
if filter.Matches(metadata) {
|
||||
matched = append(matched, path)
|
||||
}
|
||||
}
|
||||
|
||||
return matched
|
||||
}
|
||||
|
||||
// FilterFunc returns all template paths that match the given filter function.
|
||||
func (i *Index) FilterFunc(fn FilterFunc) []string {
|
||||
if fn == nil {
|
||||
return i.All()
|
||||
}
|
||||
|
||||
i.mu.RLock()
|
||||
defer i.mu.RUnlock()
|
||||
|
||||
var matched []string
|
||||
for path, metadata := range i.cache.All() {
|
||||
if fn(metadata) {
|
||||
matched = append(matched, path)
|
||||
}
|
||||
}
|
||||
|
||||
return matched
|
||||
}
|
||||
|
||||
// All returns all template paths in the index.
|
||||
func (i *Index) All() []string {
|
||||
i.mu.RLock()
|
||||
defer i.mu.RUnlock()
|
||||
|
||||
paths := make([]string, 0, i.cache.EstimatedSize())
|
||||
for path := range i.cache.All() {
|
||||
paths = append(paths, path)
|
||||
}
|
||||
|
||||
return paths
|
||||
}
|
||||
|
||||
// GetAll returns all metadata entries in the index.
|
||||
func (i *Index) GetAll() map[string]*Metadata {
|
||||
i.mu.RLock()
|
||||
defer i.mu.RUnlock()
|
||||
|
||||
result := maps.Collect(i.cache.All())
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// Count returns the number of templates matching the filter.
|
||||
func (i *Index) Count(filter *Filter) int {
|
||||
if filter == nil || filter.IsEmpty() {
|
||||
return i.Size()
|
||||
}
|
||||
|
||||
i.mu.RLock()
|
||||
defer i.mu.RUnlock()
|
||||
|
||||
count := 0
|
||||
for _, metadata := range i.cache.All() {
|
||||
if filter.Matches(metadata) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
return count
|
||||
}
|
||||
@@ -0,0 +1,737 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/code"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNewIndex(t *testing.T) {
|
||||
t.Run("with custom directory", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
cache, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err, "Failed to create cache with custom directory")
|
||||
require.NotNil(t, cache, "Cache should not be nil")
|
||||
require.Equal(t, filepath.Join(tmpDir, IndexFileName), cache.cacheFile)
|
||||
require.Equal(t, IndexVersion, cache.version)
|
||||
})
|
||||
|
||||
t.Run("with default directory", func(t *testing.T) {
|
||||
cache, err := NewDefaultIndex()
|
||||
require.NoError(t, err, "Failed to create cache with default directory")
|
||||
require.NotNil(t, cache, "Cache should not be nil")
|
||||
})
|
||||
}
|
||||
|
||||
func TestCacheBasicOperations(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
cache, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
metadata := &Metadata{
|
||||
ID: "concurrent-test",
|
||||
FilePath: "/tmp/concurrent.yaml",
|
||||
}
|
||||
|
||||
t.Run("Set and Has", func(t *testing.T) {
|
||||
cache.Set(metadata.FilePath, metadata)
|
||||
require.Equal(t, 1, cache.Size(), "Cache size should be 1 after Set")
|
||||
require.True(t, cache.Has(metadata.FilePath), "Cache should contain the path after Set")
|
||||
require.False(t, cache.Has("/nonexistent"), "Cache should not contain nonexistent path")
|
||||
})
|
||||
|
||||
t.Run("Get with validation", func(t *testing.T) {
|
||||
// Get should fail validation for nonexistent file
|
||||
retrieved, found := cache.Get(metadata.FilePath)
|
||||
require.False(t, found, "Get should fail validation for nonexistent file")
|
||||
require.Nil(t, retrieved, "Retrieved metadata should be nil for invalid entry")
|
||||
})
|
||||
|
||||
t.Run("Delete", func(t *testing.T) {
|
||||
cache.Set(metadata.FilePath, metadata)
|
||||
require.True(t, cache.Has(metadata.FilePath), "Cache should contain path before Delete")
|
||||
|
||||
cache.Delete(metadata.FilePath)
|
||||
require.False(t, cache.Has(metadata.FilePath), "Cache should not contain path after Delete")
|
||||
})
|
||||
|
||||
t.Run("Clear", func(t *testing.T) {
|
||||
cache.Set(metadata.FilePath, metadata)
|
||||
cache.Set("/tmp/test2.yaml", &Metadata{ID: "test2", FilePath: "/tmp/test2.yaml"})
|
||||
require.True(t, cache.Size() > 0, "Cache should have entries before Clear")
|
||||
|
||||
cache.Clear()
|
||||
require.Equal(t, 0, cache.Size(), "Cache should be empty after Clear")
|
||||
})
|
||||
}
|
||||
|
||||
func TestCachePersistence(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
metadata1 := &Metadata{
|
||||
ID: "persist-test-1",
|
||||
FilePath: "/tmp/persist1.yaml",
|
||||
Name: "Persistence Test 1",
|
||||
Authors: []string{"tester"},
|
||||
Tags: []string{"test"},
|
||||
Severity: "medium",
|
||||
ProtocolType: "dns",
|
||||
}
|
||||
|
||||
metadata2 := &Metadata{
|
||||
ID: "persist-test-2",
|
||||
FilePath: "/tmp/persist2.yaml",
|
||||
Name: "Persistence Test 2",
|
||||
Authors: []string{"tester2"},
|
||||
Tags: []string{"cve"},
|
||||
Severity: "critical",
|
||||
ProtocolType: "http",
|
||||
}
|
||||
|
||||
t.Run("Save and Load", func(t *testing.T) {
|
||||
// Create cache and add entries
|
||||
cache1, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
cache1.Set(metadata1.FilePath, metadata1)
|
||||
cache1.Set(metadata2.FilePath, metadata2)
|
||||
require.Equal(t, 2, cache1.Size())
|
||||
|
||||
// Save to disk
|
||||
err = cache1.Save()
|
||||
require.NoError(t, err, "Failed to save cache")
|
||||
|
||||
// Verify cache file exists
|
||||
cacheFile := filepath.Join(tmpDir, IndexFileName)
|
||||
stat, err := os.Stat(cacheFile)
|
||||
require.NoError(t, err, "Cache file should exist")
|
||||
require.Greater(t, stat.Size(), int64(0), "Cache file should not be empty")
|
||||
|
||||
// Create new cache and load
|
||||
cache2, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, cache2.Size(), "New cache should be empty before Load")
|
||||
|
||||
err = cache2.Load()
|
||||
require.NoError(t, err, "Failed to load cache")
|
||||
|
||||
// Verify data was loaded
|
||||
require.Equal(t, 2, cache2.Size(), "Loaded cache should have 2 entries")
|
||||
require.True(t, cache2.Has(metadata1.FilePath), "Loaded cache should contain first entry")
|
||||
require.True(t, cache2.Has(metadata2.FilePath), "Loaded cache should contain second entry")
|
||||
})
|
||||
|
||||
t.Run("Load non-existent cache", func(t *testing.T) {
|
||||
emptyDir := t.TempDir()
|
||||
cache, err := NewIndex(emptyDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Loading non-existent cache should not error
|
||||
err = cache.Load()
|
||||
require.NoError(t, err, "Loading non-existent cache should not error")
|
||||
require.Equal(t, 0, cache.Size(), "Cache should be empty after loading non-existent file")
|
||||
})
|
||||
|
||||
t.Run("Atomic save", func(t *testing.T) {
|
||||
cache, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
cache.Set(metadata1.FilePath, metadata1)
|
||||
err = cache.Save()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify no .tmp file left behind
|
||||
tmpFile := filepath.Join(tmpDir, IndexFileName+".tmp")
|
||||
_, err = os.Stat(tmpFile)
|
||||
require.True(t, os.IsNotExist(err), "Temporary file should not exist after save")
|
||||
|
||||
// Verify actual cache file exists
|
||||
cacheFile := filepath.Join(tmpDir, IndexFileName)
|
||||
_, err = os.Stat(cacheFile)
|
||||
require.NoError(t, err, "Cache file should exist")
|
||||
})
|
||||
}
|
||||
|
||||
func TestIndexVersionMismatch(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create cache with current version
|
||||
cache1, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
metadata := &Metadata{
|
||||
ID: "version-test",
|
||||
FilePath: "/tmp/version.yaml",
|
||||
}
|
||||
cache1.Set(metadata.FilePath, metadata)
|
||||
|
||||
// Save with current version
|
||||
err = cache1.Save()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Manually modify version and save again
|
||||
cache1.version = 999
|
||||
err = cache1.Save()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Try to load with different version
|
||||
cache2, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Load should succeed but cache should be empty (version mismatch)
|
||||
err = cache2.Load()
|
||||
require.NoError(t, err, "Load should not error on version mismatch")
|
||||
require.Equal(t, 0, cache2.Size(), "Cache should be empty after version mismatch")
|
||||
}
|
||||
|
||||
func TestCacheCorruptedFile(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
cacheFile := filepath.Join(tmpDir, IndexFileName)
|
||||
|
||||
// Create corrupted cache file
|
||||
err := os.WriteFile(cacheFile, []byte("corrupted data that is not valid gob"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Try to load corrupted cache
|
||||
cache, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = cache.Load()
|
||||
require.NoError(t, err, "Load should not error on corrupted cache")
|
||||
require.Equal(t, 0, cache.Size(), "Cache should be empty after loading corrupted file")
|
||||
|
||||
// Corrupted file should be removed
|
||||
_, err = os.Stat(cacheFile)
|
||||
require.True(t, os.IsNotExist(err), "Corrupted cache file should be removed")
|
||||
}
|
||||
|
||||
func TestMetadataValidation(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
tmpFile := filepath.Join(tmpDir, "test.yaml")
|
||||
|
||||
t.Run("Valid metadata", func(t *testing.T) {
|
||||
// Create a test file
|
||||
err := os.WriteFile(tmpFile, []byte("id: test\ninfo:\n name: Test"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
info, err := os.Stat(tmpFile)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create metadata with correct checksum
|
||||
metadata := &Metadata{
|
||||
ID: "test",
|
||||
FilePath: tmpFile,
|
||||
ModTime: info.ModTime(),
|
||||
}
|
||||
|
||||
// Should be valid
|
||||
require.True(t, metadata.IsValid(), "Metadata should be valid for unchanged file")
|
||||
})
|
||||
|
||||
t.Run("Invalid metadata after file modification", func(t *testing.T) {
|
||||
// Create the test file first to ensure it exists in this subtest
|
||||
err := os.WriteFile(tmpFile, []byte("id: test\ninfo:\n name: Test"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Set file ModTime to past to ensure modification is detectable
|
||||
oldTime := time.Now().Add(-2 * time.Second)
|
||||
err = os.Chtimes(tmpFile, oldTime, oldTime)
|
||||
require.NoError(t, err)
|
||||
|
||||
info, err := os.Stat(tmpFile)
|
||||
require.NoError(t, err)
|
||||
|
||||
metadata := &Metadata{
|
||||
ID: "test",
|
||||
FilePath: tmpFile,
|
||||
ModTime: info.ModTime(),
|
||||
}
|
||||
|
||||
// Modify file
|
||||
err = os.WriteFile(tmpFile, []byte("id: test\ninfo:\n name: Modified"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should now be invalid
|
||||
require.False(t, metadata.IsValid(), "Metadata should be invalid after file modification")
|
||||
})
|
||||
|
||||
t.Run("Invalid metadata for deleted file", func(t *testing.T) {
|
||||
// Create the test file first to ensure it exists in this subtest
|
||||
err := os.WriteFile(tmpFile, []byte("id: test\ninfo:\n name: Test"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
info, err := os.Stat(tmpFile)
|
||||
require.NoError(t, err)
|
||||
|
||||
metadata := &Metadata{
|
||||
ID: "test",
|
||||
FilePath: tmpFile,
|
||||
ModTime: info.ModTime(),
|
||||
}
|
||||
|
||||
// Delete file
|
||||
err = os.Remove(tmpFile)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should be invalid
|
||||
require.False(t, metadata.IsValid(), "Metadata should be invalid for deleted file")
|
||||
})
|
||||
}
|
||||
|
||||
func TestSetFromTemplate(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
tmpFile := filepath.Join(tmpDir, "extract.yaml")
|
||||
|
||||
// Create a test file
|
||||
err := os.WriteFile(tmpFile, []byte("id: extract-test"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
cache, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("Basic metadata extraction", func(t *testing.T) {
|
||||
template := &templates.Template{
|
||||
ID: "extract-test",
|
||||
Info: model.Info{
|
||||
Name: "Extract Test Template",
|
||||
Authors: stringslice.StringSlice{Value: "author1,author2"},
|
||||
Tags: stringslice.StringSlice{Value: "tag1,tag2"},
|
||||
Description: "Test description",
|
||||
SeverityHolder: severity.Holder{
|
||||
Severity: severity.High,
|
||||
},
|
||||
},
|
||||
SelfContained: true,
|
||||
Verified: true,
|
||||
TemplateVerifier: "test-verifier",
|
||||
}
|
||||
|
||||
metadata, ok := cache.SetFromTemplate(tmpFile, template)
|
||||
require.True(t, ok, "Failed to set metadata from template")
|
||||
require.NotNil(t, metadata, "Metadata should not be nil")
|
||||
|
||||
// Verify core fields
|
||||
require.Equal(t, "extract-test", metadata.ID)
|
||||
require.Equal(t, tmpFile, metadata.FilePath)
|
||||
|
||||
// Verify Info fields
|
||||
require.Equal(t, "Extract Test Template", metadata.Name)
|
||||
require.Equal(t, []string{"author1,author2"}, metadata.Authors)
|
||||
require.Equal(t, []string{"tag1,tag2"}, metadata.Tags)
|
||||
require.Equal(t, "high", metadata.Severity)
|
||||
|
||||
// Verify flags
|
||||
require.True(t, metadata.Verified)
|
||||
require.Equal(t, "test-verifier", metadata.TemplateVerifier)
|
||||
})
|
||||
|
||||
t.Run("HTTP protocol detection", func(t *testing.T) {
|
||||
// Create a separate test file for this test
|
||||
httpFile := filepath.Join(tmpDir, "http-test.yaml")
|
||||
err := os.WriteFile(httpFile, []byte("id: http-test"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
template := &templates.Template{
|
||||
ID: "http-test",
|
||||
Info: model.Info{
|
||||
Name: "HTTP Test",
|
||||
Authors: stringslice.StringSlice{Value: "tester"},
|
||||
SeverityHolder: severity.Holder{
|
||||
Severity: severity.Medium,
|
||||
},
|
||||
},
|
||||
RequestsHTTP: []*http.Request{{Method: http.HTTPMethodTypeHolder{MethodType: http.HTTPGet}}},
|
||||
}
|
||||
|
||||
metadata, ok := cache.SetFromTemplate(httpFile, template)
|
||||
require.True(t, ok)
|
||||
require.NotNil(t, metadata)
|
||||
require.Equal(t, "http", metadata.ProtocolType)
|
||||
})
|
||||
|
||||
t.Run("Extract with missing file", func(t *testing.T) {
|
||||
template := &templates.Template{
|
||||
ID: "missing-test",
|
||||
Info: model.Info{
|
||||
Name: "Missing File Test",
|
||||
Authors: stringslice.StringSlice{Value: "tester"},
|
||||
SeverityHolder: severity.Holder{
|
||||
Severity: severity.Low,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
metadata, ok := cache.SetFromTemplate("/nonexistent/file.yaml", template)
|
||||
require.False(t, ok, "Should return false for nonexistent file")
|
||||
require.NotNil(t, metadata, "Metadata should still be returned")
|
||||
})
|
||||
}
|
||||
|
||||
func TestMetadataMatchingHelpers(t *testing.T) {
|
||||
metadata := &Metadata{
|
||||
Tags: []string{"cve", "rce", "apache"},
|
||||
Authors: []string{"pdteam", "geeknik"},
|
||||
Severity: "critical",
|
||||
ProtocolType: "http",
|
||||
}
|
||||
|
||||
t.Run("HasTag", func(t *testing.T) {
|
||||
require.True(t, metadata.HasTag("cve"))
|
||||
require.True(t, metadata.HasTag("rce"))
|
||||
require.True(t, metadata.HasTag("apache"))
|
||||
require.False(t, metadata.HasTag("xxe"))
|
||||
require.False(t, metadata.HasTag(""))
|
||||
})
|
||||
|
||||
t.Run("HasAuthor", func(t *testing.T) {
|
||||
require.True(t, metadata.HasAuthor("pdteam"))
|
||||
require.True(t, metadata.HasAuthor("geeknik"))
|
||||
require.False(t, metadata.HasAuthor("unknown"))
|
||||
require.False(t, metadata.HasAuthor(""))
|
||||
})
|
||||
|
||||
t.Run("MatchesSeverity", func(t *testing.T) {
|
||||
require.True(t, metadata.MatchesSeverity(severity.Critical))
|
||||
require.False(t, metadata.MatchesSeverity(severity.High))
|
||||
require.False(t, metadata.MatchesSeverity(severity.Medium))
|
||||
require.False(t, metadata.MatchesSeverity(severity.Low))
|
||||
require.False(t, metadata.MatchesSeverity(severity.Info))
|
||||
})
|
||||
|
||||
t.Run("MatchesProtocol", func(t *testing.T) {
|
||||
require.True(t, metadata.MatchesProtocol(types.HTTPProtocol))
|
||||
require.False(t, metadata.MatchesProtocol(types.DNSProtocol))
|
||||
require.False(t, metadata.MatchesProtocol(types.FileProtocol))
|
||||
require.False(t, metadata.MatchesProtocol(types.NetworkProtocol))
|
||||
})
|
||||
|
||||
t.Run("Empty metadata", func(t *testing.T) {
|
||||
emptyMetadata := &Metadata{}
|
||||
require.False(t, emptyMetadata.HasTag("any"))
|
||||
require.False(t, emptyMetadata.HasAuthor("any"))
|
||||
})
|
||||
}
|
||||
|
||||
func TestCacheConcurrency(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
cache, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Test concurrent writes
|
||||
t.Run("Concurrent Set", func(t *testing.T) {
|
||||
done := make(chan bool)
|
||||
for i := 0; i < 10; i++ {
|
||||
go func(id int) {
|
||||
metadata := &Metadata{
|
||||
ID: string(rune('a' + id)),
|
||||
FilePath: filepath.Join("/tmp", string(rune('a'+id))+".yaml"),
|
||||
}
|
||||
cache.Set(metadata.FilePath, metadata)
|
||||
done <- true
|
||||
}(i)
|
||||
}
|
||||
|
||||
// Wait for all goroutines
|
||||
for i := 0; i < 10; i++ {
|
||||
<-done
|
||||
}
|
||||
|
||||
require.Equal(t, 10, cache.Size(), "All concurrent writes should succeed")
|
||||
})
|
||||
|
||||
// Test concurrent reads
|
||||
t.Run("Concurrent Has", func(t *testing.T) {
|
||||
metadata := &Metadata{
|
||||
ID: "concurrent-test",
|
||||
FilePath: "/tmp/concurrent.yaml",
|
||||
}
|
||||
cache.Set(metadata.FilePath, metadata)
|
||||
|
||||
done := make(chan bool)
|
||||
for i := 0; i < 20; i++ {
|
||||
go func() {
|
||||
_ = cache.Has(metadata.FilePath)
|
||||
done <- true
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait for all goroutines
|
||||
for i := 0; i < 20; i++ {
|
||||
<-done
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCacheSize(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
cache, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, 0, cache.Size(), "New cache should have size 0")
|
||||
|
||||
// Add entries
|
||||
for i := 0; i < 5; i++ {
|
||||
metadata := &Metadata{
|
||||
ID: string(rune('a' + i)),
|
||||
FilePath: filepath.Join("/tmp", string(rune('a'+i))+".yaml"),
|
||||
}
|
||||
cache.Set(metadata.FilePath, metadata)
|
||||
}
|
||||
|
||||
require.Equal(t, 5, cache.Size(), "Cache should have size 5 after adding 5 entries")
|
||||
|
||||
// Delete entries
|
||||
cache.Delete(filepath.Join("/tmp", "a.yaml"))
|
||||
cache.Delete(filepath.Join("/tmp", "b.yaml"))
|
||||
|
||||
require.Equal(t, 3, cache.Size(), "Cache should have size 3 after deleting 2 entries")
|
||||
|
||||
// Clear cache
|
||||
cache.Clear()
|
||||
require.Equal(t, 0, cache.Size(), "Cache should have size 0 after Clear")
|
||||
}
|
||||
|
||||
func TestCacheGetWithValidFile(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
cache, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create a real file for testing validation
|
||||
tmpFile := filepath.Join(tmpDir, "test.yaml")
|
||||
err = os.WriteFile(tmpFile, []byte("id: test"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
info, err := os.Stat(tmpFile)
|
||||
require.NoError(t, err)
|
||||
|
||||
metadata := &Metadata{
|
||||
ID: "test",
|
||||
FilePath: tmpFile,
|
||||
ModTime: info.ModTime(),
|
||||
Name: "Test Template",
|
||||
}
|
||||
|
||||
// Set and get should work with valid file
|
||||
cache.Set(metadata.FilePath, metadata)
|
||||
retrieved, found := cache.Get(metadata.FilePath)
|
||||
require.True(t, found, "Should find entry with valid file")
|
||||
require.NotNil(t, retrieved, "Retrieved metadata should not be nil")
|
||||
require.Equal(t, metadata.ID, retrieved.ID)
|
||||
}
|
||||
|
||||
func TestCacheSaveErrorHandling(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
cache, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
metadata := &Metadata{
|
||||
ID: "test",
|
||||
FilePath: filepath.Join("/tmp", "test.yaml"),
|
||||
}
|
||||
cache.Set(metadata.FilePath, metadata)
|
||||
|
||||
// Create a directory where the temp file would be created to force an error
|
||||
// The Save method creates a file at cacheFile + ".tmp"
|
||||
conflictPath := filepath.Join(tmpDir, IndexFileName+".tmp")
|
||||
err = os.Mkdir(conflictPath, 0755)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = cache.Save()
|
||||
require.Error(t, err, "Save should fail when temp file cannot be created")
|
||||
}
|
||||
|
||||
func TestNewCacheWithInvalidDirectory(t *testing.T) {
|
||||
// Try to create cache in a file path (should fail)
|
||||
tmpFile := filepath.Join(t.TempDir(), "file.txt")
|
||||
err := os.WriteFile(tmpFile, []byte("test"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
cache, err := NewIndex(tmpFile)
|
||||
require.Error(t, err, "NewCache should fail when path is a file")
|
||||
require.Nil(t, cache, "Cache should be nil on error")
|
||||
}
|
||||
|
||||
func TestCacheLoadCorruptedRemoval(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
cacheFile := filepath.Join(tmpDir, IndexFileName)
|
||||
|
||||
// Create corrupted cache file with invalid gob data
|
||||
err := os.WriteFile(cacheFile, []byte("this is not valid gob encoding at all!"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify file exists before Load
|
||||
_, err = os.Stat(cacheFile)
|
||||
require.NoError(t, err, "Corrupted file should exist")
|
||||
|
||||
// Load should not error but should remove corrupted file
|
||||
cache, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = cache.Load()
|
||||
require.NoError(t, err, "Load should not return error for corrupted file")
|
||||
|
||||
// Verify corrupted file was removed
|
||||
_, err = os.Stat(cacheFile)
|
||||
require.True(t, os.IsNotExist(err), "Corrupted file should be removed")
|
||||
require.Equal(t, 0, cache.Size(), "Cache should be empty after loading corrupted file")
|
||||
}
|
||||
|
||||
func TestMetadataExtractionWithNilClassification(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
tmpFile := filepath.Join(tmpDir, "test.yaml")
|
||||
err := os.WriteFile(tmpFile, []byte("id: test"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
template := &templates.Template{
|
||||
ID: "nil-classification",
|
||||
Info: model.Info{
|
||||
Name: "Template without classification",
|
||||
Authors: stringslice.StringSlice{Value: "tester"},
|
||||
SeverityHolder: severity.Holder{
|
||||
Severity: severity.Medium,
|
||||
},
|
||||
Classification: nil, // Explicitly nil
|
||||
},
|
||||
}
|
||||
|
||||
cache, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
metadata, ok := cache.SetFromTemplate(tmpFile, template)
|
||||
require.True(t, ok)
|
||||
require.NotNil(t, metadata)
|
||||
}
|
||||
|
||||
func TestCachePersistenceWithLargeDataset(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
cache, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add 100 entries to test bulk operations
|
||||
for i := 0; i < 100; i++ {
|
||||
metadata := &Metadata{
|
||||
ID: fmt.Sprintf("template-%d", i),
|
||||
FilePath: filepath.Join("/tmp", fmt.Sprintf("template-%d.yaml", i)),
|
||||
Name: fmt.Sprintf("Template %d", i),
|
||||
Authors: []string{fmt.Sprintf("author%d", i)},
|
||||
Tags: []string{"tag1", "tag2", "tag3"},
|
||||
Severity: "high",
|
||||
}
|
||||
cache.Set(metadata.FilePath, metadata)
|
||||
}
|
||||
|
||||
require.Equal(t, 100, cache.Size(), "Cache should contain 100 entries")
|
||||
|
||||
// Save to disk
|
||||
err = cache.Save()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Load into new cache
|
||||
cache2, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
err = cache2.Load()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, 100, cache2.Size(), "Loaded cache should contain 100 entries")
|
||||
|
||||
// Verify a sample entry
|
||||
found := cache2.Has(filepath.Join("/tmp", "template-50.yaml"))
|
||||
require.True(t, found, "Should find sample entry")
|
||||
}
|
||||
|
||||
func TestMetadataHelperMethods(t *testing.T) {
|
||||
metadata := &Metadata{
|
||||
ID: "helper-test",
|
||||
Tags: []string{},
|
||||
Authors: []string{},
|
||||
Severity: "",
|
||||
ProtocolType: "",
|
||||
}
|
||||
|
||||
t.Run("Empty tags", func(t *testing.T) {
|
||||
require.False(t, metadata.HasTag("anytag"))
|
||||
})
|
||||
|
||||
t.Run("Empty authors", func(t *testing.T) {
|
||||
require.False(t, metadata.HasAuthor("anyauthor"))
|
||||
})
|
||||
|
||||
t.Run("Empty severity", func(t *testing.T) {
|
||||
require.False(t, metadata.MatchesSeverity(severity.Critical))
|
||||
})
|
||||
|
||||
t.Run("Empty protocol", func(t *testing.T) {
|
||||
require.False(t, metadata.MatchesProtocol(types.HTTPProtocol))
|
||||
})
|
||||
}
|
||||
|
||||
func TestMultipleProtocolsDetection(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
tmpFile := filepath.Join(tmpDir, "multi.yaml")
|
||||
err := os.WriteFile(tmpFile, []byte("id: multi"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Template with multiple protocol types
|
||||
template := &templates.Template{
|
||||
ID: "multi-protocol",
|
||||
Info: model.Info{
|
||||
Name: "Multi Protocol Template",
|
||||
Authors: stringslice.StringSlice{Value: "tester"},
|
||||
SeverityHolder: severity.Holder{
|
||||
Severity: severity.High,
|
||||
},
|
||||
},
|
||||
RequestsHTTP: []*http.Request{{Method: http.HTTPMethodTypeHolder{MethodType: http.HTTPGet}}},
|
||||
RequestsHeadless: []*headless.Request{{}},
|
||||
RequestsCode: []*code.Request{{}},
|
||||
}
|
||||
|
||||
cache, err := NewIndex(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
metadata, ok := cache.SetFromTemplate(tmpFile, template)
|
||||
require.True(t, ok)
|
||||
require.NotNil(t, metadata)
|
||||
require.Equal(t, "http", metadata.ProtocolType, "Primary protocol should be http")
|
||||
}
|
||||
|
||||
func TestNewMetadataFromTemplate(t *testing.T) {
|
||||
tmpl := &templates.Template{
|
||||
ID: "test-template",
|
||||
Info: model.Info{
|
||||
Name: "Test Template",
|
||||
Authors: stringslice.StringSlice{Value: []string{"author"}},
|
||||
Tags: stringslice.StringSlice{Value: []string{"tag"}},
|
||||
SeverityHolder: severity.Holder{
|
||||
Severity: severity.Low,
|
||||
},
|
||||
},
|
||||
Verified: true,
|
||||
TemplateVerifier: "verifier",
|
||||
}
|
||||
|
||||
path := "/tmp/test.yaml"
|
||||
metadata := NewMetadataFromTemplate(path, tmpl)
|
||||
|
||||
require.Equal(t, tmpl.ID, metadata.ID)
|
||||
require.Equal(t, path, metadata.FilePath)
|
||||
require.Equal(t, tmpl.Info.Name, metadata.Name)
|
||||
require.Equal(t, tmpl.Info.Authors.ToSlice(), metadata.Authors)
|
||||
require.Equal(t, tmpl.Info.Tags.ToSlice(), metadata.Tags)
|
||||
require.Equal(t, tmpl.Info.SeverityHolder.Severity.String(), metadata.Severity)
|
||||
require.Equal(t, tmpl.Type().String(), metadata.ProtocolType)
|
||||
require.Equal(t, tmpl.Verified, metadata.Verified)
|
||||
require.Equal(t, tmpl.TemplateVerifier, metadata.TemplateVerifier)
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"os"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
)
|
||||
|
||||
// Metadata contains lightweight metadata extracted from a template.
|
||||
type Metadata struct {
|
||||
// ID is the unique identifier of the template.
|
||||
ID string `gob:"id"`
|
||||
|
||||
// FilePath is the path to the template file.
|
||||
FilePath string `gob:"file_path"`
|
||||
|
||||
// ModTime is the modification time of the template file.
|
||||
ModTime time.Time `gob:"mod_time"`
|
||||
|
||||
// Name is the name of the template.
|
||||
Name string `gob:"name"`
|
||||
|
||||
// Authors are the authors of the template.
|
||||
Authors []string `gob:"authors"`
|
||||
|
||||
// Tags are the tags associated with the template.
|
||||
Tags []string `gob:"tags"`
|
||||
|
||||
// Severity is the severity level of the template.
|
||||
Severity string `gob:"severity"`
|
||||
|
||||
// ProtocolType is the primary protocol type of the template.
|
||||
ProtocolType string `gob:"protocol_type"`
|
||||
|
||||
// Verified indicates whether the template is verified.
|
||||
Verified bool `gob:"verified"`
|
||||
|
||||
// TemplateVerifier is the verifier used for the template.
|
||||
TemplateVerifier string `gob:"verifier,omitempty"`
|
||||
|
||||
// NOTE(dwisiswant0): Consider adding more fields here in the future to
|
||||
// enhance filtering caps w/o loading full templates, such as:
|
||||
// `has_{code,headless,file}` to indicate presence of protocol-based
|
||||
// requests, and/or classification fields (CVE, CWE, CVSS, EPSS), if needed.
|
||||
//
|
||||
// For maintainers: when adding new fields, don't forget to update the
|
||||
// Weigher logic in [NewIndex] to account for the new fields in cache weight
|
||||
// calculation, because it affects cache eviction behavior. Also, consider
|
||||
// the impact on existing cached data and whether a [IndexVersion] bump is
|
||||
// needed.
|
||||
}
|
||||
|
||||
// NewMetadataFromTemplate creates a new metadata object from a template.
|
||||
func NewMetadataFromTemplate(path string, tpl *templates.Template) *Metadata {
|
||||
return &Metadata{
|
||||
ID: tpl.ID,
|
||||
FilePath: path,
|
||||
|
||||
Name: tpl.Info.Name,
|
||||
Authors: tpl.Info.Authors.ToSlice(),
|
||||
Tags: tpl.Info.Tags.ToSlice(),
|
||||
Severity: tpl.Info.SeverityHolder.Severity.String(),
|
||||
|
||||
ProtocolType: tpl.Type().String(),
|
||||
|
||||
Verified: tpl.Verified,
|
||||
TemplateVerifier: tpl.TemplateVerifier,
|
||||
}
|
||||
}
|
||||
|
||||
// IsValid checks if the cached metadata is still valid by comparing the file
|
||||
// modification time.
|
||||
func (m *Metadata) IsValid() bool {
|
||||
info, err := os.Stat(m.FilePath)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return m.ModTime.Equal(info.ModTime())
|
||||
}
|
||||
|
||||
// MatchesSeverity checks if the metadata matches the given severity.
|
||||
func (m *Metadata) MatchesSeverity(sev severity.Severity) bool {
|
||||
return m.Severity == sev.String()
|
||||
}
|
||||
|
||||
// MatchesProtocol checks if the metadata matches the given protocol type.
|
||||
func (m *Metadata) MatchesProtocol(protocolType types.ProtocolType) bool {
|
||||
return m.ProtocolType == protocolType.String()
|
||||
}
|
||||
|
||||
// HasTag checks if the metadata contains the given tag.
|
||||
func (m *Metadata) HasTag(tag string) bool {
|
||||
return slices.Contains(m.Tags, tag)
|
||||
}
|
||||
|
||||
// HasAuthor checks if the metadata contains the given author.
|
||||
func (m *Metadata) HasAuthor(author string) bool {
|
||||
return slices.Contains(m.Authors, author)
|
||||
}
|
||||
+296
-76
@@ -7,13 +7,14 @@ import (
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/logrusorgru/aurora"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/index"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/keys"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
@@ -25,6 +26,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/workflows"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
mapsutil "github.com/projectdiscovery/utils/maps"
|
||||
sliceutil "github.com/projectdiscovery/utils/slice"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
syncutil "github.com/projectdiscovery/utils/sync"
|
||||
@@ -75,7 +77,6 @@ type Config struct {
|
||||
type Store struct {
|
||||
id string // id of the store (optional)
|
||||
tagFilter *templates.TagFilter
|
||||
pathFilter *filter.PathFilter
|
||||
config *Config
|
||||
finalTemplates []string
|
||||
finalWorkflows []string
|
||||
@@ -87,6 +88,19 @@ type Store struct {
|
||||
|
||||
logger *gologger.Logger
|
||||
|
||||
// parserCacheOnce is used to cache the parser cache result
|
||||
parserCacheOnce func() *templates.Cache
|
||||
|
||||
// metadataIndex is the template metadata cache
|
||||
metadataIndex *index.Index
|
||||
|
||||
// indexFilter is the cached filter for metadata matching
|
||||
indexFilter *index.Filter
|
||||
|
||||
// saveTemplatesIndexOnce is used to ensure we only save the metadata index
|
||||
// once
|
||||
saveMetadataIndexOnce func()
|
||||
|
||||
// NotFoundCallback is called for each not found template
|
||||
// This overrides error handling for not found templates
|
||||
NotFoundCallback func(template string) bool
|
||||
@@ -124,17 +138,10 @@ func NewConfig(options *types.Options, catalog catalog.Catalog, executerOpts *pr
|
||||
|
||||
// New creates a new template store based on provided configuration
|
||||
func New(cfg *Config) (*Store, error) {
|
||||
// tagFilter only for IncludeConditions (advanced filtering).
|
||||
// All other filtering (tags, authors, severities, IDs, protocols, paths) is
|
||||
// handled by [index.Filter].
|
||||
tagFilter, err := templates.NewTagFilter(&templates.TagFilterConfig{
|
||||
Tags: cfg.Tags,
|
||||
ExcludeTags: cfg.ExcludeTags,
|
||||
Authors: cfg.Authors,
|
||||
Severities: cfg.Severities,
|
||||
ExcludeSeverities: cfg.ExcludeSeverities,
|
||||
IncludeTags: cfg.IncludeTags,
|
||||
IncludeIds: cfg.IncludeIds,
|
||||
ExcludeIds: cfg.ExcludeIds,
|
||||
Protocols: cfg.Protocols,
|
||||
ExcludeProtocols: cfg.ExcludeProtocols,
|
||||
IncludeConditions: cfg.IncludeConditions,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -142,20 +149,43 @@ func New(cfg *Config) (*Store, error) {
|
||||
}
|
||||
|
||||
store := &Store{
|
||||
id: cfg.StoreId,
|
||||
config: cfg,
|
||||
tagFilter: tagFilter,
|
||||
pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{
|
||||
IncludedTemplates: cfg.IncludeTemplates,
|
||||
ExcludedTemplates: cfg.ExcludeTemplates,
|
||||
}, cfg.Catalog),
|
||||
id: cfg.StoreId,
|
||||
config: cfg,
|
||||
tagFilter: tagFilter,
|
||||
finalTemplates: cfg.Templates,
|
||||
finalWorkflows: cfg.Workflows,
|
||||
logger: cfg.Logger,
|
||||
}
|
||||
|
||||
store.parserCacheOnce = sync.OnceValue(func() *templates.Cache {
|
||||
if cfg.ExecutorOptions == nil || cfg.ExecutorOptions.Parser == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if parser, ok := cfg.ExecutorOptions.Parser.(*templates.Parser); ok {
|
||||
return parser.Cache()
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
// Initialize metadata index and filter (load from disk & cache for reuse)
|
||||
store.metadataIndex = store.loadTemplatesIndex()
|
||||
store.indexFilter = store.buildIndexFilter()
|
||||
store.saveMetadataIndexOnce = sync.OnceFunc(func() {
|
||||
if store.metadataIndex == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err := store.metadataIndex.Save(); err != nil {
|
||||
store.logger.Warning().Msgf("Could not save metadata cache: %v", err)
|
||||
} else {
|
||||
store.logger.Verbose().Msgf("Saved %d templates to metadata cache", store.metadataIndex.Size())
|
||||
}
|
||||
})
|
||||
|
||||
// Do a check to see if we have URLs in templates flag, if so
|
||||
// we need to processs them separately and remove them from the initial list
|
||||
// we need to process them separately and remove them from the initial list
|
||||
var templatesFinal []string
|
||||
for _, template := range cfg.Templates {
|
||||
// TODO: Add and replace this with urlutil.IsURL() helper
|
||||
@@ -175,6 +205,7 @@ func New(cfg *Config) (*Store, error) {
|
||||
templatesFinal = append(templatesFinal, v) // something went wrong, treat it as a file
|
||||
}
|
||||
}
|
||||
|
||||
cfg.TemplateURLs = remoteTemplates
|
||||
store.finalTemplates = templatesFinal
|
||||
|
||||
@@ -184,6 +215,7 @@ func New(cfg *Config) (*Store, error) {
|
||||
if err != nil {
|
||||
return store, err
|
||||
}
|
||||
|
||||
store.finalTemplates = append(store.finalTemplates, remoteTemplates...)
|
||||
store.finalWorkflows = append(store.finalWorkflows, remoteWorkflows...)
|
||||
}
|
||||
@@ -219,14 +251,18 @@ func handleTemplatesEditorURLs(input string) string {
|
||||
if err != nil {
|
||||
return input
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(parsed.Hostname(), "cloud.projectdiscovery.io") {
|
||||
return input
|
||||
}
|
||||
|
||||
if strings.HasSuffix(parsed.Path, ".yaml") {
|
||||
return input
|
||||
}
|
||||
|
||||
parsed.Path = fmt.Sprintf("%s.yaml", parsed.Path)
|
||||
finalURL := parsed.String()
|
||||
|
||||
return finalURL
|
||||
}
|
||||
|
||||
@@ -236,17 +272,21 @@ func handleTemplatesEditorURLs(input string) string {
|
||||
func (store *Store) ReadTemplateFromURI(uri string, remote bool) ([]byte, error) {
|
||||
if stringsutil.HasPrefixAny(uri, httpPrefix, httpsPrefix) && remote {
|
||||
uri = handleTemplatesEditorURLs(uri)
|
||||
|
||||
remoteTemplates, _, err := getRemoteTemplatesAndWorkflows([]string{uri}, nil, store.config.RemoteTemplateDomainList)
|
||||
if err != nil || len(remoteTemplates) == 0 {
|
||||
return nil, errkit.Wrapf(err, "Could not load template %s: got %v", uri, remoteTemplates)
|
||||
}
|
||||
|
||||
resp, err := retryablehttp.Get(remoteTemplates[0])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
return io.ReadAll(resp.Body)
|
||||
} else {
|
||||
return os.ReadFile(uri)
|
||||
@@ -285,19 +325,106 @@ func init() {
|
||||
templateIDPathMap = make(map[string]string)
|
||||
}
|
||||
|
||||
// buildIndexFilter creates an [index.Filter] from the store configuration.
|
||||
// This filter handles all basic filtering (paths, tags, authors, severities,
|
||||
// IDs, protocols). Advanced IncludeConditions filtering is handled separately
|
||||
// by tagFilter.
|
||||
func (store *Store) buildIndexFilter() *index.Filter {
|
||||
includeTemplates, _ := store.config.Catalog.GetTemplatesPath(store.config.IncludeTemplates)
|
||||
excludeTemplates, _ := store.config.Catalog.GetTemplatesPath(store.config.ExcludeTemplates)
|
||||
|
||||
return &index.Filter{
|
||||
Authors: store.config.Authors,
|
||||
Tags: store.config.Tags,
|
||||
ExcludeTags: store.config.ExcludeTags,
|
||||
IncludeTags: store.config.IncludeTags,
|
||||
IDs: store.config.IncludeIds,
|
||||
ExcludeIDs: store.config.ExcludeIds,
|
||||
IncludeTemplates: includeTemplates,
|
||||
ExcludeTemplates: excludeTemplates,
|
||||
Severities: []severity.Severity(store.config.Severities),
|
||||
ExcludeSeverities: []severity.Severity(store.config.ExcludeSeverities),
|
||||
ProtocolTypes: []templateTypes.ProtocolType(store.config.Protocols),
|
||||
ExcludeProtocolTypes: []templateTypes.ProtocolType(store.config.ExcludeProtocols),
|
||||
}
|
||||
}
|
||||
|
||||
func (store *Store) loadTemplatesIndex() *index.Index {
|
||||
var metadataIdx *index.Index
|
||||
|
||||
idx, err := index.NewDefaultIndex()
|
||||
if err != nil {
|
||||
store.logger.Warning().Msgf("Could not create metadata cache: %v", err)
|
||||
} else {
|
||||
metadataIdx = idx
|
||||
if err := metadataIdx.Load(); err != nil {
|
||||
store.logger.Warning().Msgf("Could not load metadata cache: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return metadataIdx
|
||||
}
|
||||
|
||||
// LoadTemplatesOnlyMetadata loads only the metadata of the templates
|
||||
func (store *Store) LoadTemplatesOnlyMetadata() error {
|
||||
defer store.saveMetadataIndexOnce()
|
||||
|
||||
templatePaths, errs := store.config.Catalog.GetTemplatesPath(store.finalTemplates)
|
||||
store.logErroredTemplates(errs)
|
||||
|
||||
filteredTemplatePaths := store.pathFilter.Match(templatePaths)
|
||||
|
||||
indexFilter := store.indexFilter
|
||||
validPaths := make(map[string]struct{})
|
||||
for templatePath := range filteredTemplatePaths {
|
||||
|
||||
for _, templatePath := range templatePaths {
|
||||
if store.metadataIndex != nil {
|
||||
if metadata, found := store.metadataIndex.Get(templatePath); found {
|
||||
if !indexFilter.Matches(metadata) {
|
||||
continue
|
||||
}
|
||||
|
||||
if store.tagFilter != nil {
|
||||
loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog)
|
||||
if !loaded {
|
||||
if err != nil && strings.Contains(err.Error(), templates.ErrExcluded.Error()) {
|
||||
stats.Increment(templates.TemplatesExcludedStats)
|
||||
if config.DefaultConfig.LogAllEvents {
|
||||
store.logger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error())
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
validPaths[templatePath] = struct{}{}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog)
|
||||
if loaded || store.pathFilter.MatchIncluded(templatePath) {
|
||||
if loaded {
|
||||
templatesCache := store.parserCacheOnce()
|
||||
if templatesCache != nil {
|
||||
if template, _, _ := templatesCache.Has(templatePath); template != nil {
|
||||
var metadata *index.Metadata
|
||||
|
||||
if store.metadataIndex != nil {
|
||||
metadata, _ = store.metadataIndex.SetFromTemplate(templatePath, template)
|
||||
} else {
|
||||
metadata = index.NewMetadataFromTemplate(templatePath, template)
|
||||
}
|
||||
|
||||
if !indexFilter.Matches(metadata) {
|
||||
continue
|
||||
}
|
||||
|
||||
validPaths[templatePath] = struct{}{}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
validPaths[templatePath] = struct{}{}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), templates.ErrExcluded.Error()) {
|
||||
stats.Increment(templates.TemplatesExcludedStats)
|
||||
@@ -306,43 +433,47 @@ func (store *Store) LoadTemplatesOnlyMetadata() error {
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
store.logger.Warning().Msg(err.Error())
|
||||
}
|
||||
}
|
||||
parserItem, ok := store.config.ExecutorOptions.Parser.(*templates.Parser)
|
||||
if !ok {
|
||||
|
||||
templatesCache := store.parserCacheOnce()
|
||||
if templatesCache == nil {
|
||||
return errors.New("invalid parser")
|
||||
}
|
||||
templatesCache := parserItem.Cache()
|
||||
|
||||
loadedTemplateIDs := mapsutil.NewSyncLockMap[string, struct{}]()
|
||||
caps := templates.Capabilities{
|
||||
Headless: store.config.ExecutorOptions.Options.Headless,
|
||||
Code: store.config.ExecutorOptions.Options.EnableCodeTemplates,
|
||||
DAST: store.config.ExecutorOptions.Options.DAST,
|
||||
SelfContained: store.config.ExecutorOptions.Options.EnableSelfContainedTemplates,
|
||||
File: store.config.ExecutorOptions.Options.EnableFileTemplates,
|
||||
}
|
||||
isListOrDisplay := store.config.ExecutorOptions.Options.TemplateList ||
|
||||
store.config.ExecutorOptions.Options.TemplateDisplay
|
||||
|
||||
for templatePath := range validPaths {
|
||||
template, _, _ := templatesCache.Has(templatePath)
|
||||
|
||||
if len(template.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless {
|
||||
if template == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(template.RequestsCode) > 0 && !store.config.ExecutorOptions.Options.EnableCodeTemplates {
|
||||
if !isListOrDisplay && !template.IsEnabledFor(caps) {
|
||||
continue
|
||||
}
|
||||
|
||||
if template.IsFuzzing() && !store.config.ExecutorOptions.Options.DAST {
|
||||
if loadedTemplateIDs.Has(template.ID) {
|
||||
store.logger.Debug().Msgf("Skipping duplicate template ID '%s' from path '%s'", template.ID, templatePath)
|
||||
continue
|
||||
}
|
||||
|
||||
if template.SelfContained && !store.config.ExecutorOptions.Options.EnableSelfContainedTemplates {
|
||||
continue
|
||||
}
|
||||
|
||||
if template.HasFileProtocol() && !store.config.ExecutorOptions.Options.EnableFileTemplates {
|
||||
continue
|
||||
}
|
||||
|
||||
if template != nil {
|
||||
template.Path = templatePath
|
||||
store.templates = append(store.templates, template)
|
||||
}
|
||||
_ = loadedTemplateIDs.Set(template.ID, struct{}{})
|
||||
template.Path = templatePath
|
||||
store.templates = append(store.templates, template)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -351,15 +482,24 @@ func (store *Store) LoadTemplatesOnlyMetadata() error {
|
||||
func (store *Store) ValidateTemplates() error {
|
||||
templatePaths, errs := store.config.Catalog.GetTemplatesPath(store.finalTemplates)
|
||||
store.logErroredTemplates(errs)
|
||||
|
||||
workflowPaths, errs := store.config.Catalog.GetTemplatesPath(store.finalWorkflows)
|
||||
store.logErroredTemplates(errs)
|
||||
|
||||
filteredTemplatePaths := store.pathFilter.Match(templatePaths)
|
||||
filteredWorkflowPaths := store.pathFilter.Match(workflowPaths)
|
||||
templatePathsMap := make(map[string]struct{}, len(templatePaths))
|
||||
for _, path := range templatePaths {
|
||||
templatePathsMap[path] = struct{}{}
|
||||
}
|
||||
|
||||
if store.areTemplatesValid(filteredTemplatePaths) && store.areWorkflowsValid(filteredWorkflowPaths) {
|
||||
workflowPathsMap := make(map[string]struct{}, len(workflowPaths))
|
||||
for _, path := range workflowPaths {
|
||||
workflowPathsMap[path] = struct{}{}
|
||||
}
|
||||
|
||||
if store.areTemplatesValid(templatePathsMap) && store.areWorkflowsValid(workflowPathsMap) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return errors.New("errors occurred during template validation")
|
||||
}
|
||||
|
||||
@@ -377,6 +517,7 @@ func (store *Store) areTemplatesValid(filteredTemplatePaths map[string]struct{})
|
||||
|
||||
func (store *Store) areWorkflowOrTemplatesValid(filteredTemplatePaths map[string]struct{}, isWorkflow bool, load func(templatePath string, tagFilter *templates.TagFilter) (bool, error)) bool {
|
||||
areTemplatesValid := true
|
||||
parsedCache := store.parserCacheOnce()
|
||||
|
||||
for templatePath := range filteredTemplatePaths {
|
||||
if _, err := load(templatePath, store.tagFilter); err != nil {
|
||||
@@ -386,13 +527,26 @@ func (store *Store) areWorkflowOrTemplatesValid(filteredTemplatePaths map[string
|
||||
}
|
||||
}
|
||||
|
||||
template, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
|
||||
if err != nil {
|
||||
if isParsingError(store, "Error occurred parsing template %s: %s\n", templatePath, err) {
|
||||
areTemplatesValid = false
|
||||
continue
|
||||
var template *templates.Template
|
||||
var err error
|
||||
|
||||
if parsedCache != nil {
|
||||
if cachedTemplate, _, cacheErr := parsedCache.Has(templatePath); cacheErr == nil && cachedTemplate != nil {
|
||||
template = cachedTemplate
|
||||
}
|
||||
} else if template == nil {
|
||||
}
|
||||
|
||||
if template == nil {
|
||||
template, err = templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
|
||||
if err != nil {
|
||||
if isParsingError(store, "Error occurred parsing template %s: %s\n", templatePath, err) {
|
||||
areTemplatesValid = false
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if template == nil {
|
||||
// NOTE(dwisiswant0): possibly global matchers template.
|
||||
// This could definitely be handled better, for example by returning an
|
||||
// `ErrGlobalMatchersTemplate` during `templates.Parse` and checking it
|
||||
@@ -415,10 +569,12 @@ func (store *Store) areWorkflowOrTemplatesValid(filteredTemplatePaths map[string
|
||||
// areTemplatesValid = false
|
||||
store.logger.Warning().Msgf("Found duplicate template ID during validation '%s' => '%s': %s\n", templatePath, existingTemplatePath, template.ID)
|
||||
}
|
||||
if !isWorkflow && len(template.Workflows) > 0 {
|
||||
|
||||
if !isWorkflow && template.HasWorkflows() {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if isWorkflow {
|
||||
if !areWorkflowTemplatesValid(store, template.Workflows) {
|
||||
areTemplatesValid = false
|
||||
@@ -426,6 +582,7 @@ func (store *Store) areWorkflowOrTemplatesValid(filteredTemplatePaths map[string
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return areTemplatesValid
|
||||
}
|
||||
|
||||
@@ -434,6 +591,7 @@ func areWorkflowTemplatesValid(store *Store, workflows []*workflows.WorkflowTemp
|
||||
if !areWorkflowTemplatesValid(store, workflow.Subtemplates) {
|
||||
return false
|
||||
}
|
||||
|
||||
_, err := store.config.Catalog.GetTemplatePath(workflow.Template)
|
||||
if err != nil {
|
||||
if isParsingError(store, "Error occurred loading template %s: %s\n", workflow.Template, err) {
|
||||
@@ -441,6 +599,7 @@ func areWorkflowTemplatesValid(store *Store, workflows []*workflows.WorkflowTemp
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -448,10 +607,13 @@ func isParsingError(store *Store, message string, template string, err error) bo
|
||||
if errors.Is(err, templates.ErrExcluded) {
|
||||
return false
|
||||
}
|
||||
|
||||
if errors.Is(err, templates.ErrCreateTemplateExecutor) {
|
||||
return false
|
||||
}
|
||||
|
||||
store.logger.Error().Msgf(message, template, err)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -464,14 +626,14 @@ func (store *Store) LoadTemplates(templatesList []string) []*templates.Template
|
||||
func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
|
||||
includedWorkflows, errs := store.config.Catalog.GetTemplatesPath(workflowsList)
|
||||
store.logErroredTemplates(errs)
|
||||
workflowPathMap := store.pathFilter.Match(includedWorkflows)
|
||||
|
||||
loadedWorkflows := make([]*templates.Template, 0, len(workflowPathMap))
|
||||
for workflowPath := range workflowPathMap {
|
||||
loadedWorkflows := make([]*templates.Template, 0, len(includedWorkflows))
|
||||
for _, workflowPath := range includedWorkflows {
|
||||
loaded, err := store.config.ExecutorOptions.Parser.LoadWorkflow(workflowPath, store.config.Catalog)
|
||||
if err != nil {
|
||||
store.logger.Warning().Msgf("Could not load workflow %s: %s\n", workflowPath, err)
|
||||
}
|
||||
|
||||
if loaded {
|
||||
parsed, err := templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions)
|
||||
if err != nil {
|
||||
@@ -481,19 +643,31 @@ func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return loadedWorkflows
|
||||
}
|
||||
|
||||
// LoadTemplatesWithTags takes a list of templates and extra tags
|
||||
// returning templates that match.
|
||||
func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templates.Template {
|
||||
defer store.saveMetadataIndexOnce()
|
||||
|
||||
indexFilter := store.indexFilter
|
||||
|
||||
includedTemplates, errs := store.config.Catalog.GetTemplatesPath(templatesList)
|
||||
store.logErroredTemplates(errs)
|
||||
templatePathMap := store.pathFilter.Match(includedTemplates)
|
||||
|
||||
loadedTemplates := sliceutil.NewSyncSlice[*templates.Template]()
|
||||
loadedTemplateIDs := mapsutil.NewSyncLockMap[string, struct{}]()
|
||||
|
||||
loadTemplate := func(tmpl *templates.Template) {
|
||||
if loadedTemplateIDs.Has(tmpl.ID) {
|
||||
store.logger.Debug().Msgf("Skipping duplicate template ID '%s' from path '%s'", tmpl.ID, tmpl.Path)
|
||||
return
|
||||
}
|
||||
|
||||
_ = loadedTemplateIDs.Set(tmpl.ID, struct{}{})
|
||||
|
||||
loadedTemplates.Append(tmpl)
|
||||
// increment signed/unsigned counters
|
||||
if tmpl.Verified {
|
||||
@@ -507,28 +681,66 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
|
||||
}
|
||||
}
|
||||
|
||||
wgLoadTemplates, errWg := syncutil.New(syncutil.WithSize(50))
|
||||
typesOpts := store.config.ExecutorOptions.Options
|
||||
concurrency := typesOpts.TemplateLoadingConcurrency
|
||||
if concurrency <= 0 {
|
||||
concurrency = types.DefaultTemplateLoadingConcurrency
|
||||
}
|
||||
|
||||
wgLoadTemplates, errWg := syncutil.New(syncutil.WithSize(concurrency))
|
||||
if errWg != nil {
|
||||
panic("could not create wait group")
|
||||
}
|
||||
|
||||
if store.config.ExecutorOptions.Options.ExecutionId == "" {
|
||||
store.config.ExecutorOptions.Options.ExecutionId = xid.New().String()
|
||||
if typesOpts.ExecutionId == "" {
|
||||
typesOpts.ExecutionId = xid.New().String()
|
||||
}
|
||||
|
||||
dialers := protocolstate.GetDialersWithId(store.config.ExecutorOptions.Options.ExecutionId)
|
||||
dialers := protocolstate.GetDialersWithId(typesOpts.ExecutionId)
|
||||
if dialers == nil {
|
||||
panic("dialers with executionId " + store.config.ExecutorOptions.Options.ExecutionId + " not found")
|
||||
panic("dialers with executionId " + typesOpts.ExecutionId + " not found")
|
||||
}
|
||||
|
||||
for templatePath := range templatePathMap {
|
||||
for _, templatePath := range includedTemplates {
|
||||
wgLoadTemplates.Add()
|
||||
go func(templatePath string) {
|
||||
defer wgLoadTemplates.Done()
|
||||
|
||||
var (
|
||||
metadata *index.Metadata
|
||||
metadataCached bool
|
||||
)
|
||||
|
||||
if store.metadataIndex != nil {
|
||||
if cachedMetadata, found := store.metadataIndex.Get(templatePath); found {
|
||||
metadata = cachedMetadata
|
||||
if !indexFilter.Matches(metadata) {
|
||||
return
|
||||
}
|
||||
// NOTE(dwisiswant0): else, tagFilter probably exists (for
|
||||
// IncludeConditions), which still need to check via
|
||||
// LoadTemplate.
|
||||
|
||||
metadataCached = true
|
||||
}
|
||||
}
|
||||
|
||||
loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, tags, store.config.Catalog)
|
||||
if loaded || store.pathFilter.MatchIncluded(templatePath) {
|
||||
if loaded {
|
||||
parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
|
||||
|
||||
if parsed != nil && !metadataCached {
|
||||
if store.metadataIndex != nil {
|
||||
metadata, _ = store.metadataIndex.SetFromTemplate(templatePath, parsed)
|
||||
} else {
|
||||
metadata = index.NewMetadataFromTemplate(templatePath, parsed)
|
||||
}
|
||||
|
||||
if metadata != nil && !indexFilter.Matches(metadata) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// exclude templates not compatible with offline matching from total runtime warning stats
|
||||
if !errors.Is(err, templates.ErrIncompatibleWithOfflineMatching) {
|
||||
@@ -536,18 +748,18 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
|
||||
}
|
||||
store.logger.Warning().Msgf("Could not parse template %s: %s\n", templatePath, err)
|
||||
} else if parsed != nil {
|
||||
if !parsed.Verified && store.config.ExecutorOptions.Options.DisableUnsignedTemplates {
|
||||
if !parsed.Verified && typesOpts.DisableUnsignedTemplates {
|
||||
// skip unverified templates when prompted to
|
||||
stats.Increment(templates.SkippedUnsignedStats)
|
||||
return
|
||||
}
|
||||
|
||||
if parsed.SelfContained && !store.config.ExecutorOptions.Options.EnableSelfContainedTemplates {
|
||||
if parsed.SelfContained && !typesOpts.EnableSelfContainedTemplates {
|
||||
stats.Increment(templates.ExcludedSelfContainedStats)
|
||||
return
|
||||
}
|
||||
|
||||
if parsed.HasFileProtocol() && !store.config.ExecutorOptions.Options.EnableFileTemplates {
|
||||
if parsed.HasFileRequest() && !typesOpts.EnableFileTemplates {
|
||||
stats.Increment(templates.ExcludedFileStats)
|
||||
return
|
||||
}
|
||||
@@ -559,25 +771,32 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
|
||||
}
|
||||
// DAST only templates
|
||||
// Skip DAST filter when loading auth templates
|
||||
if store.ID() != AuthStoreId && store.config.ExecutorOptions.Options.DAST {
|
||||
if store.ID() != AuthStoreId && typesOpts.DAST {
|
||||
// check if the template is a DAST template
|
||||
// also allow global matchers template to be loaded
|
||||
if parsed.IsFuzzing() || parsed.Options.GlobalMatchers != nil && parsed.Options.GlobalMatchers.HasMatchers() {
|
||||
loadTemplate(parsed)
|
||||
if parsed.IsFuzzableRequest() || parsed.IsGlobalMatchersTemplate() {
|
||||
if parsed.HasHeadlessRequest() && !typesOpts.Headless {
|
||||
stats.Increment(templates.ExcludedHeadlessTmplStats)
|
||||
if config.DefaultConfig.LogAllEvents {
|
||||
store.logger.Print().Msgf("[%v] Headless flag is required for headless template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
|
||||
}
|
||||
} else {
|
||||
loadTemplate(parsed)
|
||||
}
|
||||
}
|
||||
} else if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless {
|
||||
} else if parsed.HasHeadlessRequest() && !typesOpts.Headless {
|
||||
// donot include headless template in final list if headless flag is not set
|
||||
stats.Increment(templates.ExcludedHeadlessTmplStats)
|
||||
if config.DefaultConfig.LogAllEvents {
|
||||
store.logger.Print().Msgf("[%v] Headless flag is required for headless template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
|
||||
}
|
||||
} else if len(parsed.RequestsCode) > 0 && !store.config.ExecutorOptions.Options.EnableCodeTemplates {
|
||||
} else if parsed.HasCodeRequest() && !typesOpts.EnableCodeTemplates {
|
||||
// donot include 'Code' protocol custom template in final list if code flag is not set
|
||||
stats.Increment(templates.ExcludedCodeTmplStats)
|
||||
if config.DefaultConfig.LogAllEvents {
|
||||
store.logger.Print().Msgf("[%v] Code flag is required for code protocol template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
|
||||
}
|
||||
} else if len(parsed.RequestsCode) > 0 && !parsed.Verified && len(parsed.Workflows) == 0 {
|
||||
} else if parsed.HasCodeRequest() && !parsed.Verified && !parsed.HasWorkflows() {
|
||||
// donot include unverified 'Code' protocol custom template in final list
|
||||
stats.Increment(templates.SkippedCodeTmplTamperedStats)
|
||||
// these will be skipped so increment skip counter
|
||||
@@ -585,7 +804,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
|
||||
if config.DefaultConfig.LogAllEvents {
|
||||
store.logger.Print().Msgf("[%v] Tampered/Unsigned template at %v.\n", aurora.Yellow("WRN").String(), templatePath)
|
||||
}
|
||||
} else if parsed.IsFuzzing() && !store.config.ExecutorOptions.Options.DAST {
|
||||
} else if parsed.IsFuzzableRequest() && !typesOpts.DAST {
|
||||
stats.Increment(templates.ExludedDastTmplStats)
|
||||
if config.DefaultConfig.LogAllEvents {
|
||||
store.logger.Print().Msgf("[%v] -dast flag is required for DAST template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
|
||||
@@ -623,10 +842,11 @@ func IsHTTPBasedProtocolUsed(store *Store) bool {
|
||||
templates := append(store.Templates(), store.Workflows()...)
|
||||
|
||||
for _, template := range templates {
|
||||
if len(template.RequestsHTTP) > 0 || len(template.RequestsHeadless) > 0 {
|
||||
if template.HasHTTPRequest() || template.HasHeadlessRequest() {
|
||||
return true
|
||||
}
|
||||
if len(template.Workflows) > 0 {
|
||||
|
||||
if template.HasWorkflows() {
|
||||
if workflowContainsProtocol(template.Workflows) {
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
package loader_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
|
||||
)
|
||||
|
||||
func BenchmarkStoreValidateTemplates(b *testing.B) {
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.Logger = &gologger.Logger{}
|
||||
testutils.Init(options)
|
||||
|
||||
catalog := disk.NewCatalog(config.DefaultConfig.TemplatesDirectory)
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, nil)
|
||||
executerOpts.Parser = templates.NewParser()
|
||||
|
||||
workflowLoader, err := workflow.NewLoader(executerOpts)
|
||||
if err != nil {
|
||||
b.Fatalf("could not create workflow loader: %s", err)
|
||||
}
|
||||
executerOpts.WorkflowLoader = workflowLoader
|
||||
|
||||
loaderCfg := loader.NewConfig(options, catalog, executerOpts)
|
||||
|
||||
store, err := loader.New(loaderCfg)
|
||||
if err != nil {
|
||||
b.Fatalf("could not create store: %s", err)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
|
||||
for b.Loop() {
|
||||
_ = store.ValidateTemplates()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkLoadTemplates(b *testing.B) {
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.Logger = &gologger.Logger{}
|
||||
options.ExecutionId = "bench-load-templates"
|
||||
testutils.Init(options)
|
||||
|
||||
catalog := disk.NewCatalog(config.DefaultConfig.TemplatesDirectory)
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, nil)
|
||||
executerOpts.Parser = templates.NewParser()
|
||||
|
||||
workflowLoader, err := workflow.NewLoader(executerOpts)
|
||||
if err != nil {
|
||||
b.Fatalf("could not create workflow loader: %s", err)
|
||||
}
|
||||
executerOpts.WorkflowLoader = workflowLoader
|
||||
|
||||
b.Run("NoFilter", func(b *testing.B) {
|
||||
loaderCfg := loader.NewConfig(options, catalog, executerOpts)
|
||||
store, err := loader.New(loaderCfg)
|
||||
if err != nil {
|
||||
b.Fatalf("could not create store: %s", err)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
|
||||
for b.Loop() {
|
||||
_ = store.LoadTemplates([]string{config.DefaultConfig.TemplatesDirectory})
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("FilterBySeverityCritical", func(b *testing.B) {
|
||||
opts := options.Copy()
|
||||
opts.Severities = severity.Severities{severity.Critical}
|
||||
loaderCfg := loader.NewConfig(opts, catalog, executerOpts)
|
||||
|
||||
store, err := loader.New(loaderCfg)
|
||||
if err != nil {
|
||||
b.Fatalf("could not create store: %s", err)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
|
||||
for b.Loop() {
|
||||
_ = store.LoadTemplates([]string{config.DefaultConfig.TemplatesDirectory})
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("FilterBySeverityHighCritical", func(b *testing.B) {
|
||||
opts := options.Copy()
|
||||
opts.Severities = severity.Severities{severity.High, severity.Critical}
|
||||
loaderCfg := loader.NewConfig(opts, catalog, executerOpts)
|
||||
|
||||
store, err := loader.New(loaderCfg)
|
||||
if err != nil {
|
||||
b.Fatalf("could not create store: %s", err)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
|
||||
for b.Loop() {
|
||||
_ = store.LoadTemplates([]string{config.DefaultConfig.TemplatesDirectory})
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("FilterByAuthor", func(b *testing.B) {
|
||||
opts := options.Copy()
|
||||
opts.Authors = []string{"pdteam"}
|
||||
loaderCfg := loader.NewConfig(opts, catalog, executerOpts)
|
||||
|
||||
store, err := loader.New(loaderCfg)
|
||||
if err != nil {
|
||||
b.Fatalf("could not create store: %s", err)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
|
||||
for b.Loop() {
|
||||
_ = store.LoadTemplates([]string{config.DefaultConfig.TemplatesDirectory})
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("FilterByTags", func(b *testing.B) {
|
||||
opts := options.Copy()
|
||||
opts.Tags = []string{"cve", "rce"}
|
||||
loaderCfg := loader.NewConfig(opts, catalog, executerOpts)
|
||||
|
||||
store, err := loader.New(loaderCfg)
|
||||
if err != nil {
|
||||
b.Fatalf("could not create store: %s", err)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
|
||||
for b.Loop() {
|
||||
_ = store.LoadTemplates([]string{config.DefaultConfig.TemplatesDirectory})
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("FilterByProtocol", func(b *testing.B) {
|
||||
opts := options.Copy()
|
||||
opts.Protocols = templateTypes.ProtocolTypes{templateTypes.HTTPProtocol}
|
||||
loaderCfg := loader.NewConfig(opts, catalog, executerOpts)
|
||||
|
||||
store, err := loader.New(loaderCfg)
|
||||
if err != nil {
|
||||
b.Fatalf("could not create store: %s", err)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
|
||||
for b.Loop() {
|
||||
_ = store.LoadTemplates([]string{config.DefaultConfig.TemplatesDirectory})
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("ComplexFilter", func(b *testing.B) {
|
||||
opts := options.Copy()
|
||||
opts.Severities = severity.Severities{severity.High, severity.Critical}
|
||||
opts.Authors = []string{"pdteam"}
|
||||
opts.Tags = []string{"cve"}
|
||||
loaderCfg := loader.NewConfig(opts, catalog, executerOpts)
|
||||
|
||||
store, err := loader.New(loaderCfg)
|
||||
if err != nil {
|
||||
b.Fatalf("could not create store: %s", err)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
|
||||
for b.Loop() {
|
||||
_ = store.LoadTemplates([]string{config.DefaultConfig.TemplatesDirectory})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkLoadTemplatesOnlyMetadata(b *testing.B) {
|
||||
options := testutils.DefaultOptions.Copy()
|
||||
options.Logger = &gologger.Logger{}
|
||||
options.ExecutionId = "bench-metadata"
|
||||
testutils.Init(options)
|
||||
|
||||
catalog := disk.NewCatalog(config.DefaultConfig.TemplatesDirectory)
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, nil)
|
||||
executerOpts.Parser = templates.NewParser()
|
||||
|
||||
workflowLoader, err := workflow.NewLoader(executerOpts)
|
||||
if err != nil {
|
||||
b.Fatalf("could not create workflow loader: %s", err)
|
||||
}
|
||||
executerOpts.WorkflowLoader = workflowLoader
|
||||
|
||||
b.Run("WithoutFilter", func(b *testing.B) {
|
||||
loaderCfg := loader.NewConfig(options, catalog, executerOpts)
|
||||
store, err := loader.New(loaderCfg)
|
||||
if err != nil {
|
||||
b.Fatalf("could not create store: %s", err)
|
||||
}
|
||||
|
||||
// Pre-warm the cache
|
||||
_ = store.LoadTemplatesOnlyMetadata()
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
|
||||
for b.Loop() {
|
||||
_ = store.LoadTemplatesOnlyMetadata()
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("WithSeverityFilter", func(b *testing.B) {
|
||||
opts := options.Copy()
|
||||
opts.Severities = severity.Severities{severity.Critical}
|
||||
loaderCfg := loader.NewConfig(opts, catalog, executerOpts)
|
||||
|
||||
store, err := loader.New(loaderCfg)
|
||||
if err != nil {
|
||||
b.Fatalf("could not create store: %s", err)
|
||||
}
|
||||
|
||||
// Pre-warm the cache
|
||||
_ = store.LoadTemplatesOnlyMetadata()
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
|
||||
for b.Loop() {
|
||||
_ = store.LoadTemplatesOnlyMetadata()
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -97,7 +97,7 @@ func getRemoteContent(URL string, remoteTemplateDomainList []string, contentType
|
||||
_ = response.Body.Close()
|
||||
}()
|
||||
if response.StatusCode < 200 || response.StatusCode > 299 {
|
||||
return RemoteContent{Error: fmt.Errorf("get \"%s\": unexpect status %d", URL, response.StatusCode)}
|
||||
return RemoteContent{Error: fmt.Errorf("get \"%s\": unexpected status %d", URL, response.StatusCode)}
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(response.Body)
|
||||
|
||||
@@ -109,7 +109,6 @@ func (e *Engine) executeTemplateSpray(ctx context.Context, templatesList []*temp
|
||||
defer wp.Wait()
|
||||
|
||||
for _, template := range templatesList {
|
||||
template := template
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
|
||||
+73
-56
@@ -48,8 +48,15 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
|
||||
|
||||
// executeTemplateWithTargets executes a given template on x targets (with a internal targetpool(i.e concurrency))
|
||||
func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templates.Template, target provider.InputProvider, results *atomic.Bool) {
|
||||
// this is target pool i.e max target to execute
|
||||
wg := e.workPool.InputPool(template.Type())
|
||||
if e.workPool == nil {
|
||||
e.workPool = e.GetWorkPool()
|
||||
}
|
||||
// Bounded worker pool using input concurrency
|
||||
pool := e.workPool.InputPool(template.Type())
|
||||
workerCount := 1
|
||||
if pool != nil && pool.Size > 0 {
|
||||
workerCount = pool.Size
|
||||
}
|
||||
|
||||
var (
|
||||
index uint32
|
||||
@@ -78,6 +85,41 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ
|
||||
currentInfo.Unlock()
|
||||
}
|
||||
|
||||
// task represents a single target execution unit
|
||||
type task struct {
|
||||
index uint32
|
||||
skip bool
|
||||
value *contextargs.MetaInput
|
||||
}
|
||||
|
||||
tasks := make(chan task)
|
||||
var workersWg sync.WaitGroup
|
||||
workersWg.Add(workerCount)
|
||||
for i := 0; i < workerCount; i++ {
|
||||
go func() {
|
||||
defer workersWg.Done()
|
||||
for t := range tasks {
|
||||
func() {
|
||||
defer cleanupInFlight(t.index)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
if t.skip {
|
||||
return
|
||||
}
|
||||
|
||||
match, err := e.executeTemplateOnInput(ctx, template, t.value)
|
||||
if err != nil {
|
||||
e.options.Logger.Warning().Msgf("[%s] Could not execute step on %s: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), t.value.Input, err)
|
||||
}
|
||||
results.CompareAndSwap(false, match)
|
||||
}()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
target.Iterate(func(scannedValue *contextargs.MetaInput) bool {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -128,43 +170,13 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ
|
||||
return true
|
||||
}
|
||||
|
||||
wg.Add()
|
||||
go func(index uint32, skip bool, value *contextargs.MetaInput) {
|
||||
defer wg.Done()
|
||||
defer cleanupInFlight(index)
|
||||
if skip {
|
||||
return
|
||||
}
|
||||
|
||||
var match bool
|
||||
var err error
|
||||
ctxArgs := contextargs.New(ctx)
|
||||
ctxArgs.MetaInput = value
|
||||
ctx := scan.NewScanContext(ctx, ctxArgs)
|
||||
switch template.Type() {
|
||||
case types.WorkflowProtocol:
|
||||
match = e.executeWorkflow(ctx, template.CompiledWorkflow)
|
||||
default:
|
||||
if e.Callback != nil {
|
||||
if results, err := template.Executer.ExecuteWithResults(ctx); err == nil {
|
||||
for _, result := range results {
|
||||
e.Callback(result)
|
||||
}
|
||||
}
|
||||
match = true
|
||||
} else {
|
||||
match, err = template.Executer.Execute(ctx)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
e.options.Logger.Warning().Msgf("[%s] Could not execute step on %s: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), value.Input, err)
|
||||
}
|
||||
results.CompareAndSwap(false, match)
|
||||
}(index, skip, scannedValue)
|
||||
tasks <- task{index: index, skip: skip, value: scannedValue}
|
||||
index++
|
||||
return true
|
||||
})
|
||||
wg.Wait()
|
||||
|
||||
close(tasks)
|
||||
workersWg.Wait()
|
||||
|
||||
// on completion marks the template as completed
|
||||
currentInfo.Lock()
|
||||
@@ -202,26 +214,7 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t
|
||||
go func(template *templates.Template, value *contextargs.MetaInput, wg *syncutil.AdaptiveWaitGroup) {
|
||||
defer wg.Done()
|
||||
|
||||
var match bool
|
||||
var err error
|
||||
ctxArgs := contextargs.New(ctx)
|
||||
ctxArgs.MetaInput = value
|
||||
ctx := scan.NewScanContext(ctx, ctxArgs)
|
||||
switch template.Type() {
|
||||
case types.WorkflowProtocol:
|
||||
match = e.executeWorkflow(ctx, template.CompiledWorkflow)
|
||||
default:
|
||||
if e.Callback != nil {
|
||||
if results, err := template.Executer.ExecuteWithResults(ctx); err == nil {
|
||||
for _, result := range results {
|
||||
e.Callback(result)
|
||||
}
|
||||
}
|
||||
match = true
|
||||
} else {
|
||||
match, err = template.Executer.Execute(ctx)
|
||||
}
|
||||
}
|
||||
match, err := e.executeTemplateOnInput(ctx, template, value)
|
||||
if err != nil {
|
||||
e.options.Logger.Warning().Msgf("[%s] Could not execute step on %s: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), value.Input, err)
|
||||
}
|
||||
@@ -229,3 +222,27 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t
|
||||
}(tpl, target, sg)
|
||||
}
|
||||
}
|
||||
|
||||
// executeTemplateOnInput performs template execution for a single input and returns match status and error
|
||||
func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates.Template, value *contextargs.MetaInput) (bool, error) {
|
||||
ctxArgs := contextargs.New(ctx)
|
||||
ctxArgs.MetaInput = value
|
||||
scanCtx := scan.NewScanContext(ctx, ctxArgs)
|
||||
|
||||
switch template.Type() {
|
||||
case types.WorkflowProtocol:
|
||||
return e.executeWorkflow(scanCtx, template.CompiledWorkflow), nil
|
||||
default:
|
||||
if e.Callback != nil {
|
||||
results, err := template.Executer.ExecuteWithResults(scanCtx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, result := range results {
|
||||
e.Callback(result)
|
||||
}
|
||||
return len(results) > 0, nil
|
||||
}
|
||||
return template.Executer.Execute(scanCtx)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
inputtypes "github.com/projectdiscovery/nuclei/v3/pkg/input/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
tmpltypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
)
|
||||
|
||||
// fakeExecuter is a simple stub for protocols.Executer used to test executeTemplateOnInput
|
||||
type fakeExecuter struct {
|
||||
withResults bool
|
||||
}
|
||||
|
||||
func (f *fakeExecuter) Compile() error { return nil }
|
||||
func (f *fakeExecuter) Requests() int { return 1 }
|
||||
func (f *fakeExecuter) Execute(ctx *scan.ScanContext) (bool, error) { return !f.withResults, nil }
|
||||
func (f *fakeExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) {
|
||||
if !f.withResults {
|
||||
return nil, nil
|
||||
}
|
||||
return []*output.ResultEvent{{Host: "h"}}, nil
|
||||
}
|
||||
|
||||
// newTestEngine creates a minimal Engine for tests
|
||||
func newTestEngine() *Engine {
|
||||
return New(&types.Options{})
|
||||
}
|
||||
|
||||
func Test_executeTemplateOnInput_CallbackPath(t *testing.T) {
|
||||
e := newTestEngine()
|
||||
called := 0
|
||||
e.Callback = func(*output.ResultEvent) { called++ }
|
||||
|
||||
tpl := &templates.Template{}
|
||||
tpl.Executer = &fakeExecuter{withResults: true}
|
||||
|
||||
ok, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: "x"})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatalf("expected match true")
|
||||
}
|
||||
if called == 0 {
|
||||
t.Fatalf("expected callback to be called")
|
||||
}
|
||||
}
|
||||
|
||||
func Test_executeTemplateOnInput_ExecutePath(t *testing.T) {
|
||||
e := newTestEngine()
|
||||
tpl := &templates.Template{}
|
||||
tpl.Executer = &fakeExecuter{withResults: false}
|
||||
|
||||
ok, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: "x"})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatalf("expected match true from Execute path")
|
||||
}
|
||||
}
|
||||
|
||||
type fakeExecuterErr struct{}
|
||||
|
||||
func (f *fakeExecuterErr) Compile() error { return nil }
|
||||
func (f *fakeExecuterErr) Requests() int { return 1 }
|
||||
func (f *fakeExecuterErr) Execute(ctx *scan.ScanContext) (bool, error) { return false, nil }
|
||||
func (f *fakeExecuterErr) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) {
|
||||
return nil, fmt.Errorf("boom")
|
||||
}
|
||||
|
||||
func Test_executeTemplateOnInput_CallbackErrorPropagates(t *testing.T) {
|
||||
e := newTestEngine()
|
||||
e.Callback = func(*output.ResultEvent) {}
|
||||
tpl := &templates.Template{}
|
||||
tpl.Executer = &fakeExecuterErr{}
|
||||
|
||||
ok, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: "x"})
|
||||
if err == nil {
|
||||
t.Fatalf("expected error to propagate")
|
||||
}
|
||||
if ok {
|
||||
t.Fatalf("expected match to be false on error")
|
||||
}
|
||||
}
|
||||
|
||||
type fakeTargetProvider struct {
|
||||
values []*contextargs.MetaInput
|
||||
}
|
||||
|
||||
func (f *fakeTargetProvider) Count() int64 { return int64(len(f.values)) }
|
||||
func (f *fakeTargetProvider) Iterate(cb func(value *contextargs.MetaInput) bool) {
|
||||
for _, v := range f.values {
|
||||
if !cb(v) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
func (f *fakeTargetProvider) Set(string, string) {}
|
||||
func (f *fakeTargetProvider) SetWithProbe(string, string, inputtypes.InputLivenessProbe) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakeTargetProvider) SetWithExclusions(string, string) error { return nil }
|
||||
func (f *fakeTargetProvider) InputType() string { return "test" }
|
||||
func (f *fakeTargetProvider) Close() {}
|
||||
|
||||
type slowExecuter struct{}
|
||||
|
||||
func (s *slowExecuter) Compile() error { return nil }
|
||||
func (s *slowExecuter) Requests() int { return 1 }
|
||||
func (s *slowExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
|
||||
select {
|
||||
case <-ctx.Context().Done():
|
||||
return false, ctx.Context().Err()
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
func (s *slowExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func Test_executeTemplateWithTargets_RespectsCancellation(t *testing.T) {
|
||||
e := newTestEngine()
|
||||
e.SetExecuterOptions(&protocols.ExecutorOptions{Logger: e.Logger, ResumeCfg: types.NewResumeCfg(), ProtocolType: tmpltypes.HTTPProtocol})
|
||||
|
||||
tpl := &templates.Template{}
|
||||
tpl.Executer = &slowExecuter{}
|
||||
|
||||
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "a"}, {Input: "b"}, {Input: "c"}}}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
var matched atomic.Bool
|
||||
e.executeTemplateWithTargets(ctx, tpl, targets, &matched)
|
||||
}
|
||||
@@ -34,17 +34,14 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
|
||||
swg, _ := syncutil.New(syncutil.WithSize(templateThreads))
|
||||
|
||||
for _, template := range w.Workflows {
|
||||
swg.Add()
|
||||
|
||||
func(template *workflows.WorkflowTemplate) {
|
||||
defer swg.Done()
|
||||
|
||||
if err := e.runWorkflowStep(template, ctx, results, swg, w); err != nil {
|
||||
gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
|
||||
}
|
||||
}(template)
|
||||
newCtx := scan.NewScanContext(ctx.Context(), ctx.Input.Clone())
|
||||
if err := e.runWorkflowStep(template, newCtx, results, swg, w); err != nil {
|
||||
gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
|
||||
}
|
||||
}
|
||||
|
||||
swg.Wait()
|
||||
|
||||
return results.Load()
|
||||
}
|
||||
|
||||
|
||||
+33
-6
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
folderutil "github.com/projectdiscovery/utils/folder"
|
||||
"golang.org/x/oauth2"
|
||||
@@ -46,19 +47,45 @@ func (customTemplate *customTemplateGitHubRepo) Update(ctx context.Context) {
|
||||
downloadPath := config.DefaultConfig.CustomGitHubTemplatesDirectory
|
||||
clonePath := customTemplate.getLocalRepoClonePath(downloadPath)
|
||||
|
||||
// If folder does not exits then clone/download the repo
|
||||
// If folder does not exist then clone/download the repo
|
||||
if !fileutil.FolderExists(clonePath) {
|
||||
customTemplate.Download(ctx)
|
||||
return
|
||||
}
|
||||
|
||||
// Attempt to pull changes and handle the result
|
||||
customTemplate.handlePullChanges(clonePath)
|
||||
}
|
||||
|
||||
// handlePullChanges attempts to pull changes and logs the appropriate message
|
||||
func (customTemplate *customTemplateGitHubRepo) handlePullChanges(clonePath string) {
|
||||
err := customTemplate.pullChanges(clonePath, customTemplate.githubToken)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("%s", err)
|
||||
} else {
|
||||
gologger.Info().Msgf("Repo %s/%s successfully pulled the changes.\n", customTemplate.owner, customTemplate.reponame)
|
||||
|
||||
switch {
|
||||
case err == nil:
|
||||
customTemplate.logPullSuccess()
|
||||
case errors.Is(err, git.NoErrAlreadyUpToDate):
|
||||
customTemplate.logAlreadyUpToDate(err)
|
||||
default:
|
||||
customTemplate.logPullError(err)
|
||||
}
|
||||
}
|
||||
|
||||
// logPullSuccess logs a success message when changes are pulled
|
||||
func (customTemplate *customTemplateGitHubRepo) logPullSuccess() {
|
||||
gologger.Info().Msgf("Repo %s/%s successfully pulled the changes.\n", customTemplate.owner, customTemplate.reponame)
|
||||
}
|
||||
|
||||
// logAlreadyUpToDate logs an info message when repo is already up to date
|
||||
func (customTemplate *customTemplateGitHubRepo) logAlreadyUpToDate(err error) {
|
||||
gologger.Info().Msgf("%s", err)
|
||||
}
|
||||
|
||||
// logPullError logs an error message when pull fails
|
||||
func (customTemplate *customTemplateGitHubRepo) logPullError(err error) {
|
||||
gologger.Error().Msgf("%s", err)
|
||||
}
|
||||
|
||||
// NewGitHubProviders returns new instance of GitHub providers for downloading custom templates
|
||||
func NewGitHubProviders(options *types.Options) ([]*customTemplateGitHubRepo, error) {
|
||||
providers := []*customTemplateGitHubRepo{}
|
||||
@@ -187,7 +214,7 @@ func (ctr *customTemplateGitHubRepo) pullChanges(repoPath, githubToken string) e
|
||||
|
||||
err = w.Pull(pullOpts)
|
||||
if err != nil {
|
||||
return errors.Errorf("%s/%s: %s", ctr.owner, ctr.reponame, err.Error())
|
||||
return errkit.Wrapf(err, "%s/%s", ctr.owner, ctr.reponame)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -61,7 +61,7 @@ func checkTimingDependency(
|
||||
|
||||
var requestsSent []requestsSentMetadata
|
||||
for requestsLeft > 0 {
|
||||
isCorrelationPossible, delayRecieved, err := sendRequestAndTestConfidence(regression, highSleepTimeSeconds, requestSender, baselineDelay)
|
||||
isCorrelationPossible, delayReceived, err := sendRequestAndTestConfidence(regression, highSleepTimeSeconds, requestSender, baselineDelay)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
@@ -69,29 +69,29 @@ func checkTimingDependency(
|
||||
return false, "", nil
|
||||
}
|
||||
// Check the delay is greater than baseline by seconds requested
|
||||
if delayRecieved < baselineDelay+float64(highSleepTimeSeconds)*0.8 {
|
||||
if delayReceived < baselineDelay+float64(highSleepTimeSeconds)*0.8 {
|
||||
return false, "", nil
|
||||
}
|
||||
requestsSent = append(requestsSent, requestsSentMetadata{
|
||||
delay: highSleepTimeSeconds,
|
||||
delayReceived: delayRecieved,
|
||||
delayReceived: delayReceived,
|
||||
})
|
||||
|
||||
isCorrelationPossibleSecond, delayRecievedSecond, err := sendRequestAndTestConfidence(regression, int(DefaultLowSleepTimeSeconds), requestSender, baselineDelay)
|
||||
isCorrelationPossibleSecond, delayReceivedSecond, err := sendRequestAndTestConfidence(regression, int(DefaultLowSleepTimeSeconds), requestSender, baselineDelay)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if !isCorrelationPossibleSecond {
|
||||
return false, "", nil
|
||||
}
|
||||
if delayRecievedSecond < baselineDelay+float64(DefaultLowSleepTimeSeconds)*0.8 {
|
||||
if delayReceivedSecond < baselineDelay+float64(DefaultLowSleepTimeSeconds)*0.8 {
|
||||
return false, "", nil
|
||||
}
|
||||
requestsLeft = requestsLeft - 2
|
||||
|
||||
requestsSent = append(requestsSent, requestsSentMetadata{
|
||||
delay: int(DefaultLowSleepTimeSeconds),
|
||||
delayReceived: delayRecievedSecond,
|
||||
delayReceived: delayReceivedSecond,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/dataformat"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
readerutil "github.com/projectdiscovery/utils/reader"
|
||||
)
|
||||
|
||||
// Body is a component for a request body
|
||||
@@ -132,12 +131,10 @@ func (b *Body) Rebuild() (*retryablehttp.Request, error) {
|
||||
return nil, errors.Wrap(err, "could not encode body")
|
||||
}
|
||||
cloned := b.req.Clone(context.Background())
|
||||
reusableReader, err := readerutil.NewReusableReadCloser(encoded)
|
||||
err = cloned.SetBodyString(encoded)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not create reusable reader")
|
||||
return nil, errors.Wrap(err, "could not set body")
|
||||
}
|
||||
cloned.Body = reusableReader
|
||||
cloned.ContentLength = int64(len(encoded))
|
||||
cloned.Header.Set("Content-Length", strconv.Itoa(len(encoded)))
|
||||
return cloned, nil
|
||||
}
|
||||
|
||||
@@ -89,15 +89,15 @@ func (q *Path) Delete(key string) error {
|
||||
func (q *Path) Rebuild() (*retryablehttp.Request, error) {
|
||||
// Get the original path segments
|
||||
originalSplitted := strings.Split(q.req.Path, "/")
|
||||
|
||||
|
||||
// Create a new slice to hold the rebuilt segments
|
||||
rebuiltSegments := make([]string, 0, len(originalSplitted))
|
||||
|
||||
|
||||
// Add the first empty segment (from leading "/")
|
||||
if len(originalSplitted) > 0 && originalSplitted[0] == "" {
|
||||
rebuiltSegments = append(rebuiltSegments, "")
|
||||
}
|
||||
|
||||
|
||||
// Process each segment
|
||||
segmentIndex := 1 // 1-based indexing for our stored values
|
||||
for i := 1; i < len(originalSplitted); i++ {
|
||||
@@ -106,7 +106,7 @@ func (q *Path) Rebuild() (*retryablehttp.Request, error) {
|
||||
// Skip empty segments
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
// Check if we have a replacement for this segment
|
||||
key := strconv.Itoa(segmentIndex)
|
||||
if newValue, exists := q.value.parsed.Map.GetOrDefault(key, "").(string); exists && newValue != "" {
|
||||
@@ -116,10 +116,10 @@ func (q *Path) Rebuild() (*retryablehttp.Request, error) {
|
||||
}
|
||||
segmentIndex++
|
||||
}
|
||||
|
||||
|
||||
// Join the segments back into a path
|
||||
rebuiltPath := strings.Join(rebuiltSegments, "/")
|
||||
|
||||
|
||||
if unescaped, err := urlutil.PathDecode(rebuiltPath); err == nil {
|
||||
// this is handle the case where anyportion of path has url encoded data
|
||||
// by default the http/request official library will escape/encode special characters in path
|
||||
|
||||
@@ -99,7 +99,7 @@ func TestPathComponent_SQLInjection(t *testing.T) {
|
||||
// Let's see what path segments are available for fuzzing
|
||||
err = path.Iterate(func(key string, value interface{}) error {
|
||||
t.Logf("Key: %s, Value: %s", key, value.(string))
|
||||
|
||||
|
||||
// Try fuzzing the "55" segment specifically (which should be key "2")
|
||||
if value.(string) == "55" {
|
||||
if setErr := path.SetValue(key, "55 OR True"); setErr != nil {
|
||||
@@ -116,14 +116,14 @@ func TestPathComponent_SQLInjection(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
|
||||
t.Logf("Modified path: %s", newReq.Path)
|
||||
|
||||
|
||||
// Now with PathEncode, spaces are preserved correctly for SQL injection
|
||||
if newReq.Path != "/user/55 OR True/profile" {
|
||||
t.Fatalf("expected path to be '/user/55 OR True/profile', got '%s'", newReq.Path)
|
||||
}
|
||||
|
||||
|
||||
// Let's also test what the actual URL looks like
|
||||
t.Logf("Full URL: %s", newReq.String())
|
||||
}
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
// and not arrays
|
||||
//
|
||||
// TODO: Support arrays + other JSON oddities by
|
||||
// adding more attirbutes to the map[string]interface{}
|
||||
// adding more attributes to the map[string]interface{}
|
||||
type JSON struct{}
|
||||
|
||||
var (
|
||||
|
||||
@@ -27,7 +27,29 @@ var (
|
||||
|
||||
// NewMultiPartForm returns a new MultiPartForm encoder
|
||||
func NewMultiPartForm() *MultiPartForm {
|
||||
return &MultiPartForm{}
|
||||
return &MultiPartForm{
|
||||
filesMetadata: make(map[string]FileMetadata),
|
||||
}
|
||||
}
|
||||
|
||||
// SetFileMetadata sets the file metadata for a given field name
|
||||
func (m *MultiPartForm) SetFileMetadata(fieldName string, metadata FileMetadata) {
|
||||
if m.filesMetadata == nil {
|
||||
m.filesMetadata = make(map[string]FileMetadata)
|
||||
}
|
||||
|
||||
m.filesMetadata[fieldName] = metadata
|
||||
}
|
||||
|
||||
// GetFileMetadata gets the file metadata for a given field name
|
||||
func (m *MultiPartForm) GetFileMetadata(fieldName string) (FileMetadata, bool) {
|
||||
if m.filesMetadata == nil {
|
||||
return FileMetadata{}, false
|
||||
}
|
||||
|
||||
metadata, exists := m.filesMetadata[fieldName]
|
||||
|
||||
return metadata, exists
|
||||
}
|
||||
|
||||
// IsType returns true if the data is MultiPartForm encoded
|
||||
@@ -125,16 +147,24 @@ func (m *MultiPartForm) ParseBoundary(contentType string) error {
|
||||
if m.boundary == "" {
|
||||
return fmt.Errorf("no boundary found in the content type")
|
||||
}
|
||||
|
||||
// NOTE(dwisiswant0): boundary cannot exceed 70 characters according to
|
||||
// RFC-2046.
|
||||
if len(m.boundary) > 70 {
|
||||
return fmt.Errorf("boundary exceeds maximum length of 70 characters")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Decode decodes the data from MultiPartForm format
|
||||
func (m *MultiPartForm) Decode(data string) (KV, error) {
|
||||
if m.boundary == "" {
|
||||
return KV{}, fmt.Errorf("boundary not set, call ParseBoundary first")
|
||||
}
|
||||
|
||||
// Create a buffer from the string data
|
||||
b := bytes.NewBufferString(data)
|
||||
// The boundary parameter should be extracted from the Content-Type header of the HTTP request
|
||||
// which is not available in this context, so this is a placeholder for demonstration.
|
||||
// You will need to pass the actual boundary value to this function.
|
||||
r := multipart.NewReader(b, m.boundary)
|
||||
|
||||
form, err := r.ReadForm(32 << 20) // 32MB is the max memory used to parse the form
|
||||
@@ -153,30 +183,44 @@ func (m *MultiPartForm) Decode(data string) (KV, error) {
|
||||
result.Set(key, values[0])
|
||||
}
|
||||
}
|
||||
m.filesMetadata = make(map[string]FileMetadata)
|
||||
|
||||
if m.filesMetadata == nil {
|
||||
m.filesMetadata = make(map[string]FileMetadata)
|
||||
}
|
||||
|
||||
for key, files := range form.File {
|
||||
fileContents := []interface{}{}
|
||||
var fileMetadataList []FileMetadata
|
||||
|
||||
for _, fileHeader := range files {
|
||||
file, err := fileHeader.Open()
|
||||
if err != nil {
|
||||
return KV{}, err
|
||||
}
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
}()
|
||||
|
||||
buffer := new(bytes.Buffer)
|
||||
if _, err := buffer.ReadFrom(file); err != nil {
|
||||
_ = file.Close()
|
||||
|
||||
return KV{}, err
|
||||
}
|
||||
_ = file.Close()
|
||||
|
||||
fileContents = append(fileContents, buffer.String())
|
||||
|
||||
m.filesMetadata[key] = FileMetadata{
|
||||
fileMetadataList = append(fileMetadataList, FileMetadata{
|
||||
ContentType: fileHeader.Header.Get("Content-Type"),
|
||||
Filename: fileHeader.Filename,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
result.Set(key, fileContents)
|
||||
|
||||
// NOTE(dwisiswant0): store the first file's metadata instead of the
|
||||
// last one
|
||||
if len(fileMetadataList) > 0 {
|
||||
m.filesMetadata[key] = fileMetadataList[0]
|
||||
}
|
||||
}
|
||||
return KVOrderedMap(&result), nil
|
||||
}
|
||||
|
||||
@@ -240,3 +240,131 @@ Python
|
||||
assert.Contains(t, documentsArray, "fake_pdf_content_1")
|
||||
assert.Contains(t, documentsArray, "fake_pdf_content_2")
|
||||
}
|
||||
|
||||
func TestMultiPartForm_SetGetFileMetadata(t *testing.T) {
|
||||
form := NewMultiPartForm()
|
||||
metadata := FileMetadata{
|
||||
ContentType: "image/jpeg",
|
||||
Filename: "test.jpg",
|
||||
}
|
||||
form.SetFileMetadata("avatar", metadata)
|
||||
|
||||
// Test GetFileMetadata for existing field
|
||||
retrievedMetadata, exists := form.GetFileMetadata("avatar")
|
||||
assert.True(t, exists)
|
||||
assert.Equal(t, metadata.ContentType, retrievedMetadata.ContentType)
|
||||
assert.Equal(t, metadata.Filename, retrievedMetadata.Filename)
|
||||
|
||||
// Test GetFileMetadata for non-existing field
|
||||
_, exists = form.GetFileMetadata("nonexistent")
|
||||
assert.False(t, exists)
|
||||
}
|
||||
|
||||
func TestMultiPartForm_FilesMetadataInitialization(t *testing.T) {
|
||||
form := NewMultiPartForm()
|
||||
assert.NotNil(t, form.filesMetadata)
|
||||
|
||||
metadata := FileMetadata{
|
||||
ContentType: "text/plain",
|
||||
Filename: "test.txt",
|
||||
}
|
||||
form.SetFileMetadata("file", metadata)
|
||||
|
||||
retrievedMetadata, exists := form.GetFileMetadata("file")
|
||||
assert.True(t, exists)
|
||||
assert.Equal(t, metadata, retrievedMetadata)
|
||||
}
|
||||
|
||||
func TestMultiPartForm_BoundaryValidation(t *testing.T) {
|
||||
form := NewMultiPartForm()
|
||||
|
||||
// Test valid boundary
|
||||
err := form.ParseBoundary("multipart/form-data; boundary=testboundary")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "testboundary", form.boundary)
|
||||
|
||||
// Test missing boundary
|
||||
err = form.ParseBoundary("multipart/form-data")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "no boundary found")
|
||||
|
||||
// Test boundary too long (over 70 characters)
|
||||
longBoundary := "multipart/form-data; boundary=" + string(make([]byte, 71))
|
||||
for i := range longBoundary[len("multipart/form-data; boundary="):] {
|
||||
longBoundary = longBoundary[:len("multipart/form-data; boundary=")+i] + "a" + longBoundary[len("multipart/form-data; boundary=")+i+1:]
|
||||
}
|
||||
|
||||
err = form.ParseBoundary(longBoundary)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "boundary exceeds maximum length")
|
||||
}
|
||||
|
||||
func TestMultiPartForm_DecodeRequiresBoundary(t *testing.T) {
|
||||
form := NewMultiPartForm()
|
||||
|
||||
// Decode should fail if boundary is not set
|
||||
_, err := form.Decode("some data")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "boundary not set")
|
||||
}
|
||||
|
||||
func TestMultiPartForm_MultipleFilesMetadata(t *testing.T) {
|
||||
form := NewMultiPartForm()
|
||||
form.boundary = "----WebKitFormBoundaryMultiFileTest"
|
||||
|
||||
// Test with multiple files having the same field name
|
||||
multipartData := `------WebKitFormBoundaryMultiFileTest
|
||||
Content-Disposition: form-data; name="documents"; filename="file1.txt"
|
||||
Content-Type: text/plain
|
||||
|
||||
content1
|
||||
------WebKitFormBoundaryMultiFileTest
|
||||
Content-Disposition: form-data; name="documents"; filename="file2.txt"
|
||||
Content-Type: text/plain
|
||||
|
||||
content2
|
||||
------WebKitFormBoundaryMultiFileTest--
|
||||
`
|
||||
|
||||
decoded, err := form.Decode(multipartData)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify files are decoded correctly
|
||||
documents := decoded.Get("documents")
|
||||
require.NotNil(t, documents)
|
||||
documentsArray, ok := documents.([]interface{})
|
||||
require.True(t, ok)
|
||||
require.Len(t, documentsArray, 2)
|
||||
assert.Contains(t, documentsArray, "content1")
|
||||
assert.Contains(t, documentsArray, "content2")
|
||||
|
||||
// Verify metadata for the field exists (should be from the first file)
|
||||
metadata, exists := form.GetFileMetadata("documents")
|
||||
assert.True(t, exists)
|
||||
assert.Equal(t, "text/plain", metadata.ContentType)
|
||||
assert.Equal(t, "file1.txt", metadata.Filename) // Should be from first file, not last
|
||||
}
|
||||
|
||||
func TestMultiPartForm_SetFileMetadataWithNilMap(t *testing.T) {
|
||||
form := &MultiPartForm{}
|
||||
|
||||
// SetFileMetadata should handle nil filesMetadata
|
||||
metadata := FileMetadata{
|
||||
ContentType: "application/pdf",
|
||||
Filename: "document.pdf",
|
||||
}
|
||||
form.SetFileMetadata("doc", metadata)
|
||||
|
||||
// Should be able to retrieve the metadata
|
||||
retrievedMetadata, exists := form.GetFileMetadata("doc")
|
||||
assert.True(t, exists)
|
||||
assert.Equal(t, metadata, retrievedMetadata)
|
||||
}
|
||||
|
||||
func TestMultiPartForm_GetFileMetadataWithNilMap(t *testing.T) {
|
||||
form := &MultiPartForm{}
|
||||
|
||||
// GetFileMetadata should handle nil filesMetadata gracefully
|
||||
_, exists := form.GetFileMetadata("anything")
|
||||
assert.False(t, exists)
|
||||
}
|
||||
@@ -20,15 +20,15 @@ import (
|
||||
// This is used to reduce the number of requests made during fuzzing
|
||||
// for parameters that are less likely to give results for a rule.
|
||||
type Tracker struct {
|
||||
frequencies gcache.Cache
|
||||
paramOccurenceThreshold int
|
||||
frequencies gcache.Cache
|
||||
paramOccurrenceThreshold int
|
||||
|
||||
isDebug bool
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultMaxTrackCount = 10000
|
||||
DefaultParamOccurenceThreshold = 10
|
||||
DefaultMaxTrackCount = 10000
|
||||
DefaultParamOccurrenceThreshold = 10
|
||||
)
|
||||
|
||||
type cacheItem struct {
|
||||
@@ -38,7 +38,7 @@ type cacheItem struct {
|
||||
|
||||
// New creates a new frequency tracker with a given maximum
|
||||
// number of params to track in LRU fashion with a max error threshold
|
||||
func New(maxTrackCount, paramOccurenceThreshold int) *Tracker {
|
||||
func New(maxTrackCount, paramOccurrenceThreshold int) *Tracker {
|
||||
gc := gcache.New(maxTrackCount).ARC().Build()
|
||||
|
||||
var isDebug bool
|
||||
@@ -46,9 +46,9 @@ func New(maxTrackCount, paramOccurenceThreshold int) *Tracker {
|
||||
isDebug = true
|
||||
}
|
||||
return &Tracker{
|
||||
isDebug: isDebug,
|
||||
frequencies: gc,
|
||||
paramOccurenceThreshold: paramOccurenceThreshold,
|
||||
isDebug: isDebug,
|
||||
frequencies: gc,
|
||||
paramOccurrenceThreshold: paramOccurrenceThreshold,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,10 +56,10 @@ func (t *Tracker) Close() {
|
||||
t.frequencies.Purge()
|
||||
}
|
||||
|
||||
// MarkParameter marks a parameter as frequently occuring once.
|
||||
// MarkParameter marks a parameter as frequently occurring once.
|
||||
//
|
||||
// The logic requires a parameter to be marked as frequently occuring
|
||||
// multiple times before it's considered as frequently occuring.
|
||||
// The logic requires a parameter to be marked as frequently occurring
|
||||
// multiple times before it's considered as frequently occurring.
|
||||
func (t *Tracker) MarkParameter(parameter, target, template string) {
|
||||
normalizedTarget := normalizeTarget(target)
|
||||
key := getFrequencyKey(parameter, normalizedTarget, template)
|
||||
@@ -81,7 +81,7 @@ func (t *Tracker) MarkParameter(parameter, target, template string) {
|
||||
_ = t.frequencies.Set(key, existingCacheItemValue)
|
||||
}
|
||||
|
||||
// IsParameterFrequent checks if a parameter is frequently occuring
|
||||
// IsParameterFrequent checks if a parameter is frequently occurring
|
||||
// in the input with no much results.
|
||||
func (t *Tracker) IsParameterFrequent(parameter, target, template string) bool {
|
||||
normalizedTarget := normalizeTarget(target)
|
||||
@@ -97,7 +97,7 @@ func (t *Tracker) IsParameterFrequent(parameter, target, template string) bool {
|
||||
}
|
||||
existingCacheItemValue := existingCacheItem.(*cacheItem)
|
||||
|
||||
if existingCacheItemValue.errors.Load() >= int32(t.paramOccurenceThreshold) {
|
||||
if existingCacheItemValue.errors.Load() >= int32(t.paramOccurrenceThreshold) {
|
||||
existingCacheItemValue.Do(func() {
|
||||
gologger.Verbose().Msgf("[%s] Skipped %s from parameter for %s as found uninteresting %d times", template, parameter, target, existingCacheItemValue.errors.Load())
|
||||
})
|
||||
@@ -106,7 +106,7 @@ func (t *Tracker) IsParameterFrequent(parameter, target, template string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// UnmarkParameter unmarks a parameter as frequently occuring. This carries
|
||||
// UnmarkParameter unmarks a parameter as frequently occurring. This carries
|
||||
// more weight and resets the frequency counter for the parameter causing
|
||||
// it to be checked again. This is done when results are found.
|
||||
func (t *Tracker) UnmarkParameter(parameter, target, template string) {
|
||||
|
||||
+1
-1
@@ -136,7 +136,7 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu
|
||||
return qerr
|
||||
}
|
||||
|
||||
// after building change back to original value to avoid repeating it in furthur requests
|
||||
// after building change back to original value to avoid repeating it in further requests
|
||||
if origKey != "" {
|
||||
err = ruleComponent.SetValue(origKey, types.ToString(origValue)) // change back to previous value for temp
|
||||
if err != nil {
|
||||
|
||||
+1
-1
@@ -25,5 +25,5 @@ Currently Nuclei Supports three input providers:
|
||||
func NewInputProvider(opts InputOptions) (InputProvider, error)
|
||||
```
|
||||
|
||||
This function returns a InputProvider based by appropriately selecting input provider based on the input format (i.e either list or http) and returns the provider that can handle that input format.
|
||||
This function returns a InputProvider based by appropriately selecting input provider based on the input format (i.e. either list or http) and returns the provider that can handle that input format.
|
||||
|
||||
@@ -43,7 +43,6 @@ func (j *BurpFormat) Parse(input io.Reader, resultsCb formats.ParseReqRespCallba
|
||||
|
||||
// Print the parsed data for verification
|
||||
for _, item := range items.Items {
|
||||
item := item
|
||||
binx, err := base64.StdEncoding.DecodeString(item.Request.Raw)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not decode base64")
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
@@ -28,6 +29,12 @@ type InputFormatOptions struct {
|
||||
// RequiredOnly only uses required fields when generating requests
|
||||
// instead of all fields
|
||||
RequiredOnly bool
|
||||
// VarsTextTemplating uses Variables and inject it into the input
|
||||
// this is used for text templating of variables based on carvel ytt
|
||||
// Only available for Yaml formats
|
||||
VarsTextTemplating bool
|
||||
// VarsFilePaths is the path to the file containing variables
|
||||
VarsFilePaths []string
|
||||
}
|
||||
|
||||
// Format is an interface implemented by all input formats
|
||||
@@ -41,6 +48,16 @@ type Format interface {
|
||||
SetOptions(options InputFormatOptions)
|
||||
}
|
||||
|
||||
// SpecDownloader is an interface for downloading API specifications from URLs
|
||||
type SpecDownloader interface {
|
||||
// Download downloads the spec from the given URL and saves it to tmpDir
|
||||
// Returns the path to the downloaded file
|
||||
// httpClient is a retryablehttp.Client instance (can be nil for fallback)
|
||||
Download(url, tmpDir string, httpClient *retryablehttp.Client) (string, error)
|
||||
// SupportedExtensions returns the list of supported file extensions
|
||||
SupportedExtensions() []string
|
||||
}
|
||||
|
||||
var (
|
||||
DefaultVarDumpFileName = "required_openapi_params.yaml"
|
||||
ErrNoVarsDumpFile = errors.New("no required params file found")
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
package openapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/formats"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
)
|
||||
|
||||
// OpenAPIDownloader implements the SpecDownloader interface for OpenAPI 3.0 specs
|
||||
type OpenAPIDownloader struct{}
|
||||
|
||||
// NewDownloader creates a new OpenAPI downloader
|
||||
func NewDownloader() formats.SpecDownloader {
|
||||
return &OpenAPIDownloader{}
|
||||
}
|
||||
|
||||
// This function downloads an OpenAPI 3.0 spec from the given URL and saves it to tmpDir
|
||||
func (d *OpenAPIDownloader) Download(urlStr, tmpDir string, httpClient *retryablehttp.Client) (string, error) {
|
||||
// Validate URL format, OpenAPI 3.0 specs are typically JSON
|
||||
if !strings.HasSuffix(urlStr, ".json") {
|
||||
return "", fmt.Errorf("URL does not appear to be an OpenAPI JSON spec")
|
||||
}
|
||||
|
||||
const maxSpecSizeBytes = 10 * 1024 * 1024 // 10MB
|
||||
|
||||
// Use provided httpClient or create a fallback
|
||||
var client *http.Client
|
||||
if httpClient != nil {
|
||||
client = httpClient.HTTPClient
|
||||
} else {
|
||||
// Fallback to simple client if no httpClient provided
|
||||
client = &http.Client{Timeout: 30 * time.Second}
|
||||
}
|
||||
|
||||
resp, err := client.Get(urlStr)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to download OpenAPI spec")
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("HTTP %d when downloading OpenAPI spec", resp.StatusCode)
|
||||
}
|
||||
|
||||
bodyBytes, err := io.ReadAll(io.LimitReader(resp.Body, maxSpecSizeBytes))
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to read response body")
|
||||
}
|
||||
|
||||
// Validate it's a valid JSON and has OpenAPI structure
|
||||
var spec map[string]interface{}
|
||||
if err := json.Unmarshal(bodyBytes, &spec); err != nil {
|
||||
return "", fmt.Errorf("downloaded content is not valid JSON: %w", err)
|
||||
}
|
||||
|
||||
// Check if it's an OpenAPI 3.0 spec
|
||||
if openapi, exists := spec["openapi"]; exists {
|
||||
if openapiStr, ok := openapi.(string); ok && strings.HasPrefix(openapiStr, "3.") {
|
||||
// Valid OpenAPI 3.0 spec
|
||||
} else {
|
||||
return "", fmt.Errorf("not a valid OpenAPI 3.0 spec (found version: %v)", openapi)
|
||||
}
|
||||
} else {
|
||||
return "", fmt.Errorf("not an OpenAPI spec (missing 'openapi' field)")
|
||||
}
|
||||
|
||||
// Extract host from URL for server configuration
|
||||
parsedURL, err := url.Parse(urlStr)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to parse URL")
|
||||
}
|
||||
host := parsedURL.Host
|
||||
scheme := parsedURL.Scheme
|
||||
if scheme == "" {
|
||||
scheme = "https"
|
||||
}
|
||||
|
||||
// Add servers section if missing or empty
|
||||
servers, exists := spec["servers"]
|
||||
if !exists || servers == nil {
|
||||
spec["servers"] = []map[string]interface{}{{"url": scheme + "://" + host}}
|
||||
} else if serverList, ok := servers.([]interface{}); ok && len(serverList) == 0 {
|
||||
spec["servers"] = []map[string]interface{}{{"url": scheme + "://" + host}}
|
||||
}
|
||||
|
||||
// Marshal back to JSON
|
||||
modifiedJSON, err := json.Marshal(spec)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to marshal modified spec")
|
||||
}
|
||||
|
||||
// Create output directory
|
||||
openapiDir := filepath.Join(tmpDir, "openapi")
|
||||
if err := os.MkdirAll(openapiDir, 0755); err != nil {
|
||||
return "", errors.Wrap(err, "failed to create openapi directory")
|
||||
}
|
||||
|
||||
// Generate filename
|
||||
filename := fmt.Sprintf("openapi-spec-%d.json", time.Now().Unix())
|
||||
filePath := filepath.Join(openapiDir, filename)
|
||||
|
||||
// Write file
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create file: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
}()
|
||||
|
||||
if _, writeErr := file.Write(modifiedJSON); writeErr != nil {
|
||||
_ = os.Remove(filePath)
|
||||
return "", errors.Wrap(writeErr, "failed to write OpenAPI spec to file")
|
||||
}
|
||||
|
||||
return filePath, nil
|
||||
}
|
||||
|
||||
// SupportedExtensions returns the list of supported file extensions for OpenAPI
|
||||
func (d *OpenAPIDownloader) SupportedExtensions() []string {
|
||||
return []string{".json"}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
package openapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestOpenAPIDownloader_SupportedExtensions(t *testing.T) {
|
||||
downloader := &OpenAPIDownloader{}
|
||||
extensions := downloader.SupportedExtensions()
|
||||
|
||||
expected := []string{".json"}
|
||||
if len(extensions) != len(expected) {
|
||||
t.Errorf("Expected %d extensions, got %d", len(expected), len(extensions))
|
||||
}
|
||||
|
||||
for i, ext := range extensions {
|
||||
if ext != expected[i] {
|
||||
t.Errorf("Expected extension %s, got %s", expected[i], ext)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAPIDownloader_Download_Success(t *testing.T) {
|
||||
// Create a mock OpenAPI spec
|
||||
mockSpec := map[string]interface{}{
|
||||
"openapi": "3.0.0",
|
||||
"info": map[string]interface{}{
|
||||
"title": "Test API",
|
||||
"version": "1.0.0",
|
||||
},
|
||||
"paths": map[string]interface{}{
|
||||
"/test": map[string]interface{}{
|
||||
"get": map[string]interface{}{
|
||||
"summary": "Test endpoint",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Create mock server
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(mockSpec); err != nil {
|
||||
http.Error(w, "failed to encode response", http.StatusInternalServerError)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
// Create temp directory
|
||||
tmpDir, err := os.MkdirTemp("", "openapi_test")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := os.RemoveAll(tmpDir); err != nil {
|
||||
t.Fatalf("Failed to remove temp dir: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Test download
|
||||
downloader := &OpenAPIDownloader{}
|
||||
filePath, err := downloader.Download(server.URL+"/openapi.json", tmpDir, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Download failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify file exists
|
||||
if !fileExists(filePath) {
|
||||
t.Errorf("Downloaded file does not exist: %s", filePath)
|
||||
}
|
||||
|
||||
// Verify file content
|
||||
content, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read downloaded file: %v", err)
|
||||
}
|
||||
|
||||
var downloadedSpec map[string]interface{}
|
||||
if err := json.Unmarshal(content, &downloadedSpec); err != nil {
|
||||
t.Fatalf("Failed to parse downloaded JSON: %v", err)
|
||||
}
|
||||
|
||||
// Verify servers field was added
|
||||
servers, exists := downloadedSpec["servers"]
|
||||
if !exists {
|
||||
t.Error("Servers field was not added to the spec")
|
||||
}
|
||||
|
||||
if serversList, ok := servers.([]interface{}); ok {
|
||||
if len(serversList) == 0 {
|
||||
t.Error("Servers list is empty")
|
||||
}
|
||||
} else {
|
||||
t.Error("Servers field is not a list")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAPIDownloader_Download_NonJSONURL(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "openapi_test")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := os.RemoveAll(tmpDir); err != nil {
|
||||
t.Fatalf("Failed to remove temp dir: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
downloader := &OpenAPIDownloader{}
|
||||
_, err = downloader.Download("http://example.com/spec.yaml", tmpDir, nil)
|
||||
if err == nil {
|
||||
t.Error("Expected error for non-JSON URL, but got none")
|
||||
}
|
||||
|
||||
if !strings.Contains(err.Error(), "URL does not appear to be an OpenAPI JSON spec") {
|
||||
t.Errorf("Unexpected error message: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAPIDownloader_Download_HTTPError(t *testing.T) {
|
||||
// Create mock server that returns 404
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "openapi_test")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := os.RemoveAll(tmpDir); err != nil {
|
||||
t.Fatalf("Failed to remove temp dir: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
downloader := &OpenAPIDownloader{}
|
||||
_, err = downloader.Download(server.URL+"/openapi.json", tmpDir, nil)
|
||||
if err == nil {
|
||||
t.Error("Expected error for HTTP 404, but got none")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAPIDownloader_Download_InvalidJSON(t *testing.T) {
|
||||
// Create mock server that returns invalid JSON
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if _, err := w.Write([]byte("invalid json")); err != nil {
|
||||
http.Error(w, "failed to write response", http.StatusInternalServerError)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "openapi_test")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := os.RemoveAll(tmpDir); err != nil {
|
||||
t.Fatalf("Failed to remove temp dir: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
downloader := &OpenAPIDownloader{}
|
||||
_, err = downloader.Download(server.URL+"/openapi.json", tmpDir, nil)
|
||||
if err == nil {
|
||||
t.Error("Expected error for invalid JSON, but got none")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAPIDownloader_Download_Timeout(t *testing.T) {
|
||||
// Create mock server with delay
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(35 * time.Second) // Longer than 30 second timeout
|
||||
if err := json.NewEncoder(w).Encode(map[string]interface{}{"test": "data"}); err != nil {
|
||||
http.Error(w, "failed to encode response", http.StatusInternalServerError)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "openapi_test")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := os.RemoveAll(tmpDir); err != nil {
|
||||
t.Fatalf("Failed to remove temp dir: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
downloader := &OpenAPIDownloader{}
|
||||
_, err = downloader.Download(server.URL+"/openapi.json", tmpDir, nil)
|
||||
if err == nil {
|
||||
t.Error("Expected timeout error, but got none")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAPIDownloader_Download_WithExistingServers(t *testing.T) {
|
||||
// Create a mock OpenAPI spec with existing servers
|
||||
mockSpec := map[string]interface{}{
|
||||
"openapi": "3.0.0",
|
||||
"info": map[string]interface{}{
|
||||
"title": "Test API",
|
||||
"version": "1.0.0",
|
||||
},
|
||||
"servers": []interface{}{
|
||||
map[string]interface{}{
|
||||
"url": "https://existing-server.com",
|
||||
},
|
||||
},
|
||||
"paths": map[string]interface{}{},
|
||||
}
|
||||
|
||||
// Create mock server
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(mockSpec); err != nil {
|
||||
http.Error(w, "failed to encode response", http.StatusInternalServerError)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "openapi_test")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := os.RemoveAll(tmpDir); err != nil {
|
||||
t.Fatalf("Failed to remove temp dir: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
downloader := &OpenAPIDownloader{}
|
||||
filePath, err := downloader.Download(server.URL+"/openapi.json", tmpDir, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Download failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify existing servers are preserved
|
||||
content, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read downloaded file: %v", err)
|
||||
}
|
||||
|
||||
var downloadedSpec map[string]interface{}
|
||||
if err := json.Unmarshal(content, &downloadedSpec); err != nil {
|
||||
t.Fatalf("Failed to parse downloaded JSON: %v", err)
|
||||
}
|
||||
|
||||
servers, exists := downloadedSpec["servers"]
|
||||
if !exists {
|
||||
t.Error("Servers field was removed from the spec")
|
||||
}
|
||||
|
||||
if serversList, ok := servers.([]interface{}); ok {
|
||||
if len(serversList) != 1 {
|
||||
t.Errorf("Expected 1 server, got %d", len(serversList))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to check if file exists
|
||||
func fileExists(filename string) bool {
|
||||
_, err := os.Stat(filename)
|
||||
return !os.IsNotExist(err)
|
||||
}
|
||||
Loaded 100 of 223 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user