Compare commits

...
Author SHA1 Message Date
Nakul Bharti d4800f81f8 fix(flow): bound runtime execution 2026-10-13 22:14:20 +05:30
Nakul Bharti a5715288aa fix(runtime): honor cancellation in HTTP and headless waits (#7814)
* fix(http): cancel per-host rate admission

* fix: address PR review comments

* test: keep request coverage with request tests

* fix(headless): bound browser setup by request context

* fix: address PR review comments

* fix: address PR review comments

* fix cancellation-safe rate and headless cleanup

* fix: cancel global rate admission waits

* chore: bump ratelimit to v0.0.93
2026-10-13 23:32:09 +07:00
Nakul Bharti cc5645e674 fix(http): preserve iterate-all extraction branches (#7858)
* fix(http): preserve extractor branch correlation

* fix(http): preserve re-extracted dynamic values

* fix(http): isolate iterate-all branch state

* fix(http): address iterate-all review findings
2026-10-13 20:02:58 +05:30
dependabot[bot] 40075b0b20 chore(deps): bump the modules group across 1 directory with 5 updates (#7869)
Bumps the modules group with 5 updates in the / directory:

| Package | From | To |
| --- | --- | --- |
| [github.com/projectdiscovery/interactsh](https://github.com/projectdiscovery/interactsh) | `1.4.0` | `1.4.1` |
| [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) | `0.0.91` | `0.0.92` |
| [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) | `1.3.29` | `1.3.30` |
| [github.com/projectdiscovery/tlsx](https://github.com/projectdiscovery/tlsx) | `1.4.0` | `1.4.1` |
| [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) | `0.3.2` | `0.3.4` |



Updates `github.com/projectdiscovery/interactsh` from 1.4.0 to 1.4.1
- [Release notes](https://github.com/projectdiscovery/interactsh/releases)
- [Commits](https://github.com/projectdiscovery/interactsh/compare/v1.4.0...v1.4.1)

Updates `github.com/projectdiscovery/ratelimit` from 0.0.91 to 0.0.92
- [Release notes](https://github.com/projectdiscovery/ratelimit/releases)
- [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.91...v0.0.92)

Updates `github.com/projectdiscovery/retryablehttp-go` from 1.3.29 to 1.3.30
- [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases)
- [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.3.29...v1.3.30)

Updates `github.com/projectdiscovery/tlsx` from 1.4.0 to 1.4.1
- [Release notes](https://github.com/projectdiscovery/tlsx/releases)
- [Commits](https://github.com/projectdiscovery/tlsx/compare/v1.4.0...v1.4.1)

Updates `github.com/projectdiscovery/wappalyzergo` from 0.3.2 to 0.3.4
- [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases)
- [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.3.2...v0.3.4)

---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/interactsh
  dependency-version: 1.4.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/ratelimit
  dependency-version: 0.0.92
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/retryablehttp-go
  dependency-version: 1.3.30
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/tlsx
  dependency-version: 1.4.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/wappalyzergo
  dependency-version: 0.3.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-10-13 10:45:51 +00:00
Dogan Can Bakir 90a0cce807 Merge pull request #7824 from projectdiscovery/fix/smb-connect-race
cancel a stalled SMB connect through its context
2026-10-13 13:16:21 +03:00
Doğan Can Bakır 1caa955c77 chore: bump fastdialer to v0.5.25 2026-10-13 12:43:26 +03:00
Doğan Can Bakır 9b368f3eaa Merge remote-tracking branch 'origin/dev' into fix/smb-connect-race
# Conflicts:
#	go.mod
#	go.sum
2026-10-12 11:57:44 +03:00
Dogan Can Bakir 59e6ffa6fb Merge pull request #7822 from projectdiscovery/fix/7821-js-call-cancellation
cancel javascript library calls with the template execution timeout
2026-10-12 11:33:23 +03:00
Doğan Can Bakır b870cdcd67 chore: bump goimpacket to main 2026-10-12 11:03:19 +03:00
Doğan Can Bakır 3f776b5ff0 chore: bump goja to master 2026-10-12 10:58:22 +03:00
Doğan Can Bakır ad74f0094c chore: pin goja to the projectdiscovery/goja branch instead of a fork 2026-10-11 21:57:38 +03:00
Dogan Can Bakir a48a58ef59 Merge pull request #7847 from projectdiscovery/fix/per-host-rate-limit-all-protocols
apply per-host rate limiting to every protocol
2026-10-09 21:11:47 +02:00
Mzack9999 0de51237b7 Merge pull request #7588 from projectdiscovery/5717-graphql-fuzz
fuzz: GraphQL-over-HTTP body support for DAST
2026-10-09 20:48:41 +02:00
Mzack9999 4857883667 Merge remote-tracking branch 'origin/fix/rate-limit-all-protocols' into fix/per-host-rate-limit-all-protocols 2026-10-09 20:34:20 +02:00
Mzack9999 bccaa4410d merge dev 2026-10-09 20:34:08 +02:00
Mzack9999 d8253b97e9 retrigger checks 2026-10-09 20:30:35 +02:00
Mzack9999 65f60dd0a6 retrigger checks 2026-10-09 20:30:35 +02:00
Mzack9999 1115f18b68 fix keys 2026-10-09 20:26:43 +02:00
Mzack9999 458367feb7 add tests 2026-10-09 20:20:02 +02:00
Dogan Can Bakir 59dad1b619 Merge pull request #7834 from projectdiscovery/perf/functional-runtime
walk and parse the templates directory once for workflow tag lookups
2026-10-08 09:33:23 +03:00
Dogan Can Bakir 889c8ff9c5 Merge pull request #7840 from projectdiscovery/fix/7836-fuzz-progress-total
count fuzzing requests in the progress total
2026-10-08 09:33:10 +03:00
Dogan Can Bakir c990724404 Merge pull request #7841 from projectdiscovery/fix/7837-js-port-requests
count javascript requests per port
2026-10-08 09:32:56 +03:00
Dogan Can Bakir 50e0afd188 Merge pull request #7842 from projectdiscovery/fix/7838-flow-request-count
keep flow progress at 100% for repeated and named requests
2026-10-08 09:32:41 +03:00
Mzack9999 04d0caa646 merge dev 2026-10-07 23:07:06 +02:00
Mzack9999 80a001ca0f Merge pull request #7843 from projectdiscovery/fix/7839-js-precondition-count
count javascript pre-condition requests without -debug
2026-10-07 23:04:03 +02:00
Doğan Can Bakır 1b080a7d19 fix: apply per-host rate limiting to every protocol 2026-10-07 12:45:14 +03:00
Doğan Can Bakır a847374f2c fix: apply the rate limit to network, ssl, websocket, whois, javascript and headless 2026-10-07 00:17:52 +03:00
Doğan Can Bakır 9c9247895d fix(flow): count skipped requests as processed like other protocols 2026-10-06 23:21:34 +03:00
Doğan Can Bakır 37052a7e3f fix(fuzz): count nothing up front for fuzzing requests with path or raw 2026-10-06 23:16:08 +03:00
Doğan Can Bakır c4825e98c9 fix(js): count pre-condition requests without -debug 2026-10-06 23:07:25 +03:00
Doğan Can Bakır 0c0e886c2c fix(flow): count only requests a flow sends 2026-10-06 23:03:36 +03:00
Doğan Can Bakır b3e02efa7f fix(js): count requests per port 2026-10-06 23:03:06 +03:00
Doğan Can Bakır aa3c7a361c fix(fuzz): count fuzzing requests in the progress total 2026-10-06 23:02:53 +03:00
Dogan Can Bakir afb2afaf4f Merge pull request #7800 from projectdiscovery/feat/7782-dedup-crawled-urls
deduplicate crawled urls before template execution
2026-10-06 22:13:56 +03:00
Doğan Can Bakır 6fde18d311 fix(js): reset the runtime context with context.TODO 2026-10-06 21:58:07 +03:00
Doğan Can Bakır 62721e589a perf(workflow): walk and parse the templates directory once per loader for tag lookups 2026-10-06 20:50:31 +03:00
Doğan Can Bakır 66ef296067 test(functional): report per-test durations 2026-10-06 20:50:31 +03:00
Mzack9999 bd510ac6bf Merge pull request #7827 from projectdiscovery/dwisiswant0/fix/fuzz/preserve-cookie-payloads-when-rebuilding-requests
fix(fuzz): preserve cookie payloads when rebuilding requests
2026-10-06 14:10:54 +02:00
Mzack9999 71457b4f69 Merge pull request #7820 from projectdiscovery/perf/jobs-after-lint
start integration, functional and other jobs right after lint
2026-10-06 14:08:15 +02:00
Dwi Siswanto 97c8233b72 test: avoid DNS lookups in cookie redirect tests
Dial the test listener directly while retaining distinct request
hosts for cookie scope checks. This avoids localhost lookup failures
on Windows.

Signed-off-by: Dwi Siswanto <[email protected]>
2026-10-06 12:30:55 +07:00
Mzack9999 6bb5b5dbd1 fix lint 2026-10-05 22:31:44 +02:00
Dwi Siswanto 8f23a8633f fix(fuzz): preserve cookie payloads when rebuilding requests
`Cookie.String` sanitizes values that can be deliberate fuzz payloads.
Stored cookies can also introduce a second value for the cookie under
test.

Build Cookie headers from raw fields and track the keys selected for
fuzzing. Use a filtered jar on an isolated client to retain session
updates without replacing those fields.

Fixes #7813
Fixes #7826

Signed-off-by: Dwi Siswanto <[email protected]>
2026-10-06 01:48:04 +07:00
Doğan Can Bakır 28d5b3e8c0 fix(smb): cancel a stalled connect through its context instead of closing it concurrently 2026-10-05 17:17:47 +03:00
Doğan Can Bakır c2d38083dc fix(js): cancel library calls with the template execution timeout 2026-10-05 16:35:38 +03:00
Dogan Can Bakir 1884ed920f Merge pull request #7807 from projectdiscovery/perf/fast-downloader-timeout-tests
cut unit and integration test runtime
2026-10-05 16:22:40 +03:00
Doğan Can Bakır d28dd2c745 ci: start every job after lint instead of after unit tests 2026-10-05 15:59:27 +03:00
Mzack9999 3336556187 Merge pull request #7818 from projectdiscovery/perf/release-test-after-lint
run release test right after lint
2026-10-05 14:48:01 +02:00
Mzack9999 2c8f4a0318 resolve aliases 2026-10-05 14:26:23 +02:00
Doğan Can Bakır 8c8a1a9d78 ci: run the release test after lint instead of after unit tests 2026-10-05 13:51:35 +03:00
Doğan Can Bakır 81749cf750 test(integration): install templates once before cases and never auto-update in cases 2026-10-05 12:57:09 +03:00
Doğan Can Bakır 846f4425b0 perf(installer): open the templates root and create each directory once per install 2026-10-05 12:53:47 +03:00
Doğan Can Bakır fa876a4d92 fix(installer): give every alias of a templates directory the same update lock 2026-10-05 12:49:20 +03:00
Doğan Can Bakır 0ae69a7d53 fix(installer): reload the templates version after taking the update lock 2026-10-01 18:10:08 +03:00
Doğan Can Bakır 80b58e6e9b test(integration): report per-test durations 2026-10-01 16:36:38 +03:00
Doğan Can Bakır 4d7e91e4e0 ci: drop unused build and disk cleanup from the tests job 2026-10-01 15:55:23 +03:00
Doğan Can Bakır 6b9208cdbf refactor(installer): inject the templates release and scope the update lock per directory 2026-10-01 15:55:17 +03:00
Doğan Can Bakır 26b28c69a4 test(installer): serve releases from memory and isolate the update lock 2026-10-01 15:13:39 +03:00
Dogan Can Bakir a14b3de2a4 Merge pull request #7791 from projectdiscovery/feat/7743-llm-non-http-protocols
llm matchers and extractors on all protocols
2026-10-01 14:00:19 +03:00
Mzack9999 3c434fb73a fix lint 2026-09-30 22:48:03 +02:00
Mzack9999 f40fa195b6 fix dedup 2026-09-30 22:28:21 +02:00
Mzack9999 a2b6d2abc7 Merge pull request #7805 from projectdiscovery/perf/windows-ci-defender-exclusion
exclude the workspace from defender scanning on windows
2026-09-30 22:05:13 +02:00
Doğan Can Bakır b87b28d7ad test: drop the interactsh cooldown from test defaults 2026-09-30 17:47:56 +03:00
Doğan Can Bakır 4d9f2f9867 test(headless): reset debug state between engine tests 2026-09-30 17:41:21 +03:00
Doğan Can Bakır 4177ca752b test(headless): share one browser across the engine tests 2026-09-30 17:38:49 +03:00
Doğan Can Bakır 8ffccaf639 test(input): stop waiting out the real timeout in downloader tests 2026-09-30 17:25:14 +03:00
Doğan Can Bakır fd1da307b6 ci: exclude the workspace from defender scanning on windows 2026-09-30 16:04:42 +03:00
Mzack9999 0e243e3edb seed audits 2026-09-29 21:43:54 +02:00
dw1 54f9310945 chore: migrate github clients to go-github/v92 (#7778)
Use v92 for template downloads and issue reporting, removing the direct
v30 dependency. Adapt client construction, issue request types, and
archive calls while preserving authentication fallback, custom API
paths, and the single repository-rename redirect.

Signed-off-by: Dwi Siswanto <[email protected]>
2026-09-29 21:10:11 +02:00
Doğan Can Bakır 3a827b8c8b feat(input): deduplicate crawled urls before template execution 2026-09-29 16:10:55 +03:00
Doğan Can Bakır 1b8e3515bf feat(llm): attach llm audit on every protocol 2026-09-28 13:46:17 +03:00
Doğan Can Bakır 0de1d12cc9 feat(llm): evaluate llm matchers and extractors on all protocols 2026-09-28 12:43:01 +03:00
ghost 47f74706a5 docs: update syntax & JSON schema 🤖 2026-09-28 08:35:44 +00:00
Dogan Can Bakir dad93f0ed6 Merge pull request #7774 from projectdiscovery/feat/7742-llm-matcher-inputs
feat(llm): compare multiple responses with matcher inputs
2026-09-28 11:30:19 +03:00
dependabot[bot] 3adff6347b chore(deps): bump the modules group with 6 updates (#7787)
Bumps the modules group with 6 updates:

| Package | From | To |
| --- | --- | --- |
| [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) | `0.5.21` | `0.5.22` |
| [github.com/projectdiscovery/interactsh](https://github.com/projectdiscovery/interactsh) | `1.3.1` | `1.4.0` |
| [github.com/projectdiscovery/networkpolicy](https://github.com/projectdiscovery/networkpolicy) | `0.1.51` | `0.1.52` |
| [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) | `0.0.90` | `0.0.91` |
| [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) | `1.3.27` | `1.3.28` |
| [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) | `0.3.1` | `0.3.2` |


Updates `github.com/projectdiscovery/fastdialer` from 0.5.21 to 0.5.22
- [Release notes](https://github.com/projectdiscovery/fastdialer/releases)
- [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.5.21...v0.5.22)

Updates `github.com/projectdiscovery/interactsh` from 1.3.1 to 1.4.0
- [Release notes](https://github.com/projectdiscovery/interactsh/releases)
- [Commits](https://github.com/projectdiscovery/interactsh/compare/v1.3.1...v1.4.0)

Updates `github.com/projectdiscovery/networkpolicy` from 0.1.51 to 0.1.52
- [Release notes](https://github.com/projectdiscovery/networkpolicy/releases)
- [Commits](https://github.com/projectdiscovery/networkpolicy/compare/v0.1.51...v0.1.52)

Updates `github.com/projectdiscovery/ratelimit` from 0.0.90 to 0.0.91
- [Release notes](https://github.com/projectdiscovery/ratelimit/releases)
- [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.90...v0.0.91)

Updates `github.com/projectdiscovery/retryablehttp-go` from 1.3.27 to 1.3.28
- [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases)
- [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.3.27...v1.3.28)

Updates `github.com/projectdiscovery/wappalyzergo` from 0.3.1 to 0.3.2
- [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases)
- [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.3.1...v0.3.2)

---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/fastdialer
  dependency-version: 0.5.22
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/interactsh
  dependency-version: 1.4.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/networkpolicy
  dependency-version: 0.1.52
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/ratelimit
  dependency-version: 0.0.91
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/retryablehttp-go
  dependency-version: 1.3.28
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/wappalyzergo
  dependency-version: 0.3.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-09-28 06:23:44 +00:00
dw1 c5611b9ac4 feat(js): expose SMB auth status & migrate goimpacket (#7780)
* feat(js): expose SMB auth status & migrate goimpacket

Add `SMBClient.Authenticate` to check session setup without accessing
shares. Report success and the server's guest and null-session flags so
callers can distinguish credential authentication from guest or anonymous
access. Return failures as errors and close the session after each call.

Switch to projectdiscovery/[email protected]. Adapt krbforge to upstream's
mandatory <username>.ccache output in the current directory. Validate
the destination against the existing file policy before ticket creation
and return the actual filename. Reject custom output paths and "-", as
upstream no longer supports custom paths or in-memory ticket generation.

Closes #7760
Closes #7752
Fixes #7761

Signed-off-by: Dwi Siswanto <[email protected]>

* chore: satisfy lint

Signed-off-by: Dwi Siswanto <[email protected]>

* fix(js): validate execution ID in smb.Authenticate

Return an error when the executionId context value is missing, empty, or
not a string. This prevents the unchecked type assertion from panicking
for missing or incorrectly typed values.

Signed-off-by: Dwi Siswanto <[email protected]>

---------

Signed-off-by: Dwi Siswanto <[email protected]>
2026-09-26 08:59:56 +07:00
Mzack9999 77d3b6d314 retry functional 2026-09-25 13:57:31 +02:00
Mr.XandDwi Siswanto c36316de36 chore(deps): bump github.com/projectdiscovery/[email protected] (#7766)
* Bump projectdiscovery/utils to pick up the zstd decoder leak fix

http.NewResponseChain() decompresses Content-Encoding: zstd responses
through utils's wrapDecodeReader(), which builds a *zstd.Decoder and
wraps it in io.NopCloser. The decoder is never closed, and klauspost's
zstd decoder keeps background worker goroutines alive until Close() is
called. When a response's decompressed size exceeds MaxBodyRead, the
LimitReader stops before EOF, the decoder's workers block permanently,
and the goroutines plus their window buffers leak for the life of the
process (projectdiscovery/nuclei#7749). gzip/deflate/brotli don't spawn
goroutines, so only zstd is affected.

The fix (github.com/projectdiscovery/utils@27864e5, PR #783) already
landed on utils's main branch, but no tagged release contains it yet
(latest is v0.11.5), so go.mod still points at v0.11.4. Bump to a
pseudo-version pinned at that commit as the interim step the issue
calls for, to be replaced by a tagged version once utils cuts one.

Verified with the deterministic reproducer from the issue report
(NewResponseChain + Fill against a >cap zstd body, 50 iterations):
goroutine count returns to baseline (delta=0) instead of leaking ~1
goroutine per response.

* chore(deps): bump github.com/projectdiscovery/[email protected].6

Signed-off-by: Dwi Siswanto <[email protected]>

---------

Signed-off-by: Dwi Siswanto <[email protected]>
Co-authored-by: Dwi Siswanto <[email protected]>
2026-09-25 18:50:59 +07:00
Mzack9999 010ee044b2 fix inputs 2026-09-24 20:00:05 +02:00
Mzack9999 71f9782b2c Merge pull request #7777 from projectdiscovery/fix/7775-sonic-warning
fix(json): keep sonic out of builds it does not support
2026-09-24 19:50:48 +02:00
Mzack9999 536e427c01 Merge pull request #7776 from projectdiscovery/fix/7764-go-1.27
chore(deps): require go 1.27.1
2026-09-24 19:48:35 +02:00
Doğan Can Bakır ae203e7aad test(sdk): ignore the renamed http2 idle connection goroutine on go 1.27 2026-09-24 10:22:32 +03:00
Doğan Can Bakır 2b6b1d3371 fix(json): keep sonic out of builds it does not support 2026-09-24 10:07:28 +03:00
Doğan Can Bakır 3fdd8ca091 chore(deps): require go 1.27.1 for windows overlapped I/O fix 2026-09-24 09:30:27 +03:00
ghost af77e344f0 docs: update syntax & JSON schema 🤖 2026-09-24 06:17:10 +00:00
Doğan Can Bakır 1b244b2330 feat(llm): compare multiple responses with matcher inputs 2026-09-24 09:14:01 +03:00
Dogan Can Bakir fb1a318c6a Merge pull request #7773 from projectdiscovery/dwisiswant0/refactor/enforce-scan-network-restrictions-in-network-helpers
refactor: enforce scan network restrictions in network helpers
2026-09-24 09:13:43 +03:00
Dogan Can Bakir dbc69f7bf0 Merge pull request #7772 from projectdiscovery/feat/7744-llm-prompt-interpolation
feat(llm): interpolate variables and target values in llm prompts
2026-09-24 09:13:07 +03:00
Mzack9999 aed2cb4643 fix literals 2026-09-24 00:50:53 +02:00
Mzack9999 15e2d88ca5 fix graphql 2026-09-23 21:58:13 +02:00
Mzack9999 4d967cba5c merge dev 2026-09-23 20:43:05 +02:00
Mzack9999 dece1c5ea9 fix helpers 2026-09-23 18:57:29 +02:00
Mzack9999 5e5758b7d3 fix interpolation 2026-09-23 14:23:46 +02:00
Mzack9999 06fb71ebf2 fix lint 2026-09-23 14:23:46 +02:00
Dwi Siswanto c50f92fdd3 refactor: enforce scan network restrictions in network helpers
Pass scan options through rendering, variables, matchers, and extractors
so resolve(), public_ip()/publicip(), and jarm() honor -lna and -eh.
These helpers previously performed network I/O outside the scan's network
policy.

Use the scan dialer for HTTP and JARM connections and reject restricted
IP addresses in DNS answers. Validate original and normalized hostnames
to prevent exclusion bypasses through DNS escapes and trailing dots.

Enforce -eh with or without -lna, including in unsigned templates.
Keep local access available without -lna unless excluded, and preserve
configured DNS resolvers, including local ones.

Bind helpers at evaluation time without storing scan state in cached
expressions. Reject network helper calls without initialized scan
context.

Fixes #7769

Signed-off-by: Dwi Siswanto <[email protected]>
2026-09-23 18:53:49 +07:00
Doğan Can Bakır 5b6510fb79 feat(llm): interpolate variables and target values in llm prompts 2026-09-23 12:09:18 +03:00
Dogan Can Bakir a5b59a9a4e Merge pull request #7765 from projectdiscovery/fix/7764-go-1.26.8
chore(deps): require go 1.26.8
2026-09-22 22:11:27 +03:00
Dogan Can Bakir 14283b0e13 Merge pull request #7735 from projectdiscovery/feat/llm-matchers
add llm matchers and extractors (semantic matching)
2026-09-22 16:59:11 +03:00
Doğan Can Bakır 322a40e1db fix(headless): subscribe wait-event before navigating 2026-09-22 16:03:27 +03:00
Dogan Can Bakir aa457f1c44 Merge pull request #7763 from projectdiscovery/feat/websocket-js-module
feat(js): add websocket module
2026-09-22 15:50:41 +03:00
Doğan Can Bakır bd0c398887 test(headless): cover wait-event missing events fired before waiting 2026-09-22 15:46:03 +03:00
Doğan Can Bakır b4a5cdab02 Merge remote-tracking branch 'origin/dev' into feat/llm-matchers
# Conflicts:
#	go.mod
#	go.sum
2026-09-22 15:26:55 +03:00
Doğan Can Bakır 1981edbaac chore(deps): require go 1.26.8 for windows overlapped I/O fix 2026-09-22 15:04:03 +03:00
Dogan Can Bakir 0e2fb326d6 Merge pull request #7759 from projectdiscovery/feat/per-target-profiles
feat(input): per-target profiles via target lines
2026-09-22 14:12:46 +03:00
Doğan Can Bakır ef0faa7d6f fix(js): answer pings between websocket fragments and release closed connections 2026-09-22 13:01:47 +03:00
Doğan Can Bakır e8b53505e5 fix(js): use projectdiscovery/goja import in bindgen template 2026-09-22 12:41:44 +03:00
Doğan Can Bakır 1d6b532bbc docs(js): note which headers websocket ResponseHeaders omits 2026-09-22 11:58:05 +03:00
Doğan Can Bakır b6cea874d3 feat(js): register websocket module with bindings and integration tests 2026-09-22 11:31:32 +03:00
Doğan Can Bakır 36dafa988b feat(js): add websocket module 2026-09-22 11:16:06 +03:00
Mzack9999 d7bdb9c058 Merge pull request #7746 from dwisiswant0/dwisiswant0/fix/dedupe/preserve-input-identity-serialize-indexing
fix(dedupe): preserve input identity & serialize indexing
2026-09-21 21:46:03 +02:00
Doğan Can Bakır f3c12c43eb fix(input): accept only profile IDs in target lines 2026-09-21 18:18:43 +03:00
Doğan Can Bakır 4dad173f0a fix(input): load every template when a plain duplicate lifts a restriction 2026-09-21 18:14:13 +03:00
Doğan Can Bakır 1b557852c4 test(input): cover target lines in list provider 2026-09-21 16:38:46 +03:00
Doğan Can Bakır e5052eb771 docs(input): document per-target profiles 2026-09-21 16:37:56 +03:00
Doğan Can Bakır 56411349d2 fix(input): match multi-word tech names by each word 2026-09-21 16:33:10 +03:00
Doğan Can Bakır 77dda01e67 feat(core): run per-target profiles in engine, clusters and loader 2026-09-21 15:53:54 +03:00
Doğan Can Bakır a7faddf748 feat(input): parse per-target profile lines into a target scope registry 2026-09-21 15:47:11 +03:00
dependabot[bot] d47ace7701 chore(deps): bump the modules group with 8 updates (#7758)
Bumps the modules group with 8 updates:

| Package | From | To |
| --- | --- | --- |
| [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) | `0.5.20` | `0.5.21` |
| [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) | `1.3.26` | `1.3.27` |
| [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) | `0.8.23-0.20260910174914-8e1207b367b8` | `0.8.24` |
| [github.com/projectdiscovery/networkpolicy](https://github.com/projectdiscovery/networkpolicy) | `0.1.50` | `0.1.51` |
| [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) | `0.0.90-0.20260912164306-aeeca5645294` | `0.0.90` |
| [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) | `0.11.4-0.20260910160917-83d6b6e62668` | `0.11.4-0.20260911153819-02811729e1f0` |
| [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) | `0.3.0` | `0.3.1` |
| [github.com/projectdiscovery/cdncheck](https://github.com/projectdiscovery/cdncheck) | `1.3.0` | `1.3.1` |


Updates `github.com/projectdiscovery/fastdialer` from 0.5.20 to 0.5.21
- [Release notes](https://github.com/projectdiscovery/fastdialer/releases)
- [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.5.20...v0.5.21)

Updates `github.com/projectdiscovery/retryablehttp-go` from 1.3.26 to 1.3.27
- [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases)
- [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.3.26...v1.3.27)

Updates `github.com/projectdiscovery/dsl` from 0.8.23-0.20260910174914-8e1207b367b8 to 0.8.24
- [Release notes](https://github.com/projectdiscovery/dsl/releases)
- [Commits](https://github.com/projectdiscovery/dsl/commits/v0.8.24)

Updates `github.com/projectdiscovery/networkpolicy` from 0.1.50 to 0.1.51
- [Release notes](https://github.com/projectdiscovery/networkpolicy/releases)
- [Commits](https://github.com/projectdiscovery/networkpolicy/compare/v0.1.50...v0.1.51)

Updates `github.com/projectdiscovery/ratelimit` from 0.0.90-0.20260912164306-aeeca5645294 to 0.0.90
- [Release notes](https://github.com/projectdiscovery/ratelimit/releases)
- [Commits](https://github.com/projectdiscovery/ratelimit/commits/v0.0.90)

Updates `github.com/projectdiscovery/utils` from 0.11.4-0.20260910160917-83d6b6e62668 to 0.11.4-0.20260911153819-02811729e1f0
- [Release notes](https://github.com/projectdiscovery/utils/releases)
- [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md)
- [Commits](https://github.com/projectdiscovery/utils/commits)

Updates `github.com/projectdiscovery/wappalyzergo` from 0.3.0 to 0.3.1
- [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases)
- [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.3.0...v0.3.1)

Updates `github.com/projectdiscovery/cdncheck` from 1.3.0 to 1.3.1
- [Release notes](https://github.com/projectdiscovery/cdncheck/releases)
- [Commits](https://github.com/projectdiscovery/cdncheck/compare/v1.3.0...v1.3.1)

---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/fastdialer
  dependency-version: 0.5.21
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/retryablehttp-go
  dependency-version: 1.3.27
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/dsl
  dependency-version: 0.8.24
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/networkpolicy
  dependency-version: 0.1.51
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/ratelimit
  dependency-version: 0.0.90
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/utils
  dependency-version: 0.11.4-0.20260911153819-02811729e1f0
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/wappalyzergo
  dependency-version: 0.3.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: modules
- dependency-name: github.com/projectdiscovery/cdncheck
  dependency-version: 1.3.1
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: modules
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-09-21 06:22:44 +00:00
dependabot[bot] 37c06098ad chore(deps): bump google.golang.org/grpc (#7754)
Bumps the go_modules group with 1 update in the / directory: [google.golang.org/grpc](https://github.com/grpc/grpc-go).


Updates `google.golang.org/grpc` from 1.83.1 to 1.83.2
- [Release notes](https://github.com/grpc/grpc-go/releases)
- [Commits](https://github.com/grpc/grpc-go/compare/v1.83.1...v1.83.2)

---
updated-dependencies:
- dependency-name: google.golang.org/grpc
  dependency-version: 1.83.2
  dependency-type: direct:production
  dependency-group: go_modules
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-09-18 00:53:21 +00:00
Doğan Can Bakır 12fd62163b use a process-stable framing marker so llm cache still hits 2026-09-15 15:54:37 +03:00
Dwi Siswanto 748ff1381e fix(dedupe): preserve input identity & serialize indexing
Include Port, Scheme, and URL in the deduplication key so distinct
inputs that redirect to the same matched URL remain separate findings.
Encode field and collection lengths to prevent ambiguous keys, and sort
metadata keys to make hashing deterministic.

Protect the LevelDB lookup and insert with a mutex to prevent
concurrent calls from accepting the same finding more than once.

Version the stored keys and document the upgrade behavior. Existing
report databases may report previous findings once again because old
hashes cannot recover the missing input fields.

Fixes #7745

Signed-off-by: Dwi Siswanto <[email protected]>
2026-09-15 19:20:46 +07:00
Doğan Can Bakır 026a1890b8 Merge remote-tracking branch 'origin/dev' into feat/llm-matchers
# Conflicts:
#	go.mod
#	go.sum
2026-09-15 12:51:00 +03:00
Doğan Can Bakır e14933f765 frame llm response with a random boundary to resist prompt injection 2026-09-15 12:44:43 +03:00
Doğan Can Bakır bd639a5aff document llm flags in readmes 2026-09-14 16:00:46 +03:00
Doğan Can Bakır fcba75d191 record model verdict on llm findings and regen schema 2026-09-14 15:28:32 +03:00
Doğan Can Bakır 33067508b1 bump utils to v0.11.4 2026-09-14 15:03:13 +03:00
Doğan Can Bakır d4a40d2b04 Merge remote-tracking branch 'origin/dev' into feat/llm-matchers
# Conflicts:
#	go.mod
#	go.sum
2026-09-14 14:36:46 +03:00
Doğan Can Bakır c063d77d2d run cheap matchers before llm and defer llm extractors 2026-09-14 14:21:04 +03:00
Doğan Can Bakır ddb2edb2e9 harden llm matchers: no negative, confidence floor, scoped client, http only 2026-09-14 14:00:06 +03:00
Doğan Can Bakır fe14cf5d2c reject llm as sole matcher on high or critical templates 2026-09-11 18:26:37 +03:00
Doğan Can Bakır 89db6cf43a add llm extractor and share client with matcher 2026-09-11 18:08:53 +03:00
Doğan Can Bakır a78ea81854 wire llm matcher client, flags, and capability gate 2026-09-11 17:51:19 +03:00
Mzack9999 e30c800e8c graphql fuzz 2026-07-24 17:44:01 +04:00
219 changed files with 14565 additions and 1259 deletions

No files matched your search

+21 -17
View File
@@ -17,6 +17,8 @@ concurrency:
cancel-in-progress: true
jobs:
# lint gates every other job; none of them uses another's output, so they run
# in parallel once it passes instead of queueing behind the unit tests
lint:
name: "Lint"
if: ${{ !endsWith(github.actor, '[bot]') }}
@@ -45,16 +47,6 @@ jobs:
- uses: actions/[email protected]
- uses: projectdiscovery/actions/setup/go@v1
- uses: projectdiscovery/nuclei-action/cache@v3
- uses: projectdiscovery/actions/free-disk-space@v1
with:
llvm: 'false'
php: 'false'
mongodb: 'false'
mysql: 'false'
misc-packages: 'false'
docker-images: 'false'
tools-cache: 'false'
- run: make build
# The data-race detector is OS-independent, so we only pay its ~2-3x cost
# on ubuntu; windows/macOS run the same suite without instrumentation.
- name: "Unit tests (race)"
@@ -85,7 +77,7 @@ jobs:
sdk:
name: "Run example SDK"
needs: ["tests"]
needs: ["lint"]
runs-on: ubuntu-latest
env:
GITHUB_TOKEN: "${{ github.token }}"
@@ -106,7 +98,7 @@ jobs:
integration:
name: "Integration tests"
needs: ["tests"]
needs: ["lint"]
env:
GITHUB_TOKEN: "${{ github.token }}"
strategy:
@@ -119,6 +111,18 @@ jobs:
- uses: projectdiscovery/actions/setup/python@v1
- uses: projectdiscovery/actions/setup/go@v1
- uses: projectdiscovery/nuclei-action/cache@v3
# The suite launches one nuclei process per case, ~175 of them, and the
# binary is around 200MB. This job takes roughly four times longer on
# windows than on linux for the same work; Defender scanning an executable
# on every launch is the leading suspect for the difference.
- name: Exclude the workspace from Defender scanning
if: ${{ matrix.os == 'windows-latest' }}
shell: pwsh
run: |
Add-MpPreference -ExclusionPath "${{ github.workspace }}"
Add-MpPreference -ExclusionPath "$env:USERPROFILE\go"
Add-MpPreference -ExclusionProcess "nuclei.exe"
Add-MpPreference -ExclusionProcess "go.exe"
- run: make integration
env:
PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
@@ -126,7 +130,7 @@ jobs:
functional:
name: "Functional tests"
needs: ["tests"]
needs: ["lint"]
env:
GITHUB_TOKEN: "${{ github.token }}"
strategy:
@@ -147,7 +151,7 @@ jobs:
validate:
name: "Template validate"
needs: ["tests"]
needs: ["lint"]
runs-on: ubuntu-latest
env:
GITHUB_TOKEN: "${{ github.token }}"
@@ -159,7 +163,7 @@ jobs:
codeql:
name: "CodeQL analysis"
needs: ["tests"]
needs: ["lint"]
runs-on: ubuntu-latest
permissions:
actions: read
@@ -201,7 +205,7 @@ jobs:
release:
name: "Release test"
needs: ["tests"]
needs: ["lint"]
runs-on: ubuntu-latest
steps:
- uses: actions/[email protected]
@@ -210,7 +214,7 @@ jobs:
flamegraph:
name: "Flamegraph"
needs: ["tests"]
needs: ["lint"]
uses: ./.github/workflows/flamegraph.yaml
# perf-regression:
+2 -2
View File
@@ -94,7 +94,7 @@ test:
# Keep this under the GitHub Actions step timeout (50m) so a hung nuclei process
# is reported by go test instead of the runner killing the job with no test name.
integration:
$(GOTEST) -tags=integration -timeout 40m ./internal/tests/integration
$(GOTEST) -tags=integration -timeout 40m -v ./internal/tests/integration
integration-debug:
$(GOTEST) -tags=integration ./internal/tests/integration -v $(GO_TEST_ARGS) -args $(INTEGRATION_ARGS)
@@ -112,7 +112,7 @@ functional: build
exit 1; \
fi; \
RELEASE_BINARY="$$release_binary" DEV_BINARY="$(PWD)/bin/nuclei" \
$(GOTEST) -tags=functional -timeout 1h ./internal/tests/functional
$(GOTEST) -tags=functional -timeout 1h -v ./internal/tests/functional
tidy:
$(GOMOD) tidy
+10
View File
@@ -145,6 +145,8 @@ TARGET:
-resume string resume scan from and save to specified file (clustering will be disabled)
-sa, -scan-all-ips scan all the IP's associated with dns record
-iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4)
-dun, -disable-url-normalization keep targets exactly as supplied instead of collapsing urls that address the same resource
-mup, -max-urls-per-pattern int scan at most this many urls sharing one structural shape, e.g. /user/1 and /user/2 (0 to scan all)
TARGET-FORMAT:
-im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
@@ -173,6 +175,14 @@ TEMPLATES:
-esc, -enable-self-contained enable loading self-contained templates
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
-llm-max-calls int maximum llm calls per scan (0 for unlimited) (default 5000)
-llm-concurrency int maximum concurrent llm calls (default 4)
-llm-cache cache llm responses within a scan (default true)
FILTERING:
-a, -author string[] templates to run based on authors (comma-separated, file)
+8
View File
@@ -173,6 +173,14 @@ TEMPLATES:
-esc, -enable-self-contained enable loading self-contained templates
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
-llm-max-calls int maximum llm calls per scan (0 for unlimited) (default 5000)
-llm-concurrency int maximum concurrent llm calls (default 4)
-llm-cache cache llm responses within a scan (default true)
FILTERING:
-a, -author string[] templates to run based on authors (comma-separated, file)
+8
View File
@@ -173,6 +173,14 @@ TEMPLATES:
-esc, -enable-self-contained enable loading self-contained templates
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
-llm-max-calls int maximum llm calls per scan (0 for unlimited) (default 5000)
-llm-concurrency int maximum concurrent llm calls (default 4)
-llm-cache cache llm responses within a scan (default true)
FILTERING:
-a, -author string[] templates to run based on authors (comma-separated, file)
+8
View File
@@ -173,6 +173,14 @@ TEMPLATES:
-esc, -enable-self-contained enable loading self-contained templates
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
-llm-max-calls int maximum llm calls per scan (0 for unlimited) (default 5000)
-llm-concurrency int maximum concurrent llm calls (default 4)
-llm-cache cache llm responses within a scan (default true)
FILTERING:
-a, -author string[] templates to run based on authors (comma-separated, file)
+8
View File
@@ -173,6 +173,14 @@ TEMPLATES:
-esc, -enable-self-contained enable loading self-contained templates
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
-llm-max-calls int maximum llm calls per scan (0 for unlimited) (default 5000)
-llm-concurrency int maximum concurrent llm calls (default 4)
-llm-cache cache llm responses within a scan (default true)
FILTERING:
-a, -author string[] templates to run based on authors (comma-separated, file)
+8
View File
@@ -173,6 +173,14 @@ TEMPLATES:
-esc, -enable-self-contained enable loading self-contained templates
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
-llm-max-calls int maximum llm calls per scan (0 for unlimited) (default 5000)
-llm-concurrency int maximum concurrent llm calls (default 4)
-llm-cache cache llm responses within a scan (default true)
FILTERING:
-a, -author string[] templates to run based on authors (comma-separated, file)
+8
View File
@@ -173,6 +173,14 @@ TEMPLATES:
-esc, -enable-self-contained enable loading self-contained templates
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
-llm-max-calls int maximum llm calls per scan (0 for unlimited) (default 5000)
-llm-concurrency int maximum concurrent llm calls (default 4)
-llm-cache cache llm responses within a scan (default true)
FILTERING:
-a, -author string[] templates to run based on authors (comma-separated, file)
+8
View File
@@ -173,6 +173,14 @@ TEMPLATES:
-esc, -enable-self-contained enable loading self-contained templates
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
-llm-max-calls int maximum llm calls per scan (0 for unlimited) (default 5000)
-llm-concurrency int maximum concurrent llm calls (default 4)
-llm-cache cache llm responses within a scan (default true)
FILTERING:
-a, -author string[] templates to run based on authors (comma-separated, file)
+132
View File
@@ -2475,6 +2475,136 @@ xpath:
```
</div>
<hr />
<div class="dd">
<code>prompt</code> <i>string</i>
</div>
<div class="dt">
Prompt is the natural-language question for an llm matcher. The model
judges the selected response part and returns a verdict; the matcher
fires when the verdict equals Expect with at least MinConfidence.
{{...}} placeholders are interpolated from the template variables,
-var and target. Response derived values are not interpolated, since
the response is attacker controlled.
Examples:
```yaml
prompt: Is this a working admin login form rather than a marketing page?
```
</div>
<hr />
<div class="dd">
<code>expect</code> <i>string</i>
</div>
<div class="dt">
Expect is the verdict that counts as a match for an llm matcher.
Defaults to "yes".
</div>
<hr />
<div class="dd">
<code>options</code> <i>[]string</i>
</div>
<div class="dt">
Options restricts the model to a fixed set of verdicts for an llm
matcher. Defaults to yes/no.
</div>
<hr />
<div class="dd">
<code>min-confidence</code> <i>float64</i>
</div>
<div class="dt">
MinConfidence is the minimum confidence (0-1) an llm verdict needs to
count as a match. Defaults to 0.
</div>
<hr />
<div class="dd">
<code>inputs</code> <i>[]string</i>
</div>
<div class="dt">
Inputs compares several responses instead of a single part, for
checks that are about the difference between responses (blind
boolean injection, user enumeration, IDOR). Each entry is resolved
against the runtime values, so {{body_1}} and {{body_2}} work with
req-condition, and each response is framed separately for the model.
Part is ignored when Inputs is set.
Examples:
```yaml
inputs:
- '{{body_1}}'
- '{{body_2}}'
```
</div>
<hr />
<div class="dd">
<code>max-input-tokens</code> <i>int</i>
</div>
<div class="dt">
MaxInputTokens caps how much of the response part is sent to the model
for an llm matcher, as an approximate token count.
</div>
<hr />
<div class="dd">
<code>allow-sole</code> <i>bool</i>
</div>
<div class="dt">
AllowSole permits an llm matcher to be the only matcher on a high or
critical template. Without it such a template is rejected, so a model is
never the sole arbiter of a severe finding by accident.
</div>
<hr />
@@ -2600,6 +2730,8 @@ Enum Values:
- <code>dsl</code>
- <code>xpath</code>
- <code>llm</code>
</div>
<hr />
+10
View File
@@ -273,6 +273,8 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringVar(&options.Resume, "resume", "", "resume scan from and save to specified file (clustering will be disabled)"),
flagSet.BoolVarP(&options.ScanAllIPs, "scan-all-ips", "sa", false, "scan all the IP's associated with dns record"),
flagSet.StringSliceVarP(&options.IPVersion, "ip-version", "iv", nil, "IP version to scan of hostname (4,6) - (default 4)", goflags.CommaSeparatedStringSliceOptions),
flagSet.BoolVarP(&options.DisableURLNormalization, "disable-url-normalization", "dun", false, "keep targets exactly as supplied instead of collapsing urls that address the same resource"),
flagSet.IntVarP(&options.MaxURLsPerPattern, "max-urls-per-pattern", "mup", 0, "scan at most this many urls sharing one structural shape, e.g. /user/1 and /user/2 (0 to scan all)"),
)
flagSet.CreateGroup("target-format", "Target-Format",
@@ -304,6 +306,14 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.BoolVarP(&options.EnableSelfContainedTemplates, "enable-self-contained", "esc", false, "enable loading self-contained templates"),
flagSet.BoolVarP(&options.EnableGlobalMatchersTemplates, "enable-global-matchers", "egm", false, "enable loading global matchers templates"),
flagSet.BoolVar(&options.EnableFileTemplates, "file", false, "enable loading file templates"),
flagSet.BoolVar(&options.EnableLLM, "llm", false, "enable llm matchers and extractors (semantic matching)"),
flagSet.StringVar(&options.LLMProvider, "llm-provider", "openai", "llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together)"),
flagSet.StringVar(&options.LLMBaseURL, "llm-base-url", "", "openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)"),
flagSet.StringVar(&options.LLMModel, "llm-model", "", "model used for semantic matching"),
flagSet.IntVar(&options.LLMTimeout, "llm-timeout", 30, "time in seconds to wait for a single llm call"),
flagSet.IntVar(&options.LLMMaxCalls, "llm-max-calls", 5000, "maximum llm calls per scan (0 for unlimited)"),
flagSet.IntVar(&options.LLMConcurrency, "llm-concurrency", 4, "maximum concurrent llm calls"),
flagSet.BoolVar(&options.LLMCache, "llm-cache", true, "cache llm responses within a scan"),
)
flagSet.CreateGroup("filters", "Filtering",
+86 -92
View File
@@ -1,41 +1,6 @@
module github.com/projectdiscovery/nuclei/v3
go 1.26.0
require (
github.com/andygrunwald/go-jira v1.16.1
github.com/antchfx/htmlquery v1.3.5
github.com/go-playground/validator/v10 v10.26.0
github.com/go-rod/rod v0.116.2
github.com/gobwas/ws v1.4.0
github.com/invopop/jsonschema v0.13.0
github.com/itchyny/gojq v0.12.17
github.com/json-iterator/go v1.1.12 // indirect
github.com/julienschmidt/httprouter v1.3.0
github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
github.com/miekg/dns v1.1.73
github.com/olekukonko/tablewriter v1.0.8
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.1.7
github.com/projectdiscovery/fastdialer v0.5.20
github.com/projectdiscovery/hmap v0.0.102
github.com/projectdiscovery/interactsh v1.3.1
github.com/projectdiscovery/rawhttp v0.1.92
github.com/projectdiscovery/retryabledns v1.0.116
github.com/projectdiscovery/retryablehttp-go v1.3.26
github.com/projectdiscovery/yamldoc-go v1.0.7
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.6.0
github.com/segmentio/ksuid v1.0.4
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/cast v1.10.0
github.com/syndtr/goleveldb v1.0.0
github.com/weppos/publicsuffix-go v0.50.3 // indirect
go.uber.org/multierr v1.11.0
golang.org/x/net v0.58.0
golang.org/x/oauth2 v0.36.0
golang.org/x/text v0.41.0
)
go 1.27.1
require (
carvel.dev/ytt v0.52.0
@@ -48,11 +13,13 @@ require (
github.com/FalconOpsLLC/goexec v0.3.0
github.com/Masterminds/semver/v3 v3.5.0
github.com/Mzack9999/go-rsync v0.0.0-20250821180103-81ffa574ef4d
github.com/Mzack9999/goimpacket v0.0.0-20260422121140-7085336a0415
github.com/RedTeamPentesting/adauth v0.5.4-0.20260511073005-3d18e8a5a687
github.com/adrg/xdg v0.5.3
github.com/alecthomas/chroma v0.10.0
github.com/alexsnet/go-vnc v0.1.0
github.com/alitto/pond v1.9.2
github.com/andygrunwald/go-jira v1.16.1
github.com/antchfx/htmlquery v1.3.5
github.com/antchfx/xmlquery v1.4.4
github.com/antchfx/xpath v1.3.6
github.com/aws/aws-sdk-go-v2 v1.41.5
@@ -71,19 +38,28 @@ require (
github.com/fullstorydev/grpcurl v1.9.3
github.com/getkin/kin-openapi v0.144.0
github.com/getsops/sops/v3 v3.9.2
github.com/go-echarts/go-echarts/v2 v2.6.0
github.com/go-git/go-git/v5 v5.19.2
github.com/go-ldap/ldap/v3 v3.4.12
github.com/go-pdf/fpdf v0.9.0
github.com/go-pg/pg/v10 v10.15.0
github.com/go-playground/validator/v10 v10.26.0
github.com/go-rod/rod v0.116.2
github.com/go-sql-driver/mysql v1.10.0
github.com/gobwas/ws v1.4.0
github.com/goccy/go-json v0.10.5
github.com/google/go-github/v30 v30.1.0
github.com/google/go-github/v92 v92.0.0
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/google/uuid v1.6.0
github.com/graphql-go/graphql v0.8.1
github.com/h2non/filetype v1.1.3
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/hdm/jarm-go v0.0.8
github.com/invopop/jsonschema v0.13.0
github.com/itchyny/gojq v0.12.17
github.com/jcmturner/gokrb5/v8 v8.4.4
github.com/jhump/protoreflect v1.17.0
github.com/julienschmidt/httprouter v1.3.0
github.com/kitabisa/go-ci v1.0.3
github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa
github.com/lib/pq v1.12.3
@@ -92,45 +68,72 @@ require (
github.com/maypok86/otter/v2 v2.2.1
github.com/mholt/archives v0.1.5
github.com/microsoft/go-mssqldb v1.9.2
github.com/miekg/dns v1.1.73
github.com/oiweiwei/go-msrpc v1.2.12
github.com/olekukonko/tablewriter v1.0.8
github.com/ory/dockertest/v3 v3.12.0
github.com/pkg/errors v0.9.1
github.com/praetorian-inc/fingerprintx v1.1.15
github.com/projectdiscovery/dsl v0.8.23-0.20260910174914-8e1207b367b8
github.com/projectdiscovery/clistats v0.1.7
github.com/projectdiscovery/dsl v0.8.24
github.com/projectdiscovery/fastdialer v0.5.25
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/gcache v0.0.0-20241015120333-12546c6e3f4c
github.com/projectdiscovery/goflags v0.2.1
github.com/projectdiscovery/goja v0.0.0-20260618133720-acb73e419534
github.com/projectdiscovery/goimpacket v0.1.1-0.20261007205806-6042a8213eeb
github.com/projectdiscovery/goja v0.0.0-20261012075649-9b78f2e05b9c
github.com/projectdiscovery/goja_nodejs v0.0.0-20260618132410-8519f75f703d
github.com/projectdiscovery/gologger v1.1.73
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/govaluate v0.0.0-20260615100919-5ee2581bbf7e
github.com/projectdiscovery/gozero v0.1.1-0.20260530071156-fa1dad563d76
github.com/projectdiscovery/hmap v0.0.102
github.com/projectdiscovery/httpx v1.12.0
github.com/projectdiscovery/interactsh v1.4.1
github.com/projectdiscovery/mapcidr v1.1.97
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
github.com/projectdiscovery/networkpolicy v0.1.50
github.com/projectdiscovery/ratelimit v0.0.90-0.20260912164306-aeeca5645294
github.com/projectdiscovery/networkpolicy v0.1.54
github.com/projectdiscovery/ratelimit v0.0.93
github.com/projectdiscovery/rawhttp v0.1.92
github.com/projectdiscovery/rdap v0.9.0
github.com/projectdiscovery/retryabledns v1.0.116
github.com/projectdiscovery/retryablehttp-go v1.3.30
github.com/projectdiscovery/sarif v0.1.0
github.com/projectdiscovery/tlsx v1.4.0
github.com/projectdiscovery/tlsx v1.4.1
github.com/projectdiscovery/uncover v1.2.1
github.com/projectdiscovery/useragent v0.0.109
github.com/projectdiscovery/utils v0.11.4-0.20260910160917-83d6b6e62668
github.com/projectdiscovery/wappalyzergo v0.3.0
github.com/redis/go-redis/v9 v9.11.0
github.com/projectdiscovery/utils v0.11.9
github.com/projectdiscovery/wappalyzergo v0.3.4
github.com/projectdiscovery/yamldoc-go v1.0.7
github.com/redis/go-redis/v9 v9.19.0
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.6.0
github.com/rs/zerolog v1.34.0
github.com/sandrolain/httpcache v1.4.0
github.com/segmentio/ksuid v1.0.4
github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466
github.com/sijms/go-ora/v2 v2.9.0
github.com/spf13/cast v1.10.0
github.com/stretchr/testify v1.12.1
github.com/syndtr/goleveldb v1.0.0
github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
github.com/trivago/tgo v1.0.7
github.com/yassinebenaid/godump v0.11.1
github.com/zmap/zgrab2 v0.1.8
gitlab.com/gitlab-org/api/client-go v1.9.1
go.mongodb.org/mongo-driver v1.17.9
go.uber.org/multierr v1.11.0
golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3
golang.org/x/net v0.58.0
golang.org/x/oauth2 v0.36.0
golang.org/x/sync v0.22.0
golang.org/x/sys v0.47.0
golang.org/x/term v0.45.0
google.golang.org/grpc v1.83.1
golang.org/x/text v0.41.0
golang.org/x/time v0.15.0
golang.org/x/tools v0.49.0
google.golang.org/grpc v1.83.2
google.golang.org/protobuf v1.36.11
gopkg.in/yaml.v3 v3.0.1
moul.io/http2curl v1.0.0
)
@@ -149,7 +152,6 @@ require (
cloud.google.com/go/storage v1.56.0 // indirect
dario.cat/mergo v1.0.2 // indirect
filippo.io/edwards25519 v1.2.0 // indirect
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1 // indirect
@@ -162,6 +164,7 @@ require (
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
github.com/Mzack9999/goimpacket v0.0.0-20260420131935-a9fe473cda7d // indirect
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
github.com/ProtonMail/go-crypto v1.1.6 // indirect
github.com/PuerkitoBio/goquery v1.13.0 // indirect
@@ -204,8 +207,8 @@ require (
github.com/buger/jsonparser v1.1.2 // indirect
github.com/bytedance/gopkg v0.1.3 // indirect
github.com/bytedance/sonic/loader v0.5.2 // indirect
github.com/caddyserver/certmagic v0.25.0 // indirect
github.com/caddyserver/zerossl v0.1.3 // indirect
github.com/caddyserver/certmagic v0.25.3 // indirect
github.com/caddyserver/zerossl v0.1.5 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/censys/censys-sdk-go v0.19.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
@@ -216,6 +219,7 @@ require (
github.com/charmbracelet/x/exp/slice v0.0.0-20250908092851-c2208eb08494 // indirect
github.com/charmbracelet/x/term v0.2.1 // indirect
github.com/cheggaaa/pb/v3 v3.1.7 // indirect
github.com/chzyer/readline v1.5.1 // indirect
github.com/cloudflare/cfssl v1.6.5 // indirect
github.com/cloudflare/circl v1.6.3 // indirect
github.com/cloudwego/base64x v0.1.6 // indirect
@@ -227,7 +231,7 @@ require (
github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
github.com/cyphar/filepath-securejoin v0.6.1 // indirect
github.com/davidmz/go-pageant v1.0.2 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/djherbis/times v1.6.0 // indirect
github.com/dlclark/regexp2 v1.11.5 // indirect
@@ -238,6 +242,8 @@ require (
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/ebitengine/purego v0.10.0 // indirect
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
github.com/emersion/go-smtp v0.24.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
@@ -247,7 +253,7 @@ require (
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
github.com/gaissmai/bart v0.29.0 // indirect
github.com/gaissmai/bart v0.30.0 // indirect
github.com/geoffgarside/ber v1.2.0 // indirect
github.com/getsops/gopgagent v0.0.0-20240527072608-0c14999532fe // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
@@ -267,6 +273,10 @@ require (
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/goburrow/cache v0.1.4 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
@@ -275,6 +285,7 @@ require (
github.com/golang/snappy v1.0.0 // indirect
github.com/google/certificate-transparency-go v1.3.3 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/go-querystring v1.2.0 // indirect
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
@@ -296,7 +307,6 @@ require (
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/vault/api v1.15.0 // indirect
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect
github.com/hdm/jarm-go v0.0.8 // indirect
github.com/iangcarroll/cookiemonster v1.6.0 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/indece-official/go-ebcdic v1.2.0 // indirect
@@ -308,6 +318,7 @@ require (
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/k14s/starlark-go v0.0.0-20200720175618-3a5c849cc368 // indirect
github.com/kaiakz/ubuffer v0.0.0-20200803053910-dd1083087166 // indirect
github.com/kataras/jwt v0.1.10 // indirect
@@ -318,6 +329,7 @@ require (
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/libdns/libdns v1.1.1 // indirect
github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
github.com/lor00x/goldap v0.0.0-20240304151906-8d785c64d1c8 // indirect
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
@@ -326,7 +338,7 @@ require (
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.22 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mholt/acmez/v3 v3.1.3 // indirect
github.com/mholt/acmez/v3 v3.1.6 // indirect
github.com/microcosm-cc/bluemonday v1.0.27 // indirect
github.com/mikelolasagasti/xz v1.0.1 // indirect
github.com/minio/minlz v1.0.1 // indirect
@@ -363,22 +375,24 @@ require (
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/projectdiscovery/asnmap v1.1.1 // indirect
github.com/projectdiscovery/blackrock v0.0.2 // indirect
github.com/projectdiscovery/cdncheck v1.3.0 // indirect
github.com/projectdiscovery/cdncheck v1.3.1 // indirect
github.com/projectdiscovery/freeport v0.0.7 // indirect
github.com/projectdiscovery/jarm-go v0.0.0-20260910160638-430e2bae86f6 // indirect
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 // indirect
github.com/refraction-networking/utls v1.8.2 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
github.com/sashabaranov/go-openai v1.37.0 // indirect
github.com/sergi/go-diff v1.4.0 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/shirou/gopsutil/v4 v4.26.3 // indirect
github.com/sirupsen/logrus v1.9.4 // indirect
github.com/skeema/knownhosts v1.3.1 // indirect
github.com/sorairolake/lzip-go v0.3.8 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/spiffe/go-spiffe/v2 v2.7.0 // indirect
@@ -404,6 +418,7 @@ require (
github.com/vmihailenco/tagparser v0.1.2 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/vulncheck-oss/go-exploit v1.51.0 // indirect
github.com/weppos/publicsuffix-go v0.50.3 // indirect
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
@@ -415,11 +430,18 @@ require (
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
github.com/ysmood/fetchup v0.2.3 // indirect
github.com/ysmood/goob v0.4.0 // indirect
github.com/ysmood/got v0.40.0 // indirect
github.com/ysmood/gson v0.7.3 // indirect
github.com/ysmood/leakless v0.9.0 // indirect
github.com/yuin/goldmark v1.7.17 // indirect
github.com/yuin/goldmark-emoji v1.0.6 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zcalusic/sysinfo v1.1.3 // indirect
github.com/zeebo/blake3 v0.2.4 // indirect
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 // indirect
go.etcd.io/bbolt v1.4.3 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
@@ -429,54 +451,26 @@ require (
go.opentelemetry.io/otel/sdk v1.44.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/zap v1.27.1 // indirect
go.uber.org/zap/exp v0.3.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
goftp.io/server/v2 v2.0.1 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.55.0 // indirect
golang.org/x/mod v0.39.0 // indirect
google.golang.org/api v0.264.0 // indirect
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
mellium.im/sasl v0.3.2 // indirect
software.sslmate.com/src/go-pkcs12 v0.7.2 // indirect
)
require (
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/goburrow/cache v0.1.4 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
github.com/google/go-querystring v1.2.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
github.com/trivago/tgo v1.0.7
github.com/ysmood/goob v0.4.0 // indirect
github.com/ysmood/gson v0.7.3 // indirect
github.com/ysmood/leakless v0.9.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 // indirect
go.etcd.io/bbolt v1.4.3 // indirect
go.uber.org/zap v1.27.0 // indirect
goftp.io/server/v2 v2.0.1 // indirect
golang.org/x/crypto v0.55.0 // indirect
golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3
golang.org/x/mod v0.39.0 // indirect
golang.org/x/sys v0.47.0
golang.org/x/time v0.15.0 // indirect
golang.org/x/tools v0.49.0
google.golang.org/protobuf v1.36.11
gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
)
require (
github.com/alecthomas/chroma v0.10.0
github.com/go-echarts/go-echarts/v2 v2.6.0
gopkg.in/warnings.v0 v0.1.2 // indirect
)
// https://go.dev/ref/mod#go-mod-file-retract
retract v3.2.0 // retract due to broken js protocol issue
+65 -46
View File
@@ -64,6 +64,8 @@ cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U
cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s=
code.gitea.io/sdk/gitea v0.17.0 h1:8JPBss4+Jf7AE1YcfyiGrngTXE8dFSG3si/bypsTH34=
code.gitea.io/sdk/gitea v0.17.0/go.mod h1:ndkDk99BnfiUCCYEUhpNzi0lpmApXlwRFqClBlOlEBg=
code.pfad.fr/check v1.1.0 h1:GWvjdzhSEgHvEHe2uJujDcpmZoySKuHQNrZMfzfO0bE=
code.pfad.fr/check v1.1.0/go.mod h1:NiUH13DtYsb7xp5wll0U4SXx7KhXQVCtRgdC96IPfoM=
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
@@ -71,8 +73,6 @@ filippo.io/age v1.2.1 h1:X0TZjehAZylOIj4DubWYU1vWQxv9bJpo+Uu2/LGhi1o=
filippo.io/age v1.2.1/go.mod h1:JL9ew2lTN+Pyft4RiNGguFfOpewKwSHm5ayKD/A4004=
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8=
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4=
@@ -124,8 +124,8 @@ github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb888350
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4=
github.com/Mzack9999/go-rsync v0.0.0-20250821180103-81ffa574ef4d h1:DofPB5AcjTnOU538A/YD86/dfqSNTvQsAXgwagxmpu4=
github.com/Mzack9999/go-rsync v0.0.0-20250821180103-81ffa574ef4d/go.mod h1:uzdh/m6XQJI7qRvufeBPDa+lj5SVCJO8B9eLxTbtI5U=
github.com/Mzack9999/goimpacket v0.0.0-20260422121140-7085336a0415 h1:lpSZPcbowbxvKFaFvE1reLTCStezWXcRVk0zzBtUatg=
github.com/Mzack9999/goimpacket v0.0.0-20260422121140-7085336a0415/go.mod h1:Wvb2f1Aq6NVL4Fza/dPNxv6/canpeizpgmiTCBGMD50=
github.com/Mzack9999/goimpacket v0.0.0-20260420131935-a9fe473cda7d h1:xX93NZHxxrHcmN4nn1owvJdr1E/l1kp1Rv7rMr9ly+U=
github.com/Mzack9999/goimpacket v0.0.0-20260420131935-a9fe473cda7d/go.mod h1:Wvb2f1Aq6NVL4Fza/dPNxv6/canpeizpgmiTCBGMD50=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
@@ -169,6 +169,8 @@ github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktp
github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
github.com/alexsnet/go-vnc v0.1.0 h1:vBCwPNy79WEL8V/Z5A0ngEFCvTWBAjmS048lkR2rdmY=
github.com/alexsnet/go-vnc v0.1.0/go.mod h1:bbRsg41Sh3zvrnWsw+REKJVGZd8Of2+S0V1G0ZaBhlU=
github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e2uP9jiw=
github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
github.com/alitto/pond v1.9.2 h1:9Qb75z/scEZVCoSU+osVmQ0I0JOeLfdTDafrbcJ8CLs=
github.com/alitto/pond v1.9.2/go.mod h1:xQn3P/sHTYcU/1BR3i86IGIrilcrGC2LiS+E2+CJWsI=
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
@@ -272,10 +274,10 @@ github.com/bytedance/sonic v1.15.3 h1:P3akjLPBtV/i6bHC6LbcLjY3KuoOvfiqF8wFHeP5Ih
github.com/bytedance/sonic v1.15.3/go.mod h1:8e51yTPdY8M6t+vvGL1c2Y1xL9i+frEeIAQAEl75NUc=
github.com/bytedance/sonic/loader v0.5.2 h1:0QtP1gevc1OZ6/H8Lb9BRZiCXd1Ftjd3OKuj1T1lBIo=
github.com/bytedance/sonic/loader v0.5.2/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo=
github.com/caddyserver/certmagic v0.25.0 h1:VMleO/XA48gEWes5l+Fh6tRWo9bHkhwAEhx63i+F5ic=
github.com/caddyserver/certmagic v0.25.0/go.mod h1:m9yB7Mud24OQbPHOiipAoyKPn9pKHhpSJxXR1jydBxA=
github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA=
github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4=
github.com/caddyserver/certmagic v0.25.3 h1:mGf5ba8F7xA4c5jfDZZbK2buY1VEkbnwpMDixaju94A=
github.com/caddyserver/certmagic v0.25.3/go.mod h1:YVs43D5+H/Dckt4bTga1KSO/xYfFBfVZainGDywYPAA=
github.com/caddyserver/zerossl v0.1.5 h1:dkvOjBAEEtY6LIGAHei7sw2UgqSD6TrWweXpV7lvEvE=
github.com/caddyserver/zerossl v0.1.5/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -310,10 +312,13 @@ github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0V
github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs=
github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM=
github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI=
github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04=
github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8=
github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME=
github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s=
@@ -349,8 +354,6 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0=
github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
@@ -382,6 +385,10 @@ github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3DfFi4EXdT+B+gTtzx8GC9xsc26Znk=
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
github.com/emersion/go-smtp v0.24.0 h1:g6AfoF140mvW0vLNPD/LuCBLEAdlxOjIXqbIkJIS6Wk=
github.com/emersion/go-smtp v0.24.0/go.mod h1:ZtRRkbTyp2XTHCA+BmyTFTrj8xY4I+b4McvHxCU2gsQ=
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -419,8 +426,8 @@ github.com/fullstorydev/grpcurl v1.9.3 h1:PC1Xi3w+JAvEE2Tg2Gf2RfVgPbf9+tbuQr1Zky
github.com/fullstorydev/grpcurl v1.9.3/go.mod h1:/b4Wxe8bG6ndAjlfSUjwseQReUDUvBJiFEB7UllOlUE=
github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
github.com/gaissmai/bart v0.29.0 h1:wO6HGE8g9YE0Wm0bCpYxwRzfQ4+fbJKOhL64e5ACGCI=
github.com/gaissmai/bart v0.29.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
github.com/gaissmai/bart v0.30.0 h1:K1oTSQ6Nf76bhrgxeTqzZa4hE0M9x4xDfL6n6UG67Sc=
github.com/gaissmai/bart v0.30.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
github.com/getkin/kin-openapi v0.144.0 h1:hIRcTH+KjLfkLpYU6bSSfdFpi0fZi1fp+hSPi4aQu9Y=
@@ -585,9 +592,9 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo=
github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8=
github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q=
github.com/google/go-github/v92 v92.0.0 h1:4vW4RVffwIvoEfIA4RX09mRSB47qQX557/2+HgeWRRg=
github.com/google/go-github/v92 v92.0.0/go.mod h1:w3CH62ZcmRfvW1cdXpyTztSOVMtdjxtKpgo0GouLmjY=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0=
@@ -639,6 +646,8 @@ github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6
github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc=
github.com/goware/prefixer v0.0.0-20160118172347-395022866408 h1:Y9iQJfEqnN3/Nce9cOegemcy/9Ai5k3huT6E80F3zaw=
github.com/goware/prefixer v0.0.0-20160118172347-395022866408/go.mod h1:PE1ycukgRPJ7bJ9a1fdfQ9j8i/cEcRAoLZzbxYpNB/s=
github.com/graphql-go/graphql v0.8.1 h1:p7/Ou/WpmulocJeEx7wjQy611rtXGQaAcXGqanuMMgc=
github.com/graphql-go/graphql v0.8.1/go.mod h1:nKiHzRM0qopJEwCITUuIsxk9PlVlwIiiI8pnJEhordQ=
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -773,6 +782,10 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa h1:KQKuQDgA3DZX6C396lt3WDYB9Um1gLITLbvficVbqXk=
github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa/go.mod h1:HbwNE4XGwjgtUELkvQaAOjWrpianHYZdQVNqSdYW3UM=
github.com/letsencrypt/challtestsrv v1.4.2 h1:0ON3ldMhZyWlfVNYYpFuWRTmZNnyfiL9Hh5YzC3JVwU=
github.com/letsencrypt/challtestsrv v1.4.2/go.mod h1:GhqMqcSoeGpYd5zX5TgwA6er/1MbWzx/o7yuuVya+Wk=
github.com/letsencrypt/pebble/v2 v2.10.0 h1:Wq6gYXlsY6ubqI3hhxsTzdyotvfdjFBxuwYqCLCnj/U=
github.com/letsencrypt/pebble/v2 v2.10.0/go.mod h1:Sk8cmUIPcIdv2nINo+9PB4L+ZBhzY+F9A1a/h/xmWiQ=
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
github.com/libdns/libdns v1.1.1 h1:wPrHrXILoSHKWJKGd0EiAVmiJbFShguILTg9leS/P/U=
@@ -808,8 +821,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/maypok86/otter/v2 v2.2.1 h1:hnGssisMFkdisYcvQ8L019zpYQcdtPse+g0ps2i7cfI=
github.com/maypok86/otter/v2 v2.2.1/go.mod h1:1NKY9bY+kB5jwCXBJfE59u+zAwOt6C7ni1FTlFFMqVs=
github.com/mholt/acmez/v3 v3.1.3 h1:gUl789rjbJSuM5hYzOFnNaGgWPV1xVfnOs59o0dZEcc=
github.com/mholt/acmez/v3 v3.1.3/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0DFxagQ=
github.com/mholt/acmez/v3 v3.1.6 h1:eGVQNObP0pBN4sxqrXeg7MYqTOWyoiYpQqITVWlrevk=
github.com/mholt/acmez/v3 v3.1.6/go.mod h1:5nTPosTGosLxF3+LU4ygbgMRFDhbAVpqMI4+a4aHLBY=
github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ=
github.com/mholt/archives v0.1.5/go.mod h1:3TPMmBLPsgszL+1As5zECTuKwKvIfj6YcwWPpeTAXF4=
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
@@ -922,14 +935,14 @@ github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kI
github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60=
github.com/projectdiscovery/blackrock v0.0.2 h1:mxXdu0uM8P2L2Qi210COlU8QiICPFW/Rxk5QUhlPO2k=
github.com/projectdiscovery/blackrock v0.0.2/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
github.com/projectdiscovery/cdncheck v1.3.0 h1:/bWdNXIU1P5ASZuW8QA+tAw1FNHhmWlWXk/qiUHyx2c=
github.com/projectdiscovery/cdncheck v1.3.0/go.mod h1:GIAA0SqjDfxzDQjHR/X+bK05BGmUGfsVplP5Zdv/kHA=
github.com/projectdiscovery/cdncheck v1.3.1 h1:+qPKwqdf3mFtrw3ZoVti7hembmVaapnJBjRosYTNdRo=
github.com/projectdiscovery/cdncheck v1.3.1/go.mod h1:5IkUMMlcGaeg1DWW5oJVd3r9X3EpJ88j5HxBeF4Me9M=
github.com/projectdiscovery/clistats v0.1.7 h1:NXf4lUzx1QcbJd6QycNwwYHTv2FsW9MchQ2LsgSL6C0=
github.com/projectdiscovery/clistats v0.1.7/go.mod h1:RxRGxhMzkT344VTTvZWIrojurUXdQQRN1yfwgg/88I8=
github.com/projectdiscovery/dsl v0.8.23-0.20260910174914-8e1207b367b8 h1:zaAfFHjyLPq9Aoii1mXPzXkITvXyPJTSZo6nNhzC1xI=
github.com/projectdiscovery/dsl v0.8.23-0.20260910174914-8e1207b367b8/go.mod h1:TYqDNtD4hU8bGFgVxPqhYQ9K1z5rMLV/YFyvQgcU5TM=
github.com/projectdiscovery/fastdialer v0.5.20 h1:MlXoVXQERJY4ILqMNYfuoTKVR4Ix4XonJLkFaoHzgM0=
github.com/projectdiscovery/fastdialer v0.5.20/go.mod h1:zdtIbBt5ELNygMEPDSyQ9Nr9KpP9FuAvJwzNIrZZoJA=
github.com/projectdiscovery/dsl v0.8.24 h1:rN2w6LSIjA16qKGRjyj70oxdwXGaspe6sAjEkP7F2fk=
github.com/projectdiscovery/dsl v0.8.24/go.mod h1:V+EludvS5PS19UNMlM3CuBwZ+sFnMaQhCFGxysDUanQ=
github.com/projectdiscovery/fastdialer v0.5.25 h1:xGFpc5B7cmogqnBrcVYV5yXh6UFlqRA6u5KqautKgD0=
github.com/projectdiscovery/fastdialer v0.5.25/go.mod h1:CZZ0PwdTw2KCI/PrZTW9wyBY88MIjpT8WhsKWSw3bkw=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk=
@@ -938,8 +951,10 @@ github.com/projectdiscovery/gcache v0.0.0-20241015120333-12546c6e3f4c h1:s+lLAlr
github.com/projectdiscovery/gcache v0.0.0-20241015120333-12546c6e3f4c/go.mod h1:rN35/D3lVx2YDeENFFz06uj8j3XIqK1Ym9XcISF5fzg=
github.com/projectdiscovery/goflags v0.2.1 h1:XnSYvsCjUwHbZaaJUZ3YznCJPoXRk/IhTFP9Gecmx6M=
github.com/projectdiscovery/goflags v0.2.1/go.mod h1:7nAP1r2Dqgn/rwmOE3EWbZWUCEJKNIhVSBGpuzJAIns=
github.com/projectdiscovery/goja v0.0.0-20260618133720-acb73e419534 h1:hYd1zQA/dxO2ASyQ6Re73TcJkW1LjLQvt4+86Hxefz8=
github.com/projectdiscovery/goja v0.0.0-20260618133720-acb73e419534/go.mod h1:SO0AP+uKfYeeoR6jyVH/PKRYJE/f5gJrPMAM00iGEMc=
github.com/projectdiscovery/goimpacket v0.1.1-0.20261007205806-6042a8213eeb h1:+L1yW2PblRHSSHQRTmvxLkGDPZnM52N62oeBd2xHKF0=
github.com/projectdiscovery/goimpacket v0.1.1-0.20261007205806-6042a8213eeb/go.mod h1:bfgULaYf+6pjjcLCekpXIYElnLJbmhvTttxMAbQJvhY=
github.com/projectdiscovery/goja v0.0.0-20261012075649-9b78f2e05b9c h1:4H/CfVNlue9PIwsCM1IGxOynDI7cir+emXuDvGJTLJs=
github.com/projectdiscovery/goja v0.0.0-20261012075649-9b78f2e05b9c/go.mod h1:SO0AP+uKfYeeoR6jyVH/PKRYJE/f5gJrPMAM00iGEMc=
github.com/projectdiscovery/goja_nodejs v0.0.0-20260618132410-8519f75f703d h1:fqqH9LHpN2WDz9QuxFrhKNxXSRtzk+Sa6jAhbB7tXcQ=
github.com/projectdiscovery/goja_nodejs v0.0.0-20260618132410-8519f75f703d/go.mod h1:Ezmbgdaw4EunGGBU4MQViLoGMJc37LA3ip55YV3KeRI=
github.com/projectdiscovery/gologger v1.1.73 h1:WX2nzq29cL7HdvOHqIttacoid/5lMwapyM7ukGU1jFY=
@@ -954,8 +969,8 @@ github.com/projectdiscovery/hmap v0.0.102 h1:ybPuf7UQvuOLiHLOUbVMvd+PSlItQU5iQpP
github.com/projectdiscovery/hmap v0.0.102/go.mod h1:IZ92wXFmxYpdn0euZ7VFmypzOBNDwq8c3yRyIIol6XE=
github.com/projectdiscovery/httpx v1.12.0 h1:NN5iDrw7NjPBDlACdjac4lePTQmZ5aOmgOzptMpP44c=
github.com/projectdiscovery/httpx v1.12.0/go.mod h1:OgtyibNGdu9BrJxRdNKm280Davw1Z4s2FAhfPNNtLB4=
github.com/projectdiscovery/interactsh v1.3.1 h1:5HzeVGVCAX/cjTguJ+7ClOmML5r97Ty7op9s+/F7BiM=
github.com/projectdiscovery/interactsh v1.3.1/go.mod h1:MXQ11EoBPROb4bEw+WP9e4DX4fMhrpS6EwfMfZomBsw=
github.com/projectdiscovery/interactsh v1.4.1 h1:EGHPJ1rnuxvILWfcoaRgb/Ykz67WiDXABRCSJN666GA=
github.com/projectdiscovery/interactsh v1.4.1/go.mod h1:5g207F6MvIydlKWo4ZzplXkhaCwSq2veuUTNvEFjqps=
github.com/projectdiscovery/jarm-go v0.0.0-20260910160638-430e2bae86f6 h1:GUHQYLllWhoca+C4UAT0+TTB1PrUT05klj39V6/KpK8=
github.com/projectdiscovery/jarm-go v0.0.0-20260910160638-430e2bae86f6/go.mod h1:f8P5X5+jEkUqK9z3oNWtSH1g23jMfw3nuEqmtA1jXA4=
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8=
@@ -966,32 +981,32 @@ github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8Qu
github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA=
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
github.com/projectdiscovery/networkpolicy v0.1.50 h1:z/ZMuNAtPH6Q7BlMZ4DLA1ytm+7TYPAdBCNLjNqaV0U=
github.com/projectdiscovery/networkpolicy v0.1.50/go.mod h1:QAAMLSm1YsbGeZTFNGBMtyob28B1VUVblNT56wPhDJc=
github.com/projectdiscovery/ratelimit v0.0.90-0.20260912164306-aeeca5645294 h1:d5fkf/st46mg+QpOkzOuFBBCYfV8ljO3pmi8CChwIyE=
github.com/projectdiscovery/ratelimit v0.0.90-0.20260912164306-aeeca5645294/go.mod h1:pQg2urKomS0WkMmLCF5HhWyuresBXosh1kIfn1Tlwug=
github.com/projectdiscovery/networkpolicy v0.1.54 h1:+v7yyuRBu2S7dzeIJToDENy1i+HjRhNzqnQrDlm00z8=
github.com/projectdiscovery/networkpolicy v0.1.54/go.mod h1:tN+YAFVlJjHv/znPEhkcSAjmBMkMhslC9XpWKTrKEHw=
github.com/projectdiscovery/ratelimit v0.0.93 h1:k2rRpCR31DqeqrORSgGzJ+z0T7+QGCt0Z0+Ov/TVrgk=
github.com/projectdiscovery/ratelimit v0.0.93/go.mod h1:Tqpx1QKdFI63L01T2gpSvV5Uro9CnZkZWr80ciSXSBY=
github.com/projectdiscovery/rawhttp v0.1.92 h1:38GRsVLNllES/dZcEcA6s6P+rYBp34H9mDphMJMDvvg=
github.com/projectdiscovery/rawhttp v0.1.92/go.mod h1:spfXTPjxCj+bfWLynMVB7uOb+LS3PDNAI9HexMEV1Zg=
github.com/projectdiscovery/rdap v0.9.0 h1:wPhHx5pQ2QI+WGhyNb2PjhTl0NtB39Nk7YFZ9cp8ZGA=
github.com/projectdiscovery/rdap v0.9.0/go.mod h1:zk4yrJFQ2Hy36Aqk+DvotYQxYAeALaCJ5ORySkff36Q=
github.com/projectdiscovery/retryabledns v1.0.116 h1:tK28PJ4tLO7bgM89RpIFjkfTPMSbRegwvg9mN/q6z78=
github.com/projectdiscovery/retryabledns v1.0.116/go.mod h1:r3vH0AKYtJ9fculcqRFrdVYe2ABcHwZeR4C6j3rqCbY=
github.com/projectdiscovery/retryablehttp-go v1.3.26 h1:BkVqMUIVPhx+3vOoo5CxYGNnMqQQYXOedu89XQB1KTg=
github.com/projectdiscovery/retryablehttp-go v1.3.26/go.mod h1:poffUOX+jvS0XbaPCYess221jXvSO7R/QlE7zhs3+9Q=
github.com/projectdiscovery/retryablehttp-go v1.3.30 h1:YnDCZP04/AKTiIBT5vA1nVhy5cRW5dPy1Du5nCjxHy4=
github.com/projectdiscovery/retryablehttp-go v1.3.30/go.mod h1:MXn1MzThNmHc05KldUbfks1MvwAnUKR4GGJcFhVIc08=
github.com/projectdiscovery/sarif v0.1.0 h1:O541T+a448nSJsmIMnXXSOeDQEzpnCAYvRfe0eG5h74=
github.com/projectdiscovery/sarif v0.1.0/go.mod h1:LBC+reM3bkI3qIIhE0rZaINaYX6VG+En6u2hHa5mA7E=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
github.com/projectdiscovery/tlsx v1.4.0 h1:OU2mNK6TOB/YiFX3jhmgwOepFWxPoj5rn8npWiCCz74=
github.com/projectdiscovery/tlsx v1.4.0/go.mod h1:WCrUXAHjdKbDNjsu5EbVBS7qvzap3fxi/E85v/Flyx8=
github.com/projectdiscovery/tlsx v1.4.1 h1:QuECHVJERBirNrmpjgIOAFQScjy/DxVjzXeL+L81mH0=
github.com/projectdiscovery/tlsx v1.4.1/go.mod h1:WCrUXAHjdKbDNjsu5EbVBS7qvzap3fxi/E85v/Flyx8=
github.com/projectdiscovery/uncover v1.2.1 h1:8U46T/96CLT7BPoXBgkTvWqB06lOyeTSLvh5+UjzATE=
github.com/projectdiscovery/uncover v1.2.1/go.mod h1:0p8onrWxfpXQEYs90ZDzTSpu1107gWmodX1NWqu/+z4=
github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE=
github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk=
github.com/projectdiscovery/utils v0.11.4-0.20260910160917-83d6b6e62668 h1:umwLd2H4q5hfHPdAHSQWdrmBvECZs2qsF9hskvo1jw8=
github.com/projectdiscovery/utils v0.11.4-0.20260910160917-83d6b6e62668/go.mod h1:DzaUDLQ+Tqsn2nOL47M1mRBZSVyCmwsAlR2E9dxHB4Y=
github.com/projectdiscovery/wappalyzergo v0.3.0 h1:2AsEkjnKgFZK4tjaiNhwC0zfj04rpckSvIOZINeTGyQ=
github.com/projectdiscovery/wappalyzergo v0.3.0/go.mod h1:6xTRCUQhX8m+L1eqJDQIALJlRv2Bnhrmku80YU9bBXE=
github.com/projectdiscovery/utils v0.11.9 h1:aKLje3zPcZuOGmopN51PasWxPX/CypPwzP2ke/Qa2bA=
github.com/projectdiscovery/utils v0.11.9/go.mod h1:d3BcbsHhuxW9T3jpvrgzwshvqhaL9qPSkoi+3h/DY8E=
github.com/projectdiscovery/wappalyzergo v0.3.4 h1:yO+xel+C0d/vOFWcUKVZRpIDAfcEmsjpVDjQ0jifAUw=
github.com/projectdiscovery/wappalyzergo v0.3.4/go.mod h1:6xTRCUQhX8m+L1eqJDQIALJlRv2Bnhrmku80YU9bBXE=
github.com/projectdiscovery/yamldoc-go v1.0.7 h1:Odmjd1o4EkMff2F6QHi1dIYPIQPZUFy24YqmQCprmTk=
github.com/projectdiscovery/yamldoc-go v1.0.7/go.mod h1:nqD1iPMcofpRA1DbXavPByJhWURrshT+l8WN9XjyQag=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -1018,8 +1033,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
github.com/redis/go-redis/v9 v9.11.0 h1:E3S08Gl/nJNn5vkxd2i78wZxWAPNZgUNTp8WIJUAiIs=
github.com/redis/go-redis/v9 v9.11.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k=
github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo=
github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM=
github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
@@ -1046,8 +1061,6 @@ github.com/sandrolain/httpcache v1.4.0 h1:Jf4Vx62X2ybvNPSpPvI1kT3xvMdDG1AsApQjOQ
github.com/sandrolain/httpcache v1.4.0/go.mod h1:kHBuXveitSn39SNPBhdf/ybG272X706HJ2RJqOQ+Em0=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
github.com/sashabaranov/go-openai v1.37.0 h1:hQQowgYm4OXJ1Z/wTrE+XZaO20BYsL0R3uRPSpfNZkY=
github.com/sashabaranov/go-openai v1.37.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c=
github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
@@ -1231,6 +1244,8 @@ github.com/yuin/goldmark v1.7.17 h1:p36OVWwRb246iHxA/U4p8OPEpOTESm4n+g+8t0EE5uA=
github.com/yuin/goldmark v1.7.17/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
github.com/yuin/goldmark-emoji v1.0.6 h1:QWfF2FYaXwL74tfGOW5izeiZepUDroDJfWubQI9HTHs=
github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zcalusic/sysinfo v1.1.3 h1:u/AVENkuoikKuIZ4sUEJ6iibpmQP6YpGD8SSMCrqAF0=
@@ -1241,6 +1256,8 @@ github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI=
github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE=
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30=
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
@@ -1288,12 +1305,14 @@ go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRk
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U=
go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
@@ -1682,8 +1701,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y=
google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+1 -1
View File
@@ -1,6 +1,6 @@
module nuclei-tools
go 1.26.1
go 1.27.1
tool (
github.com/dvyukov/go-fuzz/go-fuzz
+56
View File
@@ -0,0 +1,56 @@
package runner
import (
"context"
"time"
llmclient "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/llm"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
utilsllm "github.com/projectdiscovery/utils/llm"
)
// llmMatcherClient adapts the shared provider layer to the minimal interface
// the matchers package needs, keeping matchers free of any provider SDK import.
type llmMatcherClient struct {
client *utilsllm.Client
model string
}
// Model names the model behind a verdict so it can be recorded in the finding.
func (a *llmMatcherClient) Model() string {
return a.model
}
func (a *llmMatcherClient) Complete(ctx context.Context, prompt string, asJSON bool) (string, error) {
return a.client.Complete(ctx, utilsllm.Request{
Prompt: prompt,
Format: utilsllm.Format{JSON: asJSON},
})
}
// configureLLM builds the scan's llm client from options. It returns nil unless
// -llm is set, so nothing reaches a model until the user opts in. The API key is
// read from the environment by the provider layer, never from options.
//
// The client is returned rather than installed globally so two scans in one
// process cannot overwrite each other's provider, cache, budget, or timeout.
func configureLLM(options *types.Options) (llmclient.Client, error) {
if !options.EnableLLM {
return nil, nil
}
client, err := utilsllm.New(utilsllm.Config{
Provider: options.LLMProvider,
BaseURL: options.LLMBaseURL,
Model: options.LLMModel,
Timeout: time.Duration(options.LLMTimeout) * time.Second,
Cache: options.LLMCache,
MaxCalls: options.LLMMaxCalls,
MaxConcurrency: options.LLMConcurrency,
})
if err != nil {
return nil, err
}
return &llmMatcherClient{client: client, model: options.LLMModel}, nil
}
+21
View File
@@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
"reflect"
"slices"
"sort"
"strings"
"sync/atomic"
@@ -46,6 +47,7 @@ import (
fuzzStats "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats"
"github.com/projectdiscovery/nuclei/v3/pkg/input"
parsers "github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow"
llmclient "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/llm"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
"github.com/projectdiscovery/nuclei/v3/pkg/projectfile"
@@ -109,6 +111,7 @@ type Runner struct {
httpApiEndpoint *httpapi.Server
fuzzStats *fuzzStats.Tracker
dastServer *server.DASTServer
llmClient llmclient.Client
}
// New creates a new client for running the enumeration process.
@@ -122,6 +125,12 @@ func New(options *types.Options) (*Runner, error) {
return nil, fmt.Errorf("initialize nuclei configuration: %w", err)
}
llmClient, err := configureLLM(options)
if err != nil {
return nil, fmt.Errorf("configure llm: %w", err)
}
runner.llmClient = llmClient
if options.HealthCheck {
runner.Logger.Print().Msgf("%s\n", DoHealthCheck(options))
os.Exit(0)
@@ -657,6 +666,7 @@ func (r *Runner) RunEnumeration() error {
executorOpts := &protocols.ExecutorOptions{
Output: r.output,
Options: r.options,
LLMClient: r.llmClient,
Progress: r.progress,
Catalog: r.catalog,
IssuesClient: r.issuesClient,
@@ -728,6 +738,13 @@ func (r *Runner) RunEnumeration() error {
// If using input-file flags, only load http fuzzing based templates.
loaderConfig := loader.NewConfig(r.options, r.catalog, executorOpts)
profiles := r.targetProfiles()
if profiles != nil {
if r.options.AutomaticScan {
return errors.New("per-target profiles cannot be combined with automatic scan")
}
loaderConfig.TargetFilter = profiles.LoadFilter()
}
if (!strings.EqualFold(r.options.InputFileMode, "list") || r.options.DAST) && !r.options.OfflineHTTP {
// if input type is not list (implicitly enable fuzzing), unless passive/offlinehttp
r.options.DAST = true
@@ -772,6 +789,10 @@ func (r *Runner) RunEnumeration() error {
if err := store.Load(); err != nil {
return err
}
if profiles != nil {
profiles.Prepare(slices.Concat(store.Templates(), store.Workflows()))
executorOpts.TargetScope = profiles
}
// TODO: remove below functions after v3 or update warning messages
templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
+17
View File
@@ -0,0 +1,17 @@
package runner
import (
"github.com/projectdiscovery/nuclei/v3/pkg/input/targetprofile"
)
// targetProfiles returns the per-target profiles read from input, or nil when
// no target has one.
func (r *Runner) targetProfiles() *targetprofile.Registry {
provider, ok := r.inputProvider.(interface {
TargetProfiles() *targetprofile.Registry
})
if !ok || !provider.TargetProfiles().Scoped() {
return nil
}
return provider.TargetProfiles()
}
@@ -5,6 +5,7 @@ package functional_test
import (
"bufio"
"errors"
"fmt"
"os"
"os/exec"
@@ -280,6 +281,12 @@ func trimMatchingQuotes(value string) string {
return value
}
func TestIsRetryableFunctionalError(t *testing.T) {
if isRetryableFunctionalError(nil) || isRetryableFunctionalError(fmt.Errorf("load mismatch")) {
t.Fatal("non-exit errors should not retry")
}
}
func TestSplitFunctionalArgs(t *testing.T) {
t.Run("quoted values", func(t *testing.T) {
args := splitFunctionalArgs(`{{binary}} -tags "cve,exposure" -author "geeknik,pdteam" -severity high,critical`)
@@ -338,6 +345,29 @@ func runFunctionalCase(args []string, debug bool) error {
return fmt.Errorf("functional runners are not initialized")
}
const attempts = 2
var last error
for attempt := 1; attempt <= attempts; attempt++ {
last = compareFunctionalLoad(args, debug)
if last == nil {
return nil
}
if !isRetryableFunctionalError(last) {
return last
}
}
return last
}
func isRetryableFunctionalError(err error) bool {
if err == nil {
return false
}
var exitErr *exec.ExitError
return errors.As(err, &exitErr)
}
func compareFunctionalLoad(args []string, debug bool) error {
var (
releaseOutput string
currentOutput string
+26 -6
View File
@@ -73,17 +73,27 @@ func TestMain(m *testing.M) {
templatesDir := filepath.Join(homeDir, config.NucleiTemplatesDirName)
config.DefaultConfig.SetTemplatesDir(templatesDir)
baseEnv := []string{
"NUCLEI_TEMPLATES_DIR=" + templatesDir,
"XDG_CONFIG_HOME=" + filepath.Join(tempDir, "config"),
"XDG_STATE_HOME=" + filepath.Join(tempDir, "state"),
"XDG_CACHE_HOME=" + filepath.Join(tempDir, "cache"),
}
// cases run nuclei with -duc, so the official templates some of them read
// are installed once here rather than by whichever cases start first, which
// queued them behind each other's downloads inside their own timeouts
if err := installTemplates(binaryPath, baseEnv); err != nil {
fmt.Fprintln(os.Stderr, err)
_ = os.RemoveAll(tempDir)
os.Exit(1)
}
previousRunner := testutils.DefaultRunner()
runner := testutils.NewRunner(
testutils.WithBinaryPath(binaryPath),
testutils.WithWorkingDir(workingFixturesDir),
testutils.WithExtraArgs(integrationExtraArgs...),
testutils.WithBaseEnv(
"NUCLEI_TEMPLATES_DIR="+templatesDir,
"XDG_CONFIG_HOME="+filepath.Join(tempDir, "config"),
"XDG_STATE_HOME="+filepath.Join(tempDir, "state"),
"XDG_CACHE_HOME="+filepath.Join(tempDir, "cache"),
),
testutils.WithBaseEnv(baseEnv...),
)
testutils.SetDefaultRunner(runner)
@@ -124,6 +134,16 @@ func buildNucleiBinary(repoRoot, binaryPath string) error {
return nil
}
func installTemplates(binaryPath string, env []string) error {
cmd := exec.Command(binaryPath, "-update-templates", "-no-color")
cmd.Env = append(os.Environ(), env...)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("failed to install nuclei templates: %w\n%s", err, strings.TrimSpace(string(output)))
}
return nil
}
func isDebugMode() bool {
for _, envVar := range []string{"DEBUG", "ACTIONS_STEP_DEBUG", "ACTIONS_RUNNER_DEBUG", "RUNNER_DEBUG"} {
if envTruthy(envVar) {
@@ -37,6 +37,8 @@ var jsTestcases = []integrationCase{
{Path: "protocols/javascript/net-https.yaml", TestCase: &javascriptNetHttps{}},
{Path: "protocols/javascript/grpc-health.yaml", TestCase: &javascriptGRPCHealth{}},
{Path: "protocols/javascript/grpc-denied.yaml", TestCase: &javascriptGRPCDenied{}},
{Path: "protocols/javascript/websocket-echo.yaml", TestCase: &javascriptWebSocketEcho{}},
{Path: "protocols/javascript/websocket-denied.yaml", TestCase: &javascriptWebSocketDenied{}},
{Path: "protocols/javascript/http-get.yaml", TestCase: &javascriptHTTPGet{}},
{Path: "protocols/javascript/http-client-flow.yaml", TestCase: &javascriptHTTPClientFlow{}},
{Path: "protocols/javascript/http-denied.yaml", TestCase: &javascriptHTTPDenied{}},
@@ -0,0 +1,74 @@
//go:build integration
// +build integration
package integration_test
import (
"fmt"
"net"
"net/http"
"github.com/gobwas/ws"
"github.com/gobwas/ws/wsutil"
)
// startWebSocketEchoServer starts a WebSocket server on a random loopback port
// that accepts the "chat" subprotocol, greets, then echoes every message.
func startWebSocketEchoServer() (string, func(), error) {
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return "", nil, fmt.Errorf("websocket listen: %w", err)
}
srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
upgrader := ws.HTTPUpgrader{Protocol: func(protocol string) bool { return protocol == "chat" }}
conn, _, _, err := upgrader.Upgrade(r, w)
if err != nil {
return
}
defer func() { _ = conn.Close() }()
if err := wsutil.WriteServerText(conn, []byte("welcome")); err != nil {
return
}
for {
payload, op, err := wsutil.ReadClientData(conn)
if err != nil {
return
}
if err := wsutil.WriteServerMessage(conn, op, payload); err != nil {
return
}
}
})}
go func() { _ = srv.Serve(lis) }()
return lis.Addr().String(), func() { _ = srv.Close() }, nil
}
// javascriptWebSocketEcho exercises the nuclei/websocket library end to end
// against a real WebSocket server.
type javascriptWebSocketEcho struct{}
func (j *javascriptWebSocketEcho) Execute(filePath string) error {
address, stop, err := startWebSocketEchoServer()
if err != nil {
return err
}
defer stop()
results, err := runSignedNucleiTemplateAndGetResults(filePath, address, debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
// javascriptWebSocketDenied asserts the nuclei/websocket library refuses a host
// on the exclude list (network-policy enforcement) before connecting.
type javascriptWebSocketDenied struct{}
func (j *javascriptWebSocketDenied) Execute(filePath string) error {
results, err := runSignedNucleiTemplateAndGetResults(filePath, "127.0.0.1", debug, "-eh", "203.0.113.10")
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
@@ -0,0 +1,34 @@
id: websocket-denied
info:
name: WebSocket Network Policy Denial
author: pdteam
severity: info
description: |
Asserts the nuclei/websocket javascript library refuses a host on the
exclude list before connecting.
javascript:
- code: |
const websocket = require('nuclei/websocket');
let out = "";
try {
const client = new websocket.Client('ws://203.0.113.10/socket');
client.Connect();
} catch (e) {
out = "" + e;
}
out;
args:
Host: "{{Host}}"
matchers-condition: and
matchers:
- type: dsl
dsl:
- "success == true"
- type: word
part: response
words:
- "network policy"
- "203.0.113.10"
condition: and
@@ -0,0 +1,33 @@
id: websocket-echo
info:
name: WebSocket Echo
author: pdteam
severity: info
description: |
Exercises the nuclei/websocket javascript library end to end: dial through
the network policy, negotiate a subprotocol, read the server greeting and
round-trip a text message.
javascript:
- code: |
const websocket = require('nuclei/websocket');
const opts = new websocket.Options();
opts.Protocols = ['chat'];
const client = new websocket.Client('ws://' + address + '/echo', opts);
const greeting = client.Receive();
client.Send('nuclei-websocket');
const echo = client.Receive();
client.Close();
greeting + ' ' + client.Protocol + ' ' + echo;
args:
address: "{{Host}}:{{Port}}"
matchers-condition: and
matchers:
- type: dsl
dsl:
- "success == true"
- type: word
part: response
words:
- "welcome chat nuclei-websocket"
+7
View File
@@ -261,6 +261,9 @@ func (r *Runner) ArgsResults(debug bool, args ...string) ([]string, error) {
output, err := r.runCommand(false, "", func(cmd *exec.Cmd) {
cmd.Args = append(cmd.Args[:1], append(append([]string{}, args...), r.ExtraArgs...)...)
cmd.Env = r.buildEnv(nil)
if r.DisableAutoUpdate {
cmd.Args = append(cmd.Args, "-duc")
}
if debug {
cmd.Args = append(cmd.Args, "-debug")
cmd.Stderr = os.Stderr
@@ -324,6 +327,10 @@ func (r *Runner) LoadedTemplates(binaryPath string, debug bool, args []string) (
}
if err != nil {
out := strings.TrimSpace(string(data))
if out != "" {
return "", fmt.Errorf("%w: %s", err, out)
}
return "", err
}
+40 -37
View File
@@ -40,43 +40,46 @@ func Cleanup(options *types.Options) {
// DefaultOptions is the default options structure for nuclei during mocking.
var DefaultOptions = &types.Options{
Debug: false,
DebugRequests: false,
DebugResponse: false,
Silent: false,
Verbose: false,
NoColor: true,
UpdateTemplates: false,
JSONL: false,
OmitRawRequests: false,
EnableProgressBar: false,
TemplateList: false,
Stdin: false,
StopAtFirstMatch: false,
NoMeta: false,
Project: false,
MetricsPort: 0,
BulkSize: 25,
TemplateThreads: 10,
Timeout: 5,
Retries: 1,
RateLimit: 150,
RateLimitDuration: time.Second,
ProbeConcurrency: 50,
ProjectPath: "",
Severities: severity.Severities{},
Targets: []string{},
TargetsFilePath: "",
Output: "",
Proxy: []string{},
TraceLogFile: "",
Templates: []string{},
ExcludedTemplates: []string{},
CustomHeaders: []string{},
InteractshURL: "https://oast.fun",
InteractionsCacheSize: 5000,
InteractionsEviction: 60,
InteractionsCoolDownPeriod: 5,
Debug: false,
DebugRequests: false,
DebugResponse: false,
Silent: false,
Verbose: false,
NoColor: true,
UpdateTemplates: false,
JSONL: false,
OmitRawRequests: false,
EnableProgressBar: false,
TemplateList: false,
Stdin: false,
StopAtFirstMatch: false,
NoMeta: false,
Project: false,
MetricsPort: 0,
BulkSize: 25,
TemplateThreads: 10,
Timeout: 5,
Retries: 1,
RateLimit: 150,
RateLimitDuration: time.Second,
ProbeConcurrency: 50,
ProjectPath: "",
Severities: severity.Severities{},
Targets: []string{},
TargetsFilePath: "",
Output: "",
Proxy: []string{},
TraceLogFile: "",
Templates: []string{},
ExcludedTemplates: []string{},
CustomHeaders: []string{},
InteractshURL: "https://oast.fun",
InteractionsCacheSize: 5000,
InteractionsEviction: 60,
// Closing an interactsh client sleeps for the cooldown, waiting on callbacks
// that a unit test never receives, so every test generating an oast url paid
// five seconds for nothing.
InteractionsCoolDownPeriod: 0,
InteractionsPollDuration: 5,
GitHubTemplateRepo: []string{},
GitHubToken: "",
+3
View File
@@ -18,6 +18,9 @@ var knownLeaks = []goleak.Option{
// net/http transport maintains idle keep-alive connections whose goroutines
// exit on idle timeout or explicit close - not real leaks.
goleak.IgnoreAnyFunction("net/http.(*http2ClientConn).readLoop"),
// go 1.27 moved the bundled http2 transport out of net/http, which renamed
// the same idle connection goroutine
goleak.IgnoreAnyFunction("net/http/internal/http2.(*ClientConn).readLoop"),
goleak.IgnoreAnyFunction("net/http.(*persistConn).readLoop"),
goleak.IgnoreAnyFunction("net/http.(*persistConn).writeLoop"),
// expirable LRU cache creates a background goroutine for TTL expiration that persists
+60 -2
View File
@@ -386,6 +386,21 @@
"type": "boolean",
"title": "use case insensitive extract",
"description": "use case insensitive extract"
},
"prompt": {
"type": "string",
"title": "llm extraction prompt",
"description": "Natural language instruction describing what to extract; supports {{...}} interpolation of variables and target values"
},
"schema": {
"$ref": "#/$defs/map[string]string",
"title": "llm extraction schema",
"description": "Fields to extract mapped to their type"
},
"max-input-tokens": {
"type": "integer",
"title": "max llm input tokens",
"description": "Approximate cap on response tokens sent to the model"
}
},
"additionalProperties": false,
@@ -401,7 +416,8 @@
"kval",
"xpath",
"json",
"dsl"
"dsl",
"llm"
],
"title": "type of the extractor",
"description": "Type of the extractor"
@@ -1159,6 +1175,47 @@
"title": "xpath queries to match in response",
"description": "xpath are the XPath queries that will be evaluated against the response part of nuclei matching rules"
},
"prompt": {
"type": "string",
"title": "llm prompt",
"description": "Natural language question the model answers about the response part; supports {{...}} interpolation of variables and target values"
},
"expect": {
"type": "string",
"title": "expected llm verdict",
"description": "Verdict that counts as a match; defaults to yes"
},
"options": {
"items": {
"type": "string"
},
"type": "array",
"title": "allowed llm verdicts",
"description": "Fixed set of verdicts the model may return; defaults to yes and no"
},
"min-confidence": {
"type": "number",
"title": "minimum llm confidence",
"description": "Minimum confidence between 0 and 1 for an llm verdict to match"
},
"inputs": {
"items": {
"type": "string"
},
"type": "array",
"title": "llm inputs to compare",
"description": "Responses to compare instead of a single part; each entry supports {{...}} interpolation"
},
"max-input-tokens": {
"type": "integer",
"title": "max llm input tokens",
"description": "Approximate cap on response tokens sent to the model"
},
"allow-sole": {
"type": "boolean",
"title": "allow llm as sole matcher",
"description": "Permit an llm matcher to be the only matcher on a high or critical template"
},
"encoding": {
"type": "string",
"enum": [
@@ -1198,7 +1255,8 @@
"status",
"size",
"dsl",
"xpath"
"xpath",
"llm"
],
"title": "type of the matcher",
"description": "Type of the matcher"
+44
View File
@@ -0,0 +1,44 @@
package authx
import (
"context"
"net/http"
"strings"
httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
)
type cookieAuthContextKey struct{}
// IsAuthCookie reports whether an auth strategy installed the named cookie.
// HTTP cookie reuse must preserve these explicit credentials over jar values.
func IsAuthCookie(ctx context.Context, name string) bool {
names, _ := ctx.Value(cookieAuthContextKey{}).(map[string]struct{})
_, ok := names[name]
return ok
}
func markCookieAuth(req *http.Request, names []string) {
if len(names) == 0 {
return
}
existing, _ := req.Context().Value(cookieAuthContextKey{}).(map[string]struct{})
// Contexts can be shared by clones and redirects. Publish a new set rather
// than modifying the names recorded by an earlier strategy.
protected := make(map[string]struct{}, len(existing)+len(names))
// A Header strategy can replace the entire Cookie header. Keep earlier
// ownership only for cookie names still present in the request.
for _, header := range req.Header.Values("Cookie") {
for _, part := range httputil.SplitCookieHeader(header) {
name, _, _ := strings.Cut(part, "=")
name = strings.TrimSpace(name)
if _, ok := existing[name]; ok {
protected[name] = struct{}{}
}
}
}
for _, name := range names {
protected[name] = struct{}{}
}
*req = *req.WithContext(context.WithValue(req.Context(), cookieAuthContextKey{}, protected))
}
@@ -0,0 +1,42 @@
package authx
import (
"net/http"
"testing"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/stretchr/testify/require"
)
func TestCookieAuthOwnershipIsRequestLocal(t *testing.T) {
req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil)
require.NoError(t, err)
NewCookiesAuthStrategy(&Secret{Cookies: []Cookie{{Key: "session", Value: "configured"}}}).ApplyOnRR(req)
cloned := req.Clone(req.Context())
NewCookiesAuthStrategy(&Secret{Cookies: []Cookie{{Key: "other", Value: "configured"}}}).ApplyOnRR(cloned)
require.True(t, IsAuthCookie(req.Context(), "session"))
require.False(t, IsAuthCookie(req.Context(), "other"))
require.True(t, IsAuthCookie(cloned.Context(), "session"))
require.True(t, IsAuthCookie(cloned.Context(), "other"))
require.False(t, IsAuthCookie(cloned.Context(), "Session"))
NewHeadersAuthStrategy(&Secret{Headers: []KV{{Key: "Cookie", Value: "replacement=configured"}}}).ApplyOnRR(cloned)
require.True(t, IsAuthCookie(cloned.Context(), "replacement"))
require.False(t, IsAuthCookie(cloned.Context(), "session"))
require.False(t, IsAuthCookie(cloned.Context(), "other"))
require.True(t, IsAuthCookie(req.Context(), "session"))
}
func TestHeaderAuthCookieOwnershipIgnoresQuotedFragments(t *testing.T) {
for _, header := range []string{`payload="a;session=probe"`, `payload="a\";session=probe"`, `payload="a;session=probe`} {
t.Run(header, func(t *testing.T) {
req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil)
require.NoError(t, err)
NewCookiesAuthStrategy(&Secret{Cookies: []Cookie{{Key: "session", Value: "configured"}}}).ApplyOnRR(req)
NewHeadersAuthStrategy(&Secret{Headers: []KV{{Key: "Cookie", Value: header}}}).ApplyOnRR(req)
require.Equal(t, header, req.Header.Get("Cookie"))
require.True(t, IsAuthCookie(req.Context(), "payload"))
require.False(t, IsAuthCookie(req.Context(), "session"))
})
}
}
+6
View File
@@ -23,13 +23,16 @@ func NewCookiesAuthStrategy(data *Secret) *CookiesAuthStrategy {
// Apply applies the cookies auth strategy to the request
func (s *CookiesAuthStrategy) Apply(req *http.Request) {
var names []string
for _, cookie := range s.Data.Cookies {
c := &http.Cookie{
Name: cookie.Key,
Value: cookie.Value,
}
req.AddCookie(c)
names = append(names, cookie.Key)
}
markCookieAuth(req, names)
}
// ApplyOnRR applies the cookies auth strategy to the retryable request
@@ -51,10 +54,13 @@ func (s *CookiesAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
req.AddCookie(cookie)
}
// Add new cookies
var names []string
for _, cookie := range s.Data.Cookies {
req.AddCookie(&http.Cookie{
Name: cookie.Key,
Value: cookie.Value,
})
names = append(names, cookie.Key)
}
markCookieAuth(req.Request, names)
}
+11 -3
View File
@@ -2,7 +2,9 @@ package authx
import (
"net/http"
"strings"
httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
"github.com/projectdiscovery/retryablehttp-go"
)
@@ -26,6 +28,14 @@ func NewHeadersAuthStrategy(data *Secret) *HeadersAuthStrategy {
func (s *HeadersAuthStrategy) Apply(req *http.Request) {
for _, header := range s.Data.Headers {
req.Header[header.Key] = []string{header.Value}
if strings.EqualFold(header.Key, "Cookie") {
var names []string
for _, part := range httputil.SplitCookieHeader(header.Value) {
name, _, _ := strings.Cut(part, "=")
names = append(names, strings.TrimSpace(name))
}
markCookieAuth(req, names)
}
}
}
@@ -33,7 +43,5 @@ func (s *HeadersAuthStrategy) Apply(req *http.Request) {
// NOTE: This preserves exact header casing (e.g., barAuthToken stays as barAuthToken)
// This is useful for APIs that require case-sensitive header names
func (s *HeadersAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
for _, header := range s.Data.Headers {
req.Header[header.Key] = []string{header.Value}
}
s.Apply(req.Request)
}
+29
View File
@@ -59,6 +59,35 @@ func (c *Config) ReadTemplatesConfig() error {
return nil
}
// ReloadTemplateVersion refreshes the installed templates version from disk.
// Concurrent processes read the version once at startup, so one that waited on
// another's install or update must reload it to see the result instead of
// downloading the same release again. State recorded for a different templates
// directory is ignored.
func (c *Config) ReloadTemplateVersion() error {
statePath := c.getTemplatesConfigFilePath()
data, err := os.ReadFile(statePath)
if errors.Is(err, os.ErrNotExist) {
return nil
} else if err != nil {
return fmt.Errorf("read templates state %q: %w", statePath, err)
}
var state templatesState
if err := json.Unmarshal(data, &state); err != nil {
return fmt.Errorf("decode templates state %q: %w", statePath, err)
}
c.m.Lock()
defer c.m.Unlock()
// compare the resolved directory, so state written through a symlink is
// visible to a process using the real path, and the other way around
if CanonicalTemplatesPath(state.TemplatesDirectory) == CanonicalTemplatesPath(c.TemplatesDirectory) {
c.TemplateVersion = state.TemplateVersion
}
return nil
}
// WriteTemplatesConfig atomically writes restart-persistent templates state.
func (c *Config) WriteTemplatesConfig() error {
state := templatesState{
+91
View File
@@ -139,3 +139,94 @@ func writeTestFile(t *testing.T, path string, data []byte, mode os.FileMode) {
t.Fatalf("write test file: %v", err)
}
}
func TestReloadTemplateVersion(t *testing.T) {
newConfig := func(t *testing.T, templatesDir string) *Config {
t.Helper()
cfg := &Config{stateDir: t.TempDir()}
cfg.setTemplatesDir(templatesDir)
return cfg
}
t.Run("adopts the version another process installed", func(t *testing.T) {
templatesDir := t.TempDir()
cfg := newConfig(t, templatesDir)
other := &Config{stateDir: cfg.stateDir, TemplateVersion: "v2.0.0"}
other.setTemplatesDir(templatesDir)
if err := other.WriteTemplatesConfig(); err != nil {
t.Fatalf("write templates state: %v", err)
}
if err := cfg.ReloadTemplateVersion(); err != nil {
t.Fatalf("reload templates version: %v", err)
}
if cfg.TemplateVersion != "v2.0.0" {
t.Fatalf("template version = %q, want v2.0.0", cfg.TemplateVersion)
}
})
t.Run("ignores state of another templates directory", func(t *testing.T) {
cfg := newConfig(t, t.TempDir())
cfg.TemplateVersion = "v1.0.0"
other := &Config{stateDir: cfg.stateDir, TemplateVersion: "v2.0.0"}
other.setTemplatesDir(t.TempDir())
if err := other.WriteTemplatesConfig(); err != nil {
t.Fatalf("write templates state: %v", err)
}
if err := cfg.ReloadTemplateVersion(); err != nil {
t.Fatalf("reload templates version: %v", err)
}
if cfg.TemplateVersion != "v1.0.0" {
t.Fatalf("template version = %q, want v1.0.0", cfg.TemplateVersion)
}
})
t.Run("keeps the loaded version without state", func(t *testing.T) {
cfg := newConfig(t, t.TempDir())
cfg.TemplateVersion = "v1.0.0"
if err := cfg.ReloadTemplateVersion(); err != nil {
t.Fatalf("reload templates version: %v", err)
}
if cfg.TemplateVersion != "v1.0.0" {
t.Fatalf("template version = %q, want v1.0.0", cfg.TemplateVersion)
}
})
t.Run("adopts the version written through the other name of a symlink", func(t *testing.T) {
templatesDir := t.TempDir()
link := filepath.Join(t.TempDir(), "templates-link")
if err := os.Symlink(templatesDir, link); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
cfg := newConfig(t, link)
other := &Config{stateDir: cfg.stateDir, TemplateVersion: "v2.0.0"}
other.setTemplatesDir(templatesDir)
if err := other.WriteTemplatesConfig(); err != nil {
t.Fatalf("write templates state: %v", err)
}
if err := cfg.ReloadTemplateVersion(); err != nil {
t.Fatalf("reload templates version: %v", err)
}
if cfg.TemplateVersion != "v2.0.0" {
t.Fatalf("template version = %q, want v2.0.0", cfg.TemplateVersion)
}
cfg.TemplateVersion = "v1.0.0"
cfg.setTemplatesDir(templatesDir)
writer := &Config{stateDir: cfg.stateDir, TemplateVersion: "v3.0.0"}
writer.setTemplatesDir(link)
if err := writer.WriteTemplatesConfig(); err != nil {
t.Fatalf("write templates state through the link: %v", err)
}
if err := cfg.ReloadTemplateVersion(); err != nil {
t.Fatalf("reload templates version: %v", err)
}
if cfg.TemplateVersion != "v3.0.0" {
t.Fatalf("template version = %q, want v3.0.0", cfg.TemplateVersion)
}
})
}
+62
View File
@@ -0,0 +1,62 @@
package config
import (
"os"
"path/filepath"
"strings"
)
// CanonicalTemplatesPath is the identity of a templates directory. A symlink
// and its target are the same directory, including while the target does not
// exist yet, so a lock and a reloaded version follow either name.
func CanonicalTemplatesPath(path string) string {
if strings.TrimSpace(path) == "" {
return ""
}
return resolveTemplatesPath(path, map[string]struct{}{})
}
func resolveTemplatesPath(path string, seen map[string]struct{}) string {
abs, err := filepath.Abs(path)
if err != nil {
abs = filepath.Clean(path)
}
if _, ok := seen[abs]; ok {
return abs
}
seen[abs] = struct{}{}
dir := abs
var missing []string
for {
info, err := os.Lstat(dir)
if err == nil {
if info.Mode()&os.ModeSymlink != 0 {
if resolved, err := filepath.EvalSymlinks(dir); err == nil {
return filepath.Join(append([]string{resolved}, missing...)...)
}
target, readErr := os.Readlink(dir)
if readErr != nil {
return filepath.Join(append([]string{dir}, missing...)...)
}
if !filepath.IsAbs(target) {
target = filepath.Join(filepath.Dir(dir), target)
}
resolvedTarget := resolveTemplatesPath(target, seen)
return filepath.Join(append([]string{resolvedTarget}, missing...)...)
}
resolved := dir
if r, err := filepath.EvalSymlinks(dir); err == nil {
resolved = r
}
return filepath.Join(append([]string{resolved}, missing...)...)
}
parent := filepath.Dir(dir)
if parent == dir {
return filepath.Join(append([]string{dir}, missing...)...)
}
missing = append([]string{filepath.Base(dir)}, missing...)
dir = parent
}
}
+2 -1
View File
@@ -19,7 +19,7 @@ const (
// IndexVersion is the schema version for cache invalidation on breaking
// changes.
IndexVersion = 2
IndexVersion = 3
// DefaultMaxSize is the default maximum number of templates to cache.
DefaultMaxSize = 50000
@@ -70,6 +70,7 @@ func NewIndex(cacheDir string) (*Index, error) {
weight += len(value.Name)
weight += len(value.Severity)
weight += len(value.ProtocolType)
weight += len(value.Product)
weight += len(value.TemplateVerifier)
weight += len(value.VerifierFingerprint)
weight += len(value.ContentDigest)
+2
View File
@@ -905,6 +905,7 @@ func TestNewMetadataFromTemplate(t *testing.T) {
SeverityHolder: severity.Holder{
Severity: severity.Low,
},
Metadata: map[string]any{"product": " Tomcat "},
},
Verified: true,
TemplateVerifier: "verifier",
@@ -922,4 +923,5 @@ func TestNewMetadataFromTemplate(t *testing.T) {
require.Equal(t, tmpl.Type().String(), metadata.ProtocolType)
require.Equal(t, tmpl.Verified, metadata.Verified)
require.Equal(t, tmpl.TemplateVerifier, metadata.TemplateVerifier)
require.Equal(t, "tomcat", metadata.Product)
}
+10
View File
@@ -3,6 +3,7 @@ package index
import (
"os"
"slices"
"strings"
"time"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
@@ -48,6 +49,9 @@ type Metadata struct {
// ProtocolType is the primary protocol type of the template.
ProtocolType string `gob:"protocol_type"`
// Product is the lowercased info.metadata.product of the template.
Product string `gob:"product,omitempty"`
// Verified indicates whether the template is verified.
Verified bool `gob:"verified"`
@@ -114,6 +118,7 @@ func NewMetadataFromTemplate(path string, tpl *templates.Template) *Metadata {
Severity: tpl.Info.SeverityHolder.Severity.String(),
ProtocolType: tpl.Type().String(),
Product: templateProduct(tpl),
Verified: tpl.Verified,
TemplateVerifier: tpl.TemplateVerifier,
@@ -122,6 +127,11 @@ func NewMetadataFromTemplate(path string, tpl *templates.Template) *Metadata {
}
}
func templateProduct(tpl *templates.Template) string {
product, _ := tpl.Info.Metadata["product"].(string)
return strings.ToLower(strings.TrimSpace(product))
}
// IsValid checks if the cached metadata is still valid by comparing the file
// modification time.
func (m *Metadata) IsValid() bool {
+17
View File
@@ -76,6 +76,10 @@ type Config struct {
// MetadataIndex is an optional shared index borrowed by the store. The
// caller remains responsible for persisting it.
MetadataIndex *index.Index
// TargetFilter narrows loading to the templates per-target profiles
// select for at least one target; nil loads every filtered template.
TargetFilter index.FilterFunc
}
// Store is a storage for loaded nuclei templates
@@ -387,6 +391,11 @@ func (store *Store) buildIndexFilter() *index.Filter {
}
}
// selectedByTargets reports whether some target's profile selects the template.
func (store *Store) selectedByTargets(metadata *index.Metadata) bool {
return store.config.TargetFilter == nil || store.config.TargetFilter(metadata)
}
func (store *Store) loadTemplatesIndex() *index.Index {
var metadataIdx *index.Index
@@ -882,6 +891,10 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) ([]*temp
return
}
if !store.selectedByTargets(metadata) {
return
}
if len(tags) > 0 && !slices.ContainsFunc(tags, metadata.HasTag) {
return
}
@@ -916,6 +929,10 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) ([]*temp
noteExcludedByTag(templatePath, metadata)
return
}
if metadata != nil && !store.selectedByTargets(metadata) {
return
}
}
if err != nil {
+7 -1
View File
@@ -1,6 +1,8 @@
package core
import (
"sync"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
@@ -40,7 +42,11 @@ type TemplateExecutionCallback func(TemplateExecutionEvent)
// templates to leading to the final execution by the work pool, it is
// handled by the engine.
type Engine struct {
workPool *WorkPool
workPool *WorkPool
// originScoped tracks origin-scoped template attempts. A pair is finished
// only after a successful execution, so a failed first target does not
// suppress every later URL on that origin.
originScoped sync.Map
options *types.Options
executerOpts *protocols.ExecutorOptions
Callback func(*output.ResultEvent) // Executed on results
+30 -1
View File
@@ -7,6 +7,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
@@ -67,7 +68,7 @@ func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templ
// workflow requests are not counted as they can be conditional
// templateList count is user requested templates count (before clustering)
// totalReqAfterClustering is total requests count after clustering
e.executerOpts.Progress.Init(target.Count(), len(templatesList), int64(totalReqAfterClustering))
e.executerOpts.Progress.Init(target.Count(), len(templatesList), e.expectedRequests(finalTemplates, target, totalReqAfterClustering))
}
if stringsutil.EqualFoldAny(e.options.ScanStrategy, scanstrategy.Auto.String(), "") {
@@ -179,8 +180,33 @@ func (e *Engine) executeHostSpray(ctx context.Context, templatesList []*template
return results
}
// expectedRequests counts only the template and target pairs that per-target
// profiles select, so progress reflects the requests that will be sent.
func (e *Engine) expectedRequests(templatesList []*templates.Template, target provider.InputProvider, unscoped int) int64 {
scope := e.executerOpts.TargetScope
if scope == nil {
return int64(unscoped)
}
targets := make(map[protocols.TemplateSelection]int64)
target.Iterate(func(value *contextargs.MetaInput) bool {
targets[scope.For(value)]++
return true
})
var total int64
for selection, count := range targets {
total += count * int64(getSelectedRequestCount(templatesList, selection))
}
return total
}
// returns total requests count
func getRequestCount(templates []*templates.Template) int {
return getSelectedRequestCount(templates, nil)
}
// getSelectedRequestCount returns the requests of the templates selection
// selects; a nil selection selects every template.
func getSelectedRequestCount(templates []*templates.Template, selection protocols.TemplateSelection) int {
count := 0
for _, template := range templates {
// ignore requests in workflows as total requests in workflow
@@ -188,6 +214,9 @@ func getRequestCount(templates []*templates.Template) int {
if len(template.Workflows) > 0 {
continue
}
if selection != nil && !template.SelectedBy(selection) {
continue
}
count += template.TotalRequests
}
return count
+92 -5
View File
@@ -6,6 +6,7 @@ import (
"sync/atomic"
"time"
"github.com/projectdiscovery/nuclei/v3/pkg/input/normalize"
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
@@ -154,12 +155,18 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ
skip = false
}
// out of scope pairs keep their index so resume positions stay stable
inScope := e.inScope(template, scannedValue)
if !inScope {
skip = true
}
currentInfo.Lock()
currentInfo.InFlight[index] = struct{}{}
currentInfo.Unlock()
// Skip if the host has had errors
if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(e.executerOpts.ProtocolType.String(), contextargs.NewWithMetaInput(ctx, scannedValue)) {
if inScope && e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(e.executerOpts.ProtocolType.String(), contextargs.NewWithMetaInput(ctx, scannedValue)) {
skipEvent := &output.ResultEvent{
TemplateID: template.ID,
TemplatePath: template.Path,
@@ -210,6 +217,10 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t
default:
}
if !e.inScope(tpl, target) {
continue
}
// Check whether the target has already been marked as permanently
// unresponsive by HostErrorsCache before spawning another goroutine.
if e.executerOpts.HostErrorsCache != nil &&
@@ -266,8 +277,27 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t
}
}
// inScope reports whether per-target profiles select template for input.
func (e *Engine) inScope(template *templates.Template, input *contextargs.MetaInput) bool {
if e.executerOpts.TargetScope == nil {
return true
}
selection := e.executerOpts.TargetScope.For(input)
return selection == nil || template.SelectedBy(selection)
}
// executeTemplateOnInput performs template execution for a single input and returns match status and error
func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates.Template, value *contextargs.MetaInput) (bool, error) {
func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates.Template, value *contextargs.MetaInput) (matched bool, err error) {
// A template fixed to the target's origin makes the same requests for every
// target that shares one, which a crawled list produces by the hundred.
// Running it once per origin sends exactly the same traffic, minus the
// repeats. The reservation is kept only when execution succeeds, so a
// failed attempt leaves the origin free for a later target.
finish, run := e.claimOriginScoped(template, value)
if !run {
return false, nil
}
defer func() { finish(err == nil) }()
finished := e.templateExecutionStarted(template, value.Input)
defer func() { finished(ctx.Err()) }()
ctxArgs := contextargs.New(ctx)
@@ -279,9 +309,9 @@ func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates
return e.executeWorkflow(scanCtx, template.CompiledWorkflow), nil
default:
if e.Callback != nil {
results, err := template.Executer.ExecuteWithResults(scanCtx)
if err != nil {
return false, err
results, execErr := template.Executer.ExecuteWithResults(scanCtx)
if execErr != nil {
return false, execErr
}
for _, result := range results {
e.Callback(result)
@@ -291,3 +321,60 @@ func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates
return template.Executer.Execute(scanCtx)
}
}
// originAttempt is one origin-scoped execution. done is set only after a
// successful run. While inflight is set, other callers wait, then retry if
// that attempt failed.
type originAttempt struct {
mu sync.Mutex
cond *sync.Cond
inflight bool
done bool
}
func newOriginAttempt() *originAttempt {
attempt := &originAttempt{}
attempt.cond = sync.NewCond(&attempt.mu)
return attempt
}
// claimOriginScoped reports whether this caller should execute an origin-scoped
// template. finish(true) records a successful run. finish(false) releases the
// claim so a later target, including one that waited, can try again. Templates
// whose requests depend on the target path always run. CustomIP is part of the
// key so each address selected for one URL is tested on its own.
func (e *Engine) claimOriginScoped(template *templates.Template, value *contextargs.MetaInput) (finish func(bool), run bool) {
noop := func(bool) {}
if template == nil || value == nil || !template.IsOriginScoped() {
return noop, true
}
origin := normalize.Origin(value.Input)
if origin == "" {
return noop, true
}
key := template.ID + "\x00" + origin + "\x00" + value.CustomIP
actual, _ := e.originScoped.LoadOrStore(key, newOriginAttempt())
slot := actual.(*originAttempt)
slot.mu.Lock()
for {
if slot.done {
slot.mu.Unlock()
return noop, false
}
if !slot.inflight {
slot.inflight = true
slot.mu.Unlock()
return func(success bool) {
slot.mu.Lock()
slot.inflight = false
if success {
slot.done = true
}
slot.cond.Broadcast()
slot.mu.Unlock()
}, true
}
slot.cond.Wait()
}
}
+285
View File
@@ -2,6 +2,7 @@ package core
import (
"context"
"errors"
"fmt"
"reflect"
"sync"
@@ -13,6 +14,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
httpprotocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
tmpltypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
@@ -140,6 +142,289 @@ func TestExecuteTemplateOnInputReportsCancellationOnFinish(t *testing.T) {
}
}
type scriptedExecuter struct {
calls int
errs []error
miss bool
}
func (s *scriptedExecuter) Compile() error { return nil }
func (s *scriptedExecuter) Requests() int { return 1 }
func (s *scriptedExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
s.calls++
if len(s.errs) == 0 {
return !s.miss, nil
}
err := s.errs[0]
s.errs = s.errs[1:]
return false, err
}
func (s *scriptedExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) {
_, err := s.Execute(ctx)
return nil, err
}
func originScopedTemplate(path string) *templates.Template {
return &templates.Template{
ID: "root-health",
RequestsHTTP: []*httpprotocol.Request{{
Path: []string{path},
}},
}
}
func TestOriginScopedRetriesAfterFailure(t *testing.T) {
e := newTestEngine()
exec := &scriptedExecuter{errs: []error{errors.New("timeout")}}
tpl := originScopedTemplate("{{RootURL}}/health")
tpl.Executer = exec
_, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: "https://example.com/a"})
if err == nil {
t.Fatal("expected the first attempt to fail")
}
ok, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: "https://example.com/b"})
if err != nil || !ok {
t.Fatalf("later target should retry the origin, ok=%v err=%v", ok, err)
}
if exec.calls != 2 {
t.Fatalf("calls = %d, want 2", exec.calls)
}
}
func TestOriginScopedSkipsAfterSuccess(t *testing.T) {
e := newTestEngine()
exec := &scriptedExecuter{}
tpl := originScopedTemplate("{{RootURL}}/health")
tpl.Executer = exec
if _, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: "https://example.com/a"}); err != nil {
t.Fatal(err)
}
if _, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: "https://example.com/b?q=1"}); err != nil {
t.Fatal(err)
}
if exec.calls != 1 {
t.Fatalf("calls = %d, want 1", exec.calls)
}
}
func TestOriginScopedKeepsEachCustomIP(t *testing.T) {
e := newTestEngine()
exec := &scriptedExecuter{}
tpl := originScopedTemplate("{{RootURL}}/health")
tpl.Executer = exec
for _, ip := range []string{"203.0.113.1", "203.0.113.2"} {
_, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{
Input: "https://example.com/a",
CustomIP: ip,
})
if err != nil {
t.Fatal(err)
}
}
if exec.calls != 2 {
t.Fatalf("calls = %d, want 2", exec.calls)
}
}
func TestBaseURLTemplateIsNotOriginScoped(t *testing.T) {
e := newTestEngine()
exec := &scriptedExecuter{}
tpl := originScopedTemplate("{{BaseURL}}/health")
tpl.Executer = exec
for _, raw := range []string{"https://example.com/a", "https://example.com/b"} {
if _, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: raw}); err != nil {
t.Fatal(err)
}
}
if exec.calls != 2 {
t.Fatalf("calls = %d, want 2", exec.calls)
}
}
func TestOriginScopedWaiterRetriesAfterFailedAttempt(t *testing.T) {
e := newTestEngine()
started := make(chan struct{})
release := make(chan struct{})
var calls atomic.Int32
tpl := originScopedTemplate("{{RootURL}}/health")
tpl.Executer = &gatedExecuter{started: started, release: release, calls: &calls}
firstDone := make(chan error, 1)
go func() {
_, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: "https://example.com/a"})
firstDone <- err
}()
<-started
secondDone := make(chan error, 1)
go func() {
_, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: "https://example.com/b"})
secondDone <- err
}()
select {
case err := <-secondDone:
t.Fatalf("second target returned while the first attempt was still running: %v", err)
case <-time.After(50 * time.Millisecond):
}
close(release)
if err := <-firstDone; err == nil {
t.Fatal("expected the first attempt to fail")
}
if err := <-secondDone; err != nil {
t.Fatalf("waiter should run after the failed attempt: %v", err)
}
if calls.Load() != 2 {
t.Fatalf("calls = %d, want 2", calls.Load())
}
}
func TestOriginScopedKeepsDistinctOriginsAndTemplates(t *testing.T) {
e := newTestEngine()
exec := &scriptedExecuter{}
first := originScopedTemplate("{{RootURL}}/health")
first.Executer = exec
second := originScopedTemplate("{{RootURL}}/health")
second.ID = "other-template"
second.Executer = exec
inputs := []*contextargs.MetaInput{
{Input: "https://example.com/a"},
{Input: "https://other.test/a"},
{Input: "http://example.com/a"},
{Input: "https://example.com:8443/a"},
}
for _, value := range inputs {
if _, err := e.executeTemplateOnInput(context.Background(), first, value); err != nil {
t.Fatal(err)
}
}
if _, err := e.executeTemplateOnInput(context.Background(), second, inputs[0]); err != nil {
t.Fatal(err)
}
if exec.calls != 5 {
t.Fatalf("calls = %d, want 5", exec.calls)
}
}
func TestOriginScopedFoldsDefaultPort(t *testing.T) {
e := newTestEngine()
exec := &scriptedExecuter{}
tpl := originScopedTemplate("{{RootURL}}/health")
tpl.Executer = exec
for _, raw := range []string{"https://example.com/a", "https://example.com:443/b"} {
if _, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: raw}); err != nil {
t.Fatal(err)
}
}
if exec.calls != 1 {
t.Fatalf("calls = %d, want 1", exec.calls)
}
}
func TestOriginScopedSameCustomIPSkips(t *testing.T) {
e := newTestEngine()
exec := &scriptedExecuter{}
tpl := originScopedTemplate("{{RootURL}}/health")
tpl.Executer = exec
for range 2 {
_, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{
Input: "https://example.com/a",
CustomIP: "203.0.113.1",
})
if err != nil {
t.Fatal(err)
}
}
if exec.calls != 1 {
t.Fatalf("calls = %d, want 1", exec.calls)
}
}
func TestOriginScopedMissStillCounts(t *testing.T) {
e := newTestEngine()
exec := &scriptedExecuter{miss: true}
tpl := originScopedTemplate("{{RootURL}}/health")
tpl.Executer = exec
for _, raw := range []string{"https://example.com/a", "https://example.com/b"} {
ok, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: raw})
if err != nil || ok {
t.Fatalf("non-match ok=%v err=%v", ok, err)
}
}
if exec.calls != 1 {
t.Fatalf("calls = %d, want 1", exec.calls)
}
}
func TestOriginScopedCallbackErrorRetries(t *testing.T) {
e := newTestEngine()
e.Callback = func(*output.ResultEvent) {}
exec := &scriptedExecuter{errs: []error{errors.New("timeout")}}
tpl := originScopedTemplate("{{RootURL}}/health")
tpl.Executer = exec
if _, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: "https://example.com/a"}); err == nil {
t.Fatal("expected the callback path to fail")
}
if _, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: "https://example.com/b"}); err != nil {
t.Fatal(err)
}
if exec.calls != 2 {
t.Fatalf("calls = %d, want 2", exec.calls)
}
}
func TestOriginScopedLeavesValuesWithoutAHost(t *testing.T) {
e := newTestEngine()
exec := &scriptedExecuter{}
tpl := originScopedTemplate("{{RootURL}}/health")
tpl.Executer = exec
for range 2 {
if _, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: "not a url"}); err != nil {
t.Fatal(err)
}
}
for _, raw := range []string{"example.com", "other.test", "example.com"} {
if _, err := e.executeTemplateOnInput(context.Background(), tpl, &contextargs.MetaInput{Input: raw}); err != nil {
t.Fatal(err)
}
}
if exec.calls != 4 {
t.Fatalf("calls = %d, want 4", exec.calls)
}
}
type gatedExecuter struct {
started chan struct{}
release chan struct{}
calls *atomic.Int32
}
func (g *gatedExecuter) Compile() error { return nil }
func (g *gatedExecuter) Requests() int { return 1 }
func (g *gatedExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
if g.calls.Add(1) == 1 {
close(g.started)
<-g.release
return false, errors.New("timeout")
}
return true, nil
}
func (g *gatedExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) {
ok, err := g.Execute(ctx)
if err != nil || !ok {
return nil, err
}
return []*output.ResultEvent{{}}, nil
}
type fakeTargetProvider struct {
values []*contextargs.MetaInput
}
+116
View File
@@ -0,0 +1,116 @@
package core
import (
"context"
"slices"
"sync"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
tmpltypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
"github.com/stretchr/testify/require"
)
// allowList selects the templates it lists.
type allowList struct{ paths []string }
func (a *allowList) Allows(templatePath string) bool { return slices.Contains(a.paths, templatePath) }
// scopeByInput maps inputs to selections; unlisted inputs are unrestricted.
type scopeByInput map[string]*allowList
func (s scopeByInput) For(input *contextargs.MetaInput) protocols.TemplateSelection {
if selection, ok := s[input.Input]; ok {
return selection
}
return nil
}
// recordingExecuter records the inputs it runs on.
type recordingExecuter struct {
path string
mu *sync.Mutex
runs *[]string
}
func (r *recordingExecuter) Compile() error { return nil }
func (r *recordingExecuter) Requests() int { return 1 }
func (r *recordingExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
r.mu.Lock()
defer r.mu.Unlock()
*r.runs = append(*r.runs, r.path+"@"+ctx.Input.MetaInput.Input)
return false, nil
}
func (r *recordingExecuter) ExecuteWithResults(*scan.ScanContext) ([]*output.ResultEvent, error) {
return nil, nil
}
// progressRecorder captures the expected request count; the engine calls no
// other progress method on this path.
type progressRecorder struct {
progress.Progress
requests int64
}
func (p *progressRecorder) Init(_ int64, _ int, requests int64) { p.requests = requests }
func TestExecuteScanRunsOnlyScopedPairs(t *testing.T) {
for _, strategy := range []scanstrategy.ScanStrategy{scanstrategy.TemplateSpray, scanstrategy.HostSpray} {
t.Run(strategy.String(), func(t *testing.T) {
options := &types.Options{
BulkSize: 2,
TemplateThreads: 2,
HeadlessBulkSize: 1,
HeadlessTemplateThreads: 1,
ScanStrategy: strategy.String(),
}
progress := &progressRecorder{}
engine := New(options)
engine.SetExecuterOptions(&protocols.ExecutorOptions{
Logger: engine.Logger,
Options: options,
ResumeCfg: types.NewResumeCfg(),
ProtocolType: tmpltypes.HTTPProtocol,
Progress: progress,
TargetScope: scopeByInput{
"php": {paths: []string{"thinkphp.yaml"}},
"tomcat": {paths: []string{"tomcat.yaml", "generic.yaml"}},
"none": {},
},
})
var (
mu sync.Mutex
runs []string
)
var templatesList []*templates.Template
for _, path := range []string{"thinkphp.yaml", "tomcat.yaml", "generic.yaml"} {
templatesList = append(templatesList, &templates.Template{
ID: path,
Path: path,
TotalRequests: 1,
Executer: &recordingExecuter{path: path, mu: &mu, runs: &runs},
})
}
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{
{Input: "php"}, {Input: "tomcat"}, {Input: "none"}, {Input: "plain"},
}}
engine.ExecuteScanWithOpts(context.Background(), templatesList, targets, true)
require.ElementsMatch(t, []string{
"thinkphp.yaml@php",
"tomcat.yaml@tomcat", "generic.yaml@tomcat",
"thinkphp.yaml@plain", "tomcat.yaml@plain", "generic.yaml@plain",
}, runs)
require.EqualValues(t, len(runs), progress.requests, "progress counts only scoped pairs")
})
}
}
+18 -14
View File
@@ -2,13 +2,12 @@ package customtemplates
import (
"context"
httpclient "net/http"
"path/filepath"
"strings"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/google/go-github/v30/github"
"github.com/google/go-github/v92/github"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
@@ -89,7 +88,10 @@ func (customTemplate *customTemplateGitHubRepo) logPullError(err error) {
// NewGitHubProviders returns new instance of GitHub providers for downloading custom templates
func NewGitHubProviders(options *types.Options) ([]*customTemplateGitHubRepo, error) {
providers := []*customTemplateGitHubRepo{}
gitHubClient := getGHClientIncognito()
gitHubClient, err := github.NewClient()
if err != nil {
return nil, errors.Wrap(err, "could not create github client")
}
if options.GitHubTemplateDisableDownload {
return providers, nil
@@ -150,9 +152,16 @@ getRepo:
repo, _, err := gitHubClient.Repositories.Get(context.Background(), repoOwner, repoName)
if err != nil {
// retry with authentication
if gitHubClient = getGHClientWithToken(githubToken); gitHubClient != nil && !retried {
retried = true
goto getRepo
if !retried {
authClient, authErr := getGHClientWithToken(githubToken)
if authErr != nil {
return nil, errors.Wrap(authErr, "could not create authenticated github client")
}
if authClient != nil {
gitHubClient = authClient
retried = true
goto getRepo
}
}
return nil, err
}
@@ -233,20 +242,15 @@ func getAuth(username, password string) *http.BasicAuth {
return nil
}
func getGHClientWithToken(token string) *github.Client {
func getGHClientWithToken(token string) (*github.Client, error) {
if token != "" {
ctx := context.Background()
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: token},
)
oauthClient := oauth2.NewClient(ctx, ts)
return github.NewClient(oauthClient)
return github.NewClient(github.WithHTTPClient(oauthClient))
}
return nil
}
func getGHClientIncognito() *github.Client {
var tc *httpclient.Client
return github.NewClient(tc)
return nil, nil
}
+14 -4
View File
@@ -55,18 +55,28 @@ func (b *Body) Parse(req *retryablehttp.Request) (bool, error) {
b.value = NewValue(dataStr)
tmp := b.value.Parsed()
if !tmp.IsNIL() {
return true, nil
}
switch {
case strings.Contains(contentType, "application/json") && tmp.IsNIL():
case strings.Contains(contentType, "application/json"):
// GraphQL-over-HTTP uses application/json; prefer the GraphQL decoder
// so variables / inline args become fuzz points instead of raw JSON.
if graphql := dataformat.Get(dataformat.GraphqlDataFormat); graphql != nil && graphql.IsType(dataStr) {
return b.parseBody(dataformat.GraphqlDataFormat, req)
}
if !tmp.IsNIL() {
return true, nil
}
return b.parseBody(dataformat.JSONDataFormat, req)
case strings.Contains(contentType, "application/xml") && tmp.IsNIL():
return b.parseBody(dataformat.XMLDataFormat, req)
case strings.Contains(contentType, "multipart/form-data") && tmp.IsNIL():
return b.parseBody(dataformat.MultiPartFormDataFormat, req)
}
if !tmp.IsNIL() {
return true, nil
}
parsed, err := b.parseBody(dataformat.FormDataFormat, req)
if err != nil {
gologger.Warning().Msgf("Could not parse body as form data: %s\n", err)
+106
View File
@@ -244,3 +244,109 @@ func TestMultiPartFormConcurrentParse(t *testing.T) {
t.Error(err)
}
}
func TestBodyGraphqlVariablesComponent(t *testing.T) {
raw := `{
"query": "mutation ImportPaste($host: String!, $path: String!) { importPaste(host: $host, path: $path) { result } }",
"variables": {
"host": "example.com",
"path": "/robots.txt"
}
}`
req, err := retryablehttp.NewRequest("POST", "https://example.com/graphql", strings.NewReader(raw))
require.NoError(t, err)
req.Header.Set("Content-Type", "application/json")
body := New(RequestBodyComponent)
parsed, err := body.Parse(req)
require.NoError(t, err)
require.True(t, parsed)
keys := map[string]struct{}{}
_ = body.Iterate(func(key string, value interface{}) error {
keys[key] = struct{}{}
return nil
})
require.Contains(t, keys, "host")
require.Contains(t, keys, "path")
require.NotContains(t, keys, "query")
require.NoError(t, body.SetValue("path", "/robots.txt; id"))
rebuilt, err := body.Rebuild()
require.NoError(t, err)
newBody, err := io.ReadAll(rebuilt.Body)
require.NoError(t, err)
require.Contains(t, string(newBody), `/robots.txt; id`)
require.Contains(t, string(newBody), `"host":"example.com"`)
require.Contains(t, string(newBody), "mutation ImportPaste")
}
func TestBodyGraphqlInlineArgsComponent(t *testing.T) {
raw := `{"query":"query { jobs(jobType: \"front-end\") { id } }"}`
req, err := retryablehttp.NewRequest("POST", "https://example.com/graphql", strings.NewReader(raw))
require.NoError(t, err)
req.Header.Set("Content-Type", "application/json")
body := New(RequestBodyComponent)
parsed, err := body.Parse(req)
require.NoError(t, err)
require.True(t, parsed)
require.NoError(t, body.SetValue("jobType", "canary"))
rebuilt, err := body.Rebuild()
require.NoError(t, err)
newBody, err := io.ReadAll(rebuilt.Body)
require.NoError(t, err)
require.Contains(t, string(newBody), "canary")
require.NotContains(t, string(newBody), "front-end")
}
func TestBodyGraphqlNestedVariablesComponent(t *testing.T) {
raw := `{
"query": "mutation ($input: ImportInput!) { importPaste(input: $input) { result } }",
"variables": {
"input": {
"host": "example.com",
"path": "/robots.txt"
}
}
}`
req, err := retryablehttp.NewRequest("POST", "https://example.com/graphql", strings.NewReader(raw))
require.NoError(t, err)
req.Header.Set("Content-Type", "application/json")
body := New(RequestBodyComponent)
parsed, err := body.Parse(req)
require.NoError(t, err)
require.True(t, parsed)
require.NoError(t, body.SetValue("input~path", "/etc/passwd"))
rebuilt, err := body.Rebuild()
require.NoError(t, err)
newBody, err := io.ReadAll(rebuilt.Body)
require.NoError(t, err)
require.Contains(t, string(newBody), `/etc/passwd`)
require.Contains(t, string(newBody), `"host":"example.com"`)
}
func TestBodyRegularJSONNotForcedToGraphql(t *testing.T) {
req, err := retryablehttp.NewRequest("POST", "https://example.com", strings.NewReader(`{"foo":"bar"}`))
require.NoError(t, err)
req.Header.Set("Content-Type", "application/json")
body := New(RequestBodyComponent)
parsed, err := body.Parse(req)
require.NoError(t, err)
require.True(t, parsed)
require.NoError(t, body.SetValue("foo", "baz"))
rebuilt, err := body.Rebuild()
require.NoError(t, err)
newBody, err := io.ReadAll(rebuilt.Body)
require.NoError(t, err)
require.Equal(t, `{"foo":"baz"}`, string(newBody))
}
+10 -6
View File
@@ -3,9 +3,10 @@ package component
import (
"context"
"fmt"
"net/http"
"strings"
"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/dataformat"
httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
"github.com/projectdiscovery/retryablehttp-go"
mapsutil "github.com/projectdiscovery/utils/maps"
)
@@ -85,14 +86,17 @@ func (c *Cookie) Rebuild() (*retryablehttp.Request, error) {
cloned := c.req.Clone(context.Background())
cloned.Header.Del("Cookie")
var fields []string
c.value.parsed.Iterate(func(key string, value any) bool {
cookie := &http.Cookie{
Name: key,
Value: fmt.Sprint(value), // Assume the value is always a string for cookies
}
cloned.AddCookie(cookie)
fields = append(fields, key+"="+fmt.Sprint(value)) // Assume the value is always a string for cookies
return true
})
// Cookie.String sanitizes bytes that can be intentional fuzzing payloads.
// Header validation still applies when the HTTP transport sends the request.
if len(fields) > 0 {
cloned.Header.Set("Cookie", strings.Join(fields, "; "))
}
httputil.SetCookieFields(cloned.Request, fields)
return cloned, nil
}
+61
View File
@@ -2,12 +2,55 @@ package component
import (
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/stretchr/testify/require"
)
func TestCookieComponentPreservesRawPayload(t *testing.T) {
for _, payload := range []string{"old'", `old"`, `old";probe=value`} {
t.Run(payload, func(t *testing.T) {
req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil)
require.NoError(t, err)
req.Header.Set("Cookie", "session=old; preference=dark")
cookies := NewCookie()
parsed, err := cookies.Parse(req)
require.NoError(t, err)
require.True(t, parsed)
require.NoError(t, cookies.SetValue("session", payload))
rebuilt, err := cookies.Rebuild()
require.NoError(t, err)
require.Equal(t, "session="+payload+"; preference=dark", rebuilt.Header.Get("Cookie"))
require.Equal(t, "session=old; preference=dark", req.Header.Get("Cookie"))
})
}
}
func TestCookieComponentRejectsHeaderNewlines(t *testing.T) {
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests.Add(1) }))
defer server.Close()
req, err := retryablehttp.NewRequest(http.MethodGet, server.URL, nil)
require.NoError(t, err)
req.Header.Set("Cookie", "session=old")
cookies := NewCookie()
parsed, err := cookies.Parse(req)
require.NoError(t, err)
require.True(t, parsed)
require.NoError(t, cookies.SetValue("session", "old\r\nX-Injected: yes"))
rebuilt, err := cookies.Rebuild()
require.NoError(t, err)
resp, err := server.Client().Do(rebuilt.Request)
if resp != nil {
require.NoError(t, resp.Body.Close())
}
require.ErrorContains(t, err, "invalid header field value")
require.Equal(t, int32(0), requests.Load())
}
func TestCookieComponent(t *testing.T) {
req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil)
if err != nil {
@@ -55,3 +98,21 @@ func TestCookieComponent(t *testing.T) {
newCookie, _ := rebuilt.Cookie("session")
require.Equal(t, "new-session", newCookie.Value, "unexpected cookie value")
}
func BenchmarkCookieComponentRebuild(b *testing.B) {
req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil)
require.NoError(b, err)
req.Header.Set("Cookie", "session=old; preference=dark")
cookies := NewCookie()
parsed, err := cookies.Parse(req)
require.NoError(b, err)
require.True(b, parsed)
require.NoError(b, cookies.SetValue("session", "old'"))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := cookies.Rebuild(); err != nil {
b.Fatal(err)
}
}
}
+56 -11
View File
@@ -18,6 +18,7 @@ func init() {
dataformats = make(map[string]DataFormat)
// register the default data formats
RegisterDataFormat(NewGraphql())
RegisterDataFormat(NewJSON())
RegisterDataFormat(NewXML())
RegisterDataFormat(NewRaw())
@@ -26,6 +27,8 @@ func init() {
}
const (
// GraphqlDataFormat is the name of the GraphQL-over-HTTP JSON data format
GraphqlDataFormat = "graphql"
// JSONDataFormat is the name of the JSON data format
JSONDataFormat = "json"
// XMLDataFormat is the name of the XML data format
@@ -38,6 +41,17 @@ const (
MultiPartFormDataFormat = "multipart/form-data"
)
// decodePriority prefers more specific formats before generic ones (GraphQL
// bodies are valid JSON, so GraphQL must win over JSON).
var decodePriority = []string{
GraphqlDataFormat,
JSONDataFormat,
XMLDataFormat,
MultiPartFormDataFormat,
FormDataFormat,
RawDataFormat,
}
// Get returns the dataformat by name
func Get(name string) DataFormat {
return dataformats[name]
@@ -70,22 +84,53 @@ type Decoded struct {
// Decode decodes the data from a format
func Decode(data string) (*Decoded, error) {
for _, dataformat := range dataformats {
if dataformat.IsType(data) {
decoded, err := dataformat.Decode(data)
if err != nil {
return nil, err
}
value := &Decoded{
DataFormat: dataformat.Name(),
Data: decoded,
}
return value, nil
tryDecode := func(dataformat DataFormat) (*Decoded, error) {
if dataformat == nil || !dataformat.IsType(data) {
return nil, nil
}
decoded, err := dataformat.Decode(data)
if err != nil {
return nil, err
}
return &Decoded{
DataFormat: dataformat.Name(),
Data: decoded,
}, nil
}
for _, name := range decodePriority {
decoded, err := tryDecode(dataformats[name])
if err != nil {
return nil, err
}
if decoded != nil {
return decoded, nil
}
}
for name, dataformat := range dataformats {
if containsString(decodePriority, name) {
continue
}
decoded, err := tryDecode(dataformat)
if err != nil {
return nil, err
}
if decoded != nil {
return decoded, nil
}
}
return nil, nil
}
func containsString(items []string, want string) bool {
for _, item := range items {
if item == want {
return true
}
}
return false
}
// Encode encodes the data into a format
func Encode(data KV, dataformat string) (string, error) {
if dataformat == "" {
+515
View File
@@ -0,0 +1,515 @@
package dataformat
import (
"fmt"
"sort"
"strings"
"github.com/graphql-go/graphql/language/ast"
"github.com/graphql-go/graphql/language/kinds"
"github.com/graphql-go/graphql/language/lexer"
"github.com/graphql-go/graphql/language/parser"
"github.com/graphql-go/graphql/language/printer"
"github.com/graphql-go/graphql/language/source"
"github.com/graphql-go/graphql/language/visitor"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
)
const (
graphqlMetaQuery = "#_query"
graphqlMetaOperationName = "#_operationName"
graphqlMetaHasVariables = "#_hasVariables"
graphqlMetaExtensions = "#_extensions"
)
// Graphql encodes and decodes GraphQL-over-HTTP JSON bodies
// ({"query","variables","operationName"}) so DAST fuzzing can target
// variables and inline field arguments instead of the raw query string.
type Graphql struct{}
var _ DataFormat = &Graphql{}
// NewGraphql returns a new GraphQL data format encoder/decoder.
func NewGraphql() *Graphql {
return &Graphql{}
}
// Name returns the name of the data format.
func (g *Graphql) Name() string {
return GraphqlDataFormat
}
// IsType reports whether data is a GraphQL HTTP request body.
func (g *Graphql) IsType(data string) bool {
_, ok := parseGraphQLHTTPBody(data, true)
return ok
}
type graphQLHTTPBody struct {
Query string `json:"query,omitempty"`
OperationName string `json:"operationName,omitempty"`
Variables *map[string]any `json:"variables,omitempty"`
Extensions json.Message `json:"extensions,omitempty"`
}
func parseGraphQLHTTPBody(data string, validateQuery bool) (graphQLHTTPBody, bool) {
trimmed := strings.TrimSpace(data)
if !strings.HasPrefix(trimmed, "{") || !strings.HasSuffix(trimmed, "}") {
return graphQLHTTPBody{}, false
}
var body graphQLHTTPBody
if err := json.Unmarshal([]byte(trimmed), &body); err != nil {
return graphQLHTTPBody{}, false
}
if strings.TrimSpace(body.Query) == "" {
return graphQLHTTPBody{}, false
}
if !validateQuery {
return body, true
}
if _, err := parseQueryAST(body.Query); err != nil {
return graphQLHTTPBody{}, false
}
return body, true
}
func parseQueryAST(query string) (*ast.Document, error) {
doc, err := parser.Parse(parser.ParseParams{
Source: &source.Source{Body: []byte(query)},
})
if err == nil {
return doc, nil
}
// graphql-go v0.8.1 predates GraphQL null literal support. Replace null
// name tokens with an enum sentinel for parsing, then restore the AST.
rewritten, sentinel, replaced, rewriteErr := rewriteGraphQLNullTokens(query)
if rewriteErr != nil || !replaced {
return nil, err
}
doc, rewriteErr = parser.Parse(parser.ParseParams{
Source: &source.Source{Body: []byte(rewritten)},
})
if rewriteErr != nil {
return nil, err
}
visitor.Visit(doc, &visitor.VisitorOptions{
Enter: func(params visitor.VisitFuncParams) (string, interface{}) {
switch node := params.Node.(type) {
case *ast.Name:
if node.Value == sentinel {
node.Value = "null"
}
case *ast.EnumValue:
if node.Value == sentinel {
node.Value = "null"
}
}
return visitor.ActionNoChange, nil
},
}, nil)
return doc, nil
}
func rewriteGraphQLNullTokens(query string) (string, string, bool, error) {
sentinel := "__NUCLEI_GRAPHQL_NULL__"
for strings.Contains(query, sentinel) {
sentinel += "_"
}
src := &source.Source{Body: []byte(query)}
next := lexer.Lex(src)
var out strings.Builder
last := 0
replaced := false
for {
token, err := next(0)
if err != nil {
return "", "", false, err
}
if token.Kind == lexer.EOF {
break
}
if token.Kind != lexer.NAME || token.Value != "null" {
continue
}
out.WriteString(query[last:token.Start])
out.WriteString(sentinel)
last = token.End
replaced = true
}
if !replaced {
return query, sentinel, false, nil
}
out.WriteString(query[last:])
return out.String(), sentinel, true, nil
}
// Decode extracts fuzzable GraphQL variables / inline arguments.
func (g *Graphql) Decode(data string) (KV, error) {
body, ok := parseGraphQLHTTPBody(data, true)
if !ok {
return KV{}, fmt.Errorf("not a graphql http body")
}
doc, err := parseQueryAST(body.Query)
if err != nil {
return KV{}, fmt.Errorf("could not parse graphql query: %w", err)
}
kv := KVMap(map[string]any{})
kv.Set(graphqlMetaQuery, body.Query)
if body.OperationName != "" {
kv.Set(graphqlMetaOperationName, body.OperationName)
}
if body.Extensions != nil {
kv.Set(graphqlMetaExtensions, body.Extensions)
}
hasVariables := body.Variables != nil
if hasVariables {
kv.Set(graphqlMetaHasVariables, true)
for key, value := range *body.Variables {
kv.Set(key, value)
}
}
// Inline arguments are only exposed when variables are absent; otherwise
// variables are the stable fuzz surface (and rewrites stay in JSON).
if !hasVariables {
for key, value := range collectInlineArguments(doc) {
if kv.Get(key) != nil {
continue
}
kv.Set(key, value)
}
}
return kv, nil
}
// Encode rebuilds a GraphQL HTTP JSON body from fuzzed KV values.
func (g *Graphql) Encode(data KV) (string, error) {
queryVal := data.Get(graphqlMetaQuery)
if queryVal == nil {
return "", fmt.Errorf("graphql query metadata missing")
}
query := types.ToString(queryVal)
body := graphQLHTTPBody{Query: query}
if op := data.Get(graphqlMetaOperationName); op != nil {
body.OperationName = types.ToString(op)
}
if extensions, ok := data.Get(graphqlMetaExtensions).(json.Message); ok {
body.Extensions = extensions
}
hasVariables, _ := data.Get(graphqlMetaHasVariables).(bool)
if hasVariables {
variables := make(map[string]any)
body.Variables = &variables
data.Iterate(func(key string, value any) bool {
if strings.HasPrefix(key, "#_") {
return true
}
variables[key] = value
return true
})
} else {
doc, err := parseQueryAST(query)
if err != nil {
return "", fmt.Errorf("could not parse graphql query: %w", err)
}
data.Iterate(func(key string, value any) bool {
if strings.HasPrefix(key, "#_") {
return true
}
applyInlineArgument(doc, key, value)
return true
})
body.Query = types.ToString(printer.Print(doc))
}
encoded, err := json.Marshal(body)
if err != nil {
return "", err
}
return string(encoded), nil
}
func collectInlineArguments(doc *ast.Document) map[string]any {
args := make(map[string]any)
for _, ref := range inlineArgumentRefs(doc) {
if containsVariable(ref.argument.Value) {
continue
}
args[ref.key] = astValueToGo(ref.argument.Value)
}
return args
}
func applyInlineArgument(doc *ast.Document, key string, value any) {
for _, ref := range inlineArgumentRefs(doc) {
if ref.key == key && !containsVariable(ref.argument.Value) {
ref.argument.Value = goValueToAST(value, ref.argument.Value)
return
}
}
}
func containsVariable(value ast.Value) bool {
switch typed := value.(type) {
case *ast.Variable:
return true
case *ast.ListValue:
for _, item := range typed.Values {
if containsVariable(item) {
return true
}
}
case *ast.ObjectValue:
for _, field := range typed.Fields {
if containsVariable(field.Value) {
return true
}
}
}
return false
}
type inlineArgumentRef struct {
key string
path string
argument *ast.Argument
}
func inlineArgumentRefs(doc *ast.Document) []inlineArgumentRef {
var refs []inlineArgumentRef
walkFields(doc, func(path string, field *ast.Field) {
for _, argument := range field.Arguments {
if argument.Name == nil {
continue
}
refs = append(refs, inlineArgumentRef{
path: path,
argument: argument,
})
}
})
counts := make(map[string]int)
for _, ref := range refs {
counts[ref.argument.Name.Value]++
}
for index := range refs {
name := refs[index].argument.Name.Value
refs[index].key = name
if counts[name] > 1 {
refs[index].key = refs[index].path + "." + name
}
}
keyCounts := make(map[string]int)
for _, ref := range refs {
keyCounts[ref.key]++
}
keyIndexes := make(map[string]int)
for index := range refs {
key := refs[index].key
keyIndexes[key]++
if keyCounts[key] > 1 {
refs[index].key = fmt.Sprintf("%s[%d]", key, keyIndexes[key])
}
}
return refs
}
func walkFields(doc *ast.Document, fn func(string, *ast.Field)) {
if doc == nil {
return
}
for _, def := range doc.Definitions {
switch typed := def.(type) {
case *ast.OperationDefinition:
prefix := ""
if typed.Name != nil {
prefix = typed.Name.Value
}
walkSelectionSet(typed.SelectionSet, prefix, fn)
case *ast.FragmentDefinition:
prefix := "fragment"
if typed.Name != nil {
prefix += "." + typed.Name.Value
}
walkSelectionSet(typed.SelectionSet, prefix, fn)
}
}
}
func walkSelectionSet(set *ast.SelectionSet, parentPath string, fn func(string, *ast.Field)) {
if set == nil {
return
}
fieldCounts := make(map[string]int)
for _, selection := range set.Selections {
if field, ok := selection.(*ast.Field); ok {
fieldCounts[fieldResponseName(field)]++
}
}
fieldIndexes := make(map[string]int)
for _, selection := range set.Selections {
switch typed := selection.(type) {
case *ast.Field:
name := fieldResponseName(typed)
fieldIndexes[name]++
if fieldCounts[name] > 1 {
name = fmt.Sprintf("%s[%d]", name, fieldIndexes[name])
}
path := name
if parentPath != "" {
path = parentPath + "." + name
}
fn(path, typed)
walkSelectionSet(typed.SelectionSet, path, fn)
case *ast.InlineFragment:
walkSelectionSet(typed.SelectionSet, parentPath, fn)
}
}
}
func fieldResponseName(field *ast.Field) string {
if field.Alias != nil {
return field.Alias.Value
}
if field.Name != nil {
return field.Name.Value
}
return "field"
}
func goValueToAST(value any, original ast.Value) ast.Value {
if enum, ok := original.(*ast.EnumValue); ok && enum.Value == "null" && value == nil {
return &ast.EnumValue{Kind: kinds.EnumValue, Value: "null"}
}
switch original.(type) {
case *ast.IntValue:
return &ast.IntValue{Kind: kinds.IntValue, Value: types.ToString(value)}
case *ast.FloatValue:
return &ast.FloatValue{Kind: kinds.FloatValue, Value: types.ToString(value)}
case *ast.EnumValue:
return &ast.EnumValue{Kind: kinds.EnumValue, Value: types.ToString(value)}
case *ast.StringValue:
return &ast.StringValue{Kind: kinds.StringValue, Value: types.ToString(value)}
}
switch v := value.(type) {
case string:
return &ast.StringValue{Kind: kinds.StringValue, Value: v}
case bool:
return &ast.BooleanValue{Kind: kinds.BooleanValue, Value: v}
case int:
return &ast.IntValue{Kind: kinds.IntValue, Value: fmt.Sprintf("%d", v)}
case int32:
return &ast.IntValue{Kind: kinds.IntValue, Value: fmt.Sprintf("%d", v)}
case int64:
return &ast.IntValue{Kind: kinds.IntValue, Value: fmt.Sprintf("%d", v)}
case float32:
return &ast.FloatValue{Kind: kinds.FloatValue, Value: fmt.Sprintf("%v", v)}
case float64:
// JSON numbers decode as float64; keep integers looking like ints when possible.
if v == float64(int64(v)) {
return &ast.IntValue{Kind: kinds.IntValue, Value: fmt.Sprintf("%d", int64(v))}
}
return &ast.FloatValue{Kind: kinds.FloatValue, Value: fmt.Sprintf("%v", v)}
case []any:
originalList, _ := original.(*ast.ListValue)
values := make([]ast.Value, 0, len(v))
for index, item := range v {
var originalItem ast.Value
if originalList != nil && index < len(originalList.Values) {
originalItem = originalList.Values[index]
}
values = append(values, goValueToAST(item, originalItem))
}
return &ast.ListValue{Kind: kinds.ListValue, Values: values}
case map[string]any:
originalObject, _ := original.(*ast.ObjectValue)
fields := make([]*ast.ObjectField, 0, len(v))
used := make(map[string]struct{}, len(v))
if originalObject != nil {
for _, field := range originalObject.Fields {
if field.Name == nil {
continue
}
name := field.Name.Value
value, ok := v[name]
if !ok {
continue
}
fields = append(fields, &ast.ObjectField{
Kind: kinds.ObjectField,
Name: &ast.Name{Kind: kinds.Name, Value: name},
Value: goValueToAST(value, field.Value),
})
used[name] = struct{}{}
}
}
names := make([]string, 0, len(v)-len(used))
for name := range v {
if _, ok := used[name]; !ok {
names = append(names, name)
}
}
sort.Strings(names)
for _, name := range names {
fields = append(fields, &ast.ObjectField{
Kind: kinds.ObjectField,
Name: &ast.Name{Kind: kinds.Name, Value: name},
Value: goValueToAST(v[name], nil),
})
}
return &ast.ObjectValue{Kind: kinds.ObjectValue, Fields: fields}
default:
return &ast.StringValue{Kind: kinds.StringValue, Value: types.ToString(v)}
}
}
func astValueToGo(value ast.Value) any {
switch v := value.(type) {
case *ast.StringValue:
return v.Value
case *ast.IntValue:
return v.Value
case *ast.FloatValue:
return v.Value
case *ast.BooleanValue:
return v.Value
case *ast.EnumValue:
if v.Value == "null" {
return nil
}
return v.Value
case *ast.ListValue:
out := make([]any, 0, len(v.Values))
for _, item := range v.Values {
out = append(out, astValueToGo(item))
}
return out
case *ast.ObjectValue:
out := make(map[string]any, len(v.Fields))
for _, field := range v.Fields {
if field.Name == nil {
continue
}
out[field.Name.Value] = astValueToGo(field.Value)
}
return out
default:
return nil
}
}
+292
View File
@@ -0,0 +1,292 @@
package dataformat
import (
"encoding/json"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
const (
graphqlInlineArgsBody = `{
"query": "query { jobs(jobType: \"front-end\") { id name } }"
}`
graphqlVariablesBody = `{
"query": "mutation ImportPaste($host: String!, $port: Int!, $path: String!, $scheme: String!) { importPaste(host: $host, port: $port, path: $path, scheme: $scheme) { result } }",
"variables": {
"host": "example.com",
"port": 80,
"path": "/robots.txt",
"scheme": "http"
}
}`
graphqlNestedVariablesBody = `{
"query": "mutation ($input: ImportInput!) { importPaste(input: $input) { result } }",
"operationName": "Import",
"variables": {
"input": {
"host": "example.com",
"path": "/robots.txt"
}
}
}`
)
func fuzzableKV(t *testing.T, decoded KV) map[string]any {
t.Helper()
out := make(map[string]any)
decoded.Iterate(func(key string, value any) bool {
if strings.HasPrefix(key, "#_") {
return true
}
out[key] = value
return true
})
return out
}
func TestGraphqlIsType(t *testing.T) {
g := NewGraphql()
require.True(t, g.IsType(graphqlInlineArgsBody))
require.True(t, g.IsType(graphqlVariablesBody))
require.False(t, g.IsType("not-json"))
require.False(t, g.IsType(`{"foo":"bar"}`))
require.False(t, g.IsType(`{"query":"not a graphql query"}`))
require.False(t, g.IsType(`{"variables":{"host":"example.com"}}`))
}
func TestGraphqlDecodeEncodeVariables(t *testing.T) {
g := NewGraphql()
decoded, err := g.Decode(graphqlVariablesBody)
require.NoError(t, err)
require.Equal(t, map[string]any{
"host": "example.com",
"port": float64(80),
"path": "/robots.txt",
"scheme": "http",
}, fuzzableKV(t, decoded))
decoded.Set("path", "/robots.txt; cat /etc/passwd")
encoded, err := g.Encode(decoded)
require.NoError(t, err)
roundTrip, err := g.Decode(encoded)
require.NoError(t, err)
require.Equal(t, "/robots.txt; cat /etc/passwd", fuzzableKV(t, roundTrip)["path"])
require.Contains(t, encoded, `"variables"`)
require.Contains(t, encoded, `/robots.txt; cat /etc/passwd`)
// Injection stays in variables JSON, query text still references $path only.
require.Contains(t, encoded, `path: $path`)
require.NotContains(t, encoded, `path: "/robots.txt; cat /etc/passwd"`)
}
func TestGraphqlDecodeEncodeEmptyVariables(t *testing.T) {
g := NewGraphql()
body := `{
"query": "query Lookup($input: String) { lookup(input: $input) { id } }",
"variables": {}
}`
decoded, err := g.Decode(body)
require.NoError(t, err)
require.Empty(t, fuzzableKV(t, decoded))
require.Equal(t, true, decoded.Get(graphqlMetaHasVariables))
encoded, err := g.Encode(decoded)
require.NoError(t, err)
require.Contains(t, encoded, `"variables":{}`)
require.Contains(t, encoded, `input: $input`)
require.NotContains(t, encoded, `input: ""`)
roundTrip, err := g.Decode(encoded)
require.NoError(t, err)
require.Equal(t, true, roundTrip.Get(graphqlMetaHasVariables))
require.Empty(t, fuzzableKV(t, roundTrip))
}
func TestGraphqlDecodeEncodeInlineArgs(t *testing.T) {
g := NewGraphql()
decoded, err := g.Decode(graphqlInlineArgsBody)
require.NoError(t, err)
require.Equal(t, map[string]any{"jobType": "front-end"}, fuzzableKV(t, decoded))
decoded.Set("jobType", "canary-payload")
encoded, err := g.Encode(decoded)
require.NoError(t, err)
roundTrip, err := g.Decode(encoded)
require.NoError(t, err)
require.Equal(t, map[string]any{"jobType": "canary-payload"}, fuzzableKV(t, roundTrip))
require.Contains(t, encoded, "canary-payload")
}
func TestGraphqlDecodeEncodeNullLiteral(t *testing.T) {
g := NewGraphql()
body := `{"query":"query { jobs(filter: null, nested: {value: null}) { id } }"}`
decoded, err := g.Decode(body)
require.NoError(t, err)
require.Contains(t, fuzzableKV(t, decoded), "filter")
require.Nil(t, decoded.Get("filter"))
encoded, err := g.Encode(decoded)
require.NoError(t, err)
require.Contains(t, encoded, "filter: null")
require.Contains(t, encoded, "nested: {value: null}")
roundTrip, err := g.Decode(encoded)
require.NoError(t, err)
require.Contains(t, fuzzableKV(t, roundTrip), "filter")
require.Nil(t, roundTrip.Get("filter"))
}
func TestGraphqlInlineArgsPreserveDefaultedVariable(t *testing.T) {
g := NewGraphql()
body := `{"query":"query Jobs($limit: Int = 10) { jobs(limit: $limit, status: ACTIVE) { id } }"}`
decoded, err := g.Decode(body)
require.NoError(t, err)
require.Equal(t, map[string]any{"status": "ACTIVE"}, fuzzableKV(t, decoded))
decoded.Set("status", "PAUSED")
encoded, err := g.Encode(decoded)
require.NoError(t, err)
require.Contains(t, encoded, "$limit: Int = 10")
require.Contains(t, encoded, "limit: $limit")
require.Contains(t, encoded, "status: PAUSED")
roundTrip, err := g.Decode(encoded)
require.NoError(t, err)
require.Equal(t, map[string]any{"status": "PAUSED"}, fuzzableKV(t, roundTrip))
}
func TestGraphqlDecodeEncodePreservesExtensions(t *testing.T) {
g := NewGraphql()
body := `{"query":"query { viewer { id } }","extensions":{"persistedQuery":{"version":1,"sha256Hash":"abc123"}}}`
decoded, err := g.Decode(body)
require.NoError(t, err)
encoded, err := g.Encode(decoded)
require.NoError(t, err)
var roundTrip map[string]any
require.NoError(t, json.Unmarshal([]byte(encoded), &roundTrip))
require.Equal(t, map[string]any{
"persistedQuery": map[string]any{
"version": float64(1),
"sha256Hash": "abc123",
},
}, roundTrip["extensions"])
}
func TestGraphqlInlineArgumentKeysIdentifyLocations(t *testing.T) {
g := NewGraphql()
body := `{"query":"query { users(first: 10) { id } posts(first: 20) { id } }"}`
decoded, err := g.Decode(body)
require.NoError(t, err)
require.Equal(t, map[string]any{
"users.first": "10",
"posts.first": "20",
}, fuzzableKV(t, decoded))
decoded.Set("users.first", "99")
encoded, err := g.Encode(decoded)
require.NoError(t, err)
require.Contains(t, encoded, "users(first: 99)")
require.Contains(t, encoded, "posts(first: 20)")
roundTrip, err := g.Decode(encoded)
require.NoError(t, err)
require.Equal(t, "99", roundTrip.Get("users.first"))
require.Equal(t, "20", roundTrip.Get("posts.first"))
}
func TestGraphqlInlineArgumentsPreserveLiteralTypes(t *testing.T) {
g := NewGraphql()
body := `{"query":"query { jobs(name: \"example\", status: ACTIVE, limit: 10, ratio: 1.5, flags: [ACTIVE, PAUSED], filter: {retries: 3, state: READY}) { id } }"}`
decoded, err := g.Decode(body)
require.NoError(t, err)
decoded.Set("name", "canary")
flags, ok := decoded.Get("flags").([]any)
require.True(t, ok)
flags[0] = "DISABLED"
decoded.Set("flags", flags)
filter, ok := decoded.Get("filter").(map[string]any)
require.True(t, ok)
filter["retries"] = "4"
decoded.Set("filter", filter)
encoded, err := g.Encode(decoded)
require.NoError(t, err)
require.Contains(t, encoded, `name: \"canary\"`)
require.Contains(t, encoded, "status: ACTIVE")
require.Contains(t, encoded, "limit: 10")
require.Contains(t, encoded, "ratio: 1.5")
require.Contains(t, encoded, "flags: [DISABLED, PAUSED]")
require.Contains(t, encoded, "filter: {retries: 4, state: READY}")
require.NotContains(t, encoded, `status: \"ACTIVE\"`)
require.NotContains(t, encoded, `limit: \"10\"`)
}
func TestGraphqlDecodeEncodeNestedVariables(t *testing.T) {
g := NewGraphql()
decoded, err := g.Decode(graphqlNestedVariablesBody)
require.NoError(t, err)
require.Equal(t, "Import", decoded.Get(graphqlMetaOperationName))
input, ok := fuzzableKV(t, decoded)["input"].(map[string]any)
require.True(t, ok)
require.Equal(t, "example.com", input["host"])
require.Equal(t, "/robots.txt", input["path"])
input["path"] = "/robots.txt' OR 1=1--"
decoded.Set("input", input)
encoded, err := g.Encode(decoded)
require.NoError(t, err)
require.Contains(t, encoded, `/robots.txt' OR 1=1--`)
require.Contains(t, encoded, `"operationName":"Import"`)
}
func TestGraphqlDoesNotTreatRegularJSONAsGraphql(t *testing.T) {
decoded, err := Decode(`{"foo":"bar","query":123}`)
require.NoError(t, err)
require.NotNil(t, decoded)
require.Equal(t, JSONDataFormat, decoded.DataFormat)
}
func TestGraphqlPreferredOverJSONInDecode(t *testing.T) {
decoded, err := Decode(graphqlVariablesBody)
require.NoError(t, err)
require.NotNil(t, decoded)
require.Equal(t, GraphqlDataFormat, decoded.DataFormat)
require.Equal(t, "example.com", decoded.Data.Get("host"))
}
func TestGraphqlEncodeRoundTripPreservesQueryWhenFuzzingVariables(t *testing.T) {
g := NewGraphql()
decoded, err := g.Decode(graphqlVariablesBody)
require.NoError(t, err)
originalQuery := decoded.Get(graphqlMetaQuery)
decoded.Set("host", "evil.example")
encoded, err := g.Encode(decoded)
require.NoError(t, err)
again, err := g.Decode(encoded)
require.NoError(t, err)
require.Equal(t, originalQuery, again.Get(graphqlMetaQuery))
require.Equal(t, "evil.example", again.Get("host"))
}
+9 -3
View File
@@ -76,6 +76,8 @@ type GeneratedRequest struct {
// Key is the key for the request
Key string
// FuzzedKeys contains the keys under test, including multiple mode.
FuzzedKeys []string
// Value is the value for the request
Value string
// OriginalValue is the original value for the request
@@ -309,8 +311,9 @@ func (rule *Rule) evaluateVars(input string) (string, error) {
}
result, err := render.Render(render.Input{
Text: input,
Values: data,
Options: rule.options.GetOptions(),
Text: input,
Values: data,
})
if err != nil {
return input, err
@@ -339,6 +342,7 @@ func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, intera
}
got, err := render.Render(render.Input{
Options: rule.options.GetOptions(),
Text: fmt.Sprint(v),
Values: data,
Interactsh: rule.options.Interactsh,
@@ -412,6 +416,7 @@ func (rule *Rule) executeRuleValues(input *ExecuteRuleInput, ruleComponent compo
// if mode is multiple now build and execute it
if rule.modeType == multipleModeType {
var fuzzedKeys []string
rule.Fuzz.KV.Iterate(func(key, value string) bool {
var evaluated string
var err error
@@ -426,6 +431,7 @@ func (rule *Rule) executeRuleValues(input *ExecuteRuleInput, ruleComponent compo
if err := ruleComponent.SetValue(key, evaluated); err != nil {
return true
}
fuzzedKeys = append(fuzzedKeys, key)
return true
})
@@ -439,7 +445,7 @@ func (rule *Rule) executeRuleValues(input *ExecuteRuleInput, ruleComponent compo
return err
}
if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, "", "", "", "", "", ""); gotErr != nil {
if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, "", "", "", "", "", "", fuzzedKeys...); gotErr != nil {
return gotErr
}
}
+12 -3
View File
@@ -50,6 +50,7 @@ func (rule *Rule) executePartComponent(input *ExecuteRuleInput, payload ValueOrK
// this supports both single and multiple [ruleType] modes
// i.e if component has multiple values, they can be replaced once or all depending on mode
func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadStr, originalPayload string, ruleComponent component.Component) error {
var fuzzedKeys []string
finalErr := ruleComponent.Iterate(func(key string, value interface{}) error {
valueStr := types.ToString(value)
if !rule.matchKeyOrValue(key, valueStr) {
@@ -77,6 +78,9 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS
// gologger.Warning().Msgf("could not set value due to format restriction original(%s, %s[%T]) , new(%s,%s[%T])", key, valueStr, value, key, evaluated, evaluated)
return nil
}
if rule.modeType == multipleModeType {
fuzzedKeys = append(fuzzedKeys, key)
}
if rule.modeType == singleModeType {
req, err := ruleComponent.Rebuild()
@@ -109,7 +113,7 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS
return err
}
if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, "", "", "", "", "", ""); qerr != nil {
if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, "", "", "", "", "", "", fuzzedKeys...); qerr != nil {
err = qerr
return err
}
@@ -149,7 +153,7 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu
return err
}
if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key, value, "", "", "", ""); qerr != nil {
if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key, value, "", "", "", "", key); qerr != nil {
return qerr
}
@@ -168,7 +172,7 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu
}
// execWithInput executes a rule with input via callback
func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.Request, interactURLs []string, component component.Component, parameter, parameterValue, originalPayload, originalValue, key, value string) error {
func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.Request, interactURLs []string, component component.Component, parameter, parameterValue, originalPayload, originalValue, key, value string, fuzzedKeys ...string) error {
// If the parameter is a number, replace it with the parameter value
// or if the parameter is empty and the parameter value is not empty
// replace it with the parameter value
@@ -188,6 +192,9 @@ func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.
return nil
}
}
if len(fuzzedKeys) == 0 && key != "" {
fuzzedKeys = []string{key}
}
request := GeneratedRequest{
Request: httpReq,
InteractURLs: interactURLs,
@@ -195,6 +202,7 @@ func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.
Component: component,
Parameter: actualParameter,
Key: key,
FuzzedKeys: append([]string(nil), fuzzedKeys...),
Value: value,
OriginalValue: originalValue,
OriginalPayload: originalPayload,
@@ -214,6 +222,7 @@ func (rule *Rule) executeEvaluate(input *ExecuteRuleInput, _, value, payload str
}, rule.options.Options.Vars.AsMap(), input.Values)
result, err := render.Render(render.Input{
Options: rule.options.GetOptions(),
Text: payload,
Values: values,
Interactsh: rule.options.Interactsh,
+26
View File
@@ -28,6 +28,32 @@ func NewInputProvider(opts InputOptions) (InputProvider, error)
This function returns a InputProvider based by appropriately selecting input provider based on the input format (i.e. either list or http) and returns the provider that can handle that input format.
## Per-target profiles
A list input line starting with `{` is a target line: a JSON object binding one target to its own template selection, so a single scan runs a different profile on each target. The keys mirror template profiles and CLI flags:
```jsonl
{"target": "https://a.example", "tags": ["thinkphp"]}
{"target": "https://b.example", "profile": "wordpress", "severity": ["critical", "high"]}
{"target": "https://c.example", "tech": ["Apache Tomcat", "Java"]}
{"target": "10.0.0.0/24", "type": "http", "exclude-tags": "dos"}
https://d.example
```
| Key | Selects |
| --- | --- |
| `target` | the target (URL, host, IP, CIDR or ASN), required |
| `profile` | a template profile ID from the templates `profiles` directory (paths are rejected); the line's other keys override it |
| `templates` | template paths, relative to the templates directory |
| `tags`, `exclude-tags`, `include-tags`, `author`, `template-id`, `exclude-id`, `severity`, `exclude-severity`, `type`, `exclude-type` | same as the CLI flags |
| `tech` | technologies the target runs: templates bound to another product (`metadata.product`) are skipped, every other template runs |
- Global flags apply first; a target line only narrows them. Plain targets run every template.
- A target listed more than once runs the union of its selections, and is scanned once.
- When every target has a selection, only the templates some target selects are loaded.
- Unknown keys fail the run rather than widening the scan. Profile keys that configure the whole scan, such as `code` or `var`, are ignored per target.
- Not supported with `-automatic-scan`. Global matchers still apply to every target.
## Formats
Fuzzing and DAST take the request shape (method, body, params) from the input, not from CLI flags, so a bare URL is always fuzzed as a `GET`. To fuzz anything else, feed a request-shaped input with `-im`:
+15 -3
View File
@@ -8,6 +8,8 @@ import (
"strings"
"testing"
"time"
"github.com/projectdiscovery/retryablehttp-go"
)
func TestOpenAPIDownloader_SupportedExtensions(t *testing.T) {
@@ -181,7 +183,9 @@ func TestOpenAPIDownloader_Download_InvalidJSON(t *testing.T) {
func TestOpenAPIDownloader_Download_Timeout(t *testing.T) {
// Create mock server with delay
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(35 * time.Second) // Longer than 30 second timeout
// Outlasts the client timeout below. Waiting out the real 30s default
// cost this test 35s, which was a seventh of the whole unit suite.
time.Sleep(500 * time.Millisecond)
if err := json.NewEncoder(w).Encode(map[string]interface{}{"test": "data"}); err != nil {
http.Error(w, "failed to encode response", http.StatusInternalServerError)
}
@@ -200,9 +204,17 @@ func TestOpenAPIDownloader_Download_Timeout(t *testing.T) {
}()
downloader := &OpenAPIDownloader{}
_, err = downloader.Download(server.URL+"/openapi.json", tmpDir, nil)
client := retryablehttp.NewClient(retryablehttp.DefaultOptionsSingle)
client.HTTPClient.Timeout = 100 * time.Millisecond
_, err = downloader.Download(server.URL+"/openapi.json", tmpDir, client)
if err == nil {
t.Error("Expected timeout error, but got none")
t.Fatal("Expected timeout error, but got none")
}
// Assert it timed out rather than merely erroring: the spec this server
// returns is invalid, so a test that only checks for any error passes even
// when the timeout never fires.
if !os.IsTimeout(err) && !strings.Contains(err.Error(), "Timeout") {
t.Fatalf("Expected a timeout error, got: %v", err)
}
}
+15 -3
View File
@@ -9,6 +9,8 @@ import (
"testing"
"time"
"github.com/projectdiscovery/retryablehttp-go"
"gopkg.in/yaml.v3"
)
@@ -268,7 +270,9 @@ func TestSwaggerDownloader_Download_InvalidYAML(t *testing.T) {
func TestSwaggerDownloader_Download_Timeout(t *testing.T) {
// Create mock server with delay
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(35 * time.Second) // Longer than 30 second timeout
// Outlasts the client timeout below. Waiting out the real 30s default
// cost this test 35s, which was a seventh of the whole unit suite.
time.Sleep(500 * time.Millisecond)
if err := json.NewEncoder(w).Encode(map[string]interface{}{"test": "data"}); err != nil {
http.Error(w, "failed to encode response", http.StatusInternalServerError)
}
@@ -287,9 +291,17 @@ func TestSwaggerDownloader_Download_Timeout(t *testing.T) {
}()
downloader := &SwaggerDownloader{}
_, err = downloader.Download(server.URL+"/swagger.json", tmpDir, nil)
client := retryablehttp.NewClient(retryablehttp.DefaultOptionsSingle)
client.HTTPClient.Timeout = 100 * time.Millisecond
_, err = downloader.Download(server.URL+"/swagger.json", tmpDir, client)
if err == nil {
t.Error("Expected timeout error, but got none")
t.Fatal("Expected timeout error, but got none")
}
// Assert it timed out rather than merely erroring: the spec this server
// returns is invalid, so a test that only checks for any error passes even
// when the timeout never fires.
if !os.IsTimeout(err) && !strings.Contains(err.Error(), "Timeout") {
t.Fatalf("Expected a timeout error, got: %v", err)
}
}
+198
View File
@@ -0,0 +1,198 @@
package normalize
import (
"regexp"
"sort"
"strconv"
"strings"
"sync"
urlutil "github.com/projectdiscovery/utils/url"
)
// Placeholders a structurally variable path segment collapses to. They are
// spelled out rather than hashed so a skipped-target log reads plainly.
const (
placeholderNum = "{num}"
placeholderUUID = "{uuid}"
placeholderHash = "{hash}"
placeholderDate = "{date}"
placeholderVar = "{var}"
)
var (
numericSegment = regexp.MustCompile(`^\d+$`)
uuidSegment = regexp.MustCompile(`(?i)^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
hashSegment = regexp.MustCompile(`(?i)^[0-9a-f]{16,}$`)
// a date split across path positions is already caught as numeric segments
dateSegment = regexp.MustCompile(`^\d{4}-\d{2}(-\d{2})?$`)
)
// classifySegment replaces a path segment that identifies a particular record
// with a placeholder, leaving segments that name a route alone.
func classifySegment(segment string) string {
switch {
case segment == "":
return segment
case numericSegment.MatchString(segment):
return placeholderNum
case uuidSegment.MatchString(segment):
return placeholderUUID
case dateSegment.MatchString(segment):
return placeholderDate
case hashSegment.MatchString(segment):
return placeholderHash
}
return segment
}
// DefaultCollapseAfter is how many distinct values a single path position may
// hold on one host before the position itself is treated as an identifier. It
// catches slugs and names, which no pattern can recognise by shape alone.
const DefaultCollapseAfter = 20
// Fingerprinter groups URLs by the shape of their path and query, so a crawl
// that produced ten thousand product pages is recognised as one pattern.
//
// It is stateful by design: whether a path position is an identifier depends on
// how many distinct values that position has taken on that host, which is only
// knowable across the whole input.
type Fingerprinter struct {
// MaxPerPattern is how many targets to keep per pattern. Zero keeps every
// target, which is the default: dropping one is a target never scanned.
MaxPerPattern int
// CollapseAfter is the distinct-value threshold for a path position.
CollapseAfter int
mu sync.Mutex
kept map[string]int
position map[string]map[string]struct{}
}
// NewFingerprinter returns a fingerprinter keeping at most maxPerPattern
// targets for each pattern. A maxPerPattern of zero disables grouping.
func NewFingerprinter(maxPerPattern int) *Fingerprinter {
return &Fingerprinter{
MaxPerPattern: maxPerPattern,
CollapseAfter: DefaultCollapseAfter,
kept: make(map[string]int),
position: make(map[string]map[string]struct{}),
}
}
// Enabled reports whether the fingerprinter drops anything.
func (f *Fingerprinter) Enabled() bool {
return f != nil && f.MaxPerPattern > 0
}
// Accept reports whether a target should be scanned, recording it against its
// pattern. A value that is not a URL is always accepted: it has no path to
// group by, and guessing would lose targets.
func (f *Fingerprinter) Accept(rawURL string) bool {
if !f.Enabled() {
return true
}
pattern, ok := f.pattern(rawURL)
if !ok {
return true
}
f.mu.Lock()
defer f.mu.Unlock()
if f.kept[pattern] >= f.MaxPerPattern {
return false
}
f.kept[pattern]++
return true
}
// Pattern returns the structural fingerprint of a URL, for tests and logging.
func (f *Fingerprinter) Pattern(rawURL string) string {
pattern, _ := f.pattern(rawURL)
return pattern
}
func (f *Fingerprinter) pattern(rawURL string) (string, bool) {
trimmed := strings.TrimSpace(rawURL)
if idx := strings.Index(trimmed, "://"); idx > 0 {
trimmed = strings.ToLower(trimmed[:idx]) + trimmed[idx:]
}
parsed, err := urlParse(trimmed)
if err != nil || parsed == nil || parsed.Host == "" {
return "", false
}
host := strings.ToLower(parsed.Host)
segments := strings.Split(strings.Trim(parsed.Path, "/"), "/")
shaped := make([]string, 0, len(segments))
for index, segment := range segments {
parent := strings.Join(segments[:index], "/")
shaped = append(shaped, f.shapeSegment(host, parent, index, segment))
}
var builder strings.Builder
builder.WriteString(host)
builder.WriteString("/")
builder.WriteString(strings.Join(shaped, "/"))
if keys := queryKeys(parsed); len(keys) > 0 {
// names without values: two searches differ by what was searched for,
// not by the shape of the request
builder.WriteString("?")
builder.WriteString(strings.Join(keys, "&"))
}
return builder.String(), true
}
// shapeSegment classifies a segment by shape, then by how many distinct values
// the same position has already taken under this parent path on this host.
// The parent is part of the key so a busy /blog/<slug> does not also collapse
// /admin/<name> just because both sit at the same index.
func (f *Fingerprinter) shapeSegment(host, parent string, index int, segment string) string {
if shaped := classifySegment(segment); shaped != segment {
return shaped
}
if f.CollapseAfter <= 0 || segment == "" {
return segment
}
key := host + "\x00" + parent + "\x00" + itoa(index)
f.mu.Lock()
values, ok := f.position[key]
if !ok {
values = make(map[string]struct{})
f.position[key] = values
}
if len(values) < f.CollapseAfter {
values[segment] = struct{}{}
}
collapsed := len(values) >= f.CollapseAfter
f.mu.Unlock()
if collapsed {
return placeholderVar
}
return segment
}
// urlParse and queryKeys keep the url library dependency in one place.
func urlParse(raw string) (*urlutil.URL, error) {
return urlutil.Parse(raw)
}
// queryKeys returns the query parameter names, sorted, with values dropped.
func queryKeys(parsed *urlutil.URL) []string {
if parsed.Params == nil || parsed.Params.IsEmpty() {
return nil
}
var keys []string
parsed.Params.Iterate(func(key string, _ []string) bool {
if !IsTrackingParam(key) {
keys = append(keys, key)
}
return true
})
sort.Strings(keys)
return keys
}
func itoa(value int) string { return strconv.Itoa(value) }
+101
View File
@@ -0,0 +1,101 @@
package normalize
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
)
func TestPatternGroupsIdentifiers(t *testing.T) {
f := NewFingerprinter(1)
base := f.Pattern("http://acme.test/user/1/profile")
for _, raw := range []string{
"http://acme.test/user/2/profile",
"http://acme.test/user/99999/profile",
} {
require.Equal(t, base, f.Pattern(raw), "numeric ids are the same shape")
}
require.Contains(t, base, placeholderNum)
}
func TestPatternGroupsUUIDsHashesAndDates(t *testing.T) {
f := NewFingerprinter(1)
require.Contains(t, f.Pattern("http://acme.test/o/3f2504e0-4f89-11d3-9a0c-0305e82c3301"), placeholderUUID)
require.Contains(t, f.Pattern("http://acme.test/a/d41d8cd98f00b204e9800998ecf8427e"), placeholderHash)
require.Contains(t, f.Pattern("http://acme.test/posts/2024-05-01"), placeholderDate)
// a date spread across positions is already grouped as numeric segments
require.Equal(t, "acme.test/posts/{num}/{num}", f.Pattern("http://acme.test/posts/2024/05"))
}
func TestPatternKeepsRouteNames(t *testing.T) {
f := NewFingerprinter(1)
require.NotEqual(t, f.Pattern("http://acme.test/admin"), f.Pattern("http://acme.test/login"),
"route names are the structure, not identifiers within it")
}
func TestPatternDropsQueryValuesButKeepsSortedNames(t *testing.T) {
f := NewFingerprinter(1)
a := f.Pattern("http://acme.test/search?q=admin&page=2")
b := f.Pattern("http://acme.test/search?page=9&q=root")
require.Equal(t, a, b, "same parameters in a different order with different values is one shape")
require.Contains(t, a, "page&q")
require.NotContains(t, a, "admin")
require.NotEqual(t, a, f.Pattern("http://acme.test/search?q=admin"), "a different parameter set is a different shape")
}
func TestPatternCollapsesHighCardinalityPosition(t *testing.T) {
f := NewFingerprinter(1)
f.CollapseAfter = 5
// slugs have no recognisable shape, so the position itself is what gives
// them away once it has held enough distinct values
for i := 0; i < 5; i++ {
f.Pattern(fmt.Sprintf("http://acme.test/blog/%s-post", []string{"alpha", "beta", "gamma", "delta", "epsilon"}[i]))
}
require.Contains(t, f.Pattern("http://acme.test/blog/zeta-post"), placeholderVar)
// a different host is tracked separately
require.NotContains(t, f.Pattern("http://other.test/blog/zeta-post"), placeholderVar)
}
func TestCollapseStaysInsideTheParentPath(t *testing.T) {
f := NewFingerprinter(1)
f.CollapseAfter = 3
require.Equal(t, "acme.test/blog/one", f.Pattern("http://acme.test/blog/one"))
require.Equal(t, "acme.test/blog/two", f.Pattern("http://acme.test/blog/two"))
require.Equal(t, "acme.test/blog/{var}", f.Pattern("http://acme.test/blog/three"))
// same index, different parent: a busy /blog/<slug> must not collapse /admin/<name>
require.Equal(t, "acme.test/admin/a", f.Pattern("http://acme.test/admin/a"))
require.Equal(t, "acme.test/admin/b", f.Pattern("http://acme.test/admin/b"))
require.True(t, f.Accept("http://acme.test/admin/a"))
require.True(t, f.Accept("http://acme.test/admin/b"), "a different parent still has its own cap")
}
func TestAcceptKeepsUpToTheCap(t *testing.T) {
f := NewFingerprinter(2)
require.True(t, f.Accept("http://acme.test/user/1"))
require.True(t, f.Accept("http://acme.test/user/2"))
require.False(t, f.Accept("http://acme.test/user/3"), "past the cap the pattern is already covered")
require.True(t, f.Accept("http://acme.test/admin"), "a different pattern has its own budget")
}
// Off by default: dropping a target means never scanning it, so it has to be
// asked for.
func TestDisabledFingerprinterKeepsEverything(t *testing.T) {
f := NewFingerprinter(0)
require.False(t, f.Enabled())
for i := 0; i < 100; i++ {
require.True(t, f.Accept(fmt.Sprintf("http://acme.test/user/%d", i)))
}
var nilF *Fingerprinter
require.False(t, nilF.Enabled())
require.True(t, nilF.Accept("http://acme.test/user/1"))
}
func TestAcceptKeepsNonURLTargets(t *testing.T) {
f := NewFingerprinter(1)
require.True(t, f.Accept("acme.test"))
require.True(t, f.Accept("not a url"))
require.True(t, f.Accept("not a url"), "a value with no pattern is never grouped away")
}
+159
View File
@@ -0,0 +1,159 @@
// Package normalize collapses URLs that address the same resource onto one
// form, so a target list built by a crawler does not run every template several
// times over for what is effectively the same target.
//
// Only differences that cannot change the response are removed: host case, the
// fragment (never sent to the server), a default port, a trailing slash, and
// well known tracking parameters. Path case is left alone, because paths are
// case sensitive on most servers.
package normalize
import (
"strings"
urlutil "github.com/projectdiscovery/utils/url"
)
// trackingParams are query keys added by ad and analytics platforms. They are
// echoed back at most and never select a different resource, so two URLs that
// differ only by these are the same target. The list is deliberately limited to
// vendor prefixed names: a generic key like "ref" or "id" can be functional.
var trackingParams = map[string]struct{}{
"utm_source": {},
"utm_medium": {},
"utm_campaign": {},
"utm_term": {},
"utm_content": {},
"utm_id": {},
"utm_reader": {},
"utm_name": {},
"gclid": {},
"gbraid": {},
"wbraid": {},
"dclid": {},
"fbclid": {},
"msclkid": {},
"mc_cid": {},
"mc_eid": {},
"igshid": {},
"twclid": {},
"ttclid": {},
"yclid": {},
"_ga": {},
"_gl": {},
}
// IsTrackingParam reports whether a query key is a known tracking parameter.
func IsTrackingParam(key string) bool {
_, ok := trackingParams[strings.ToLower(key)]
return ok
}
// URL returns the canonical form of raw. A value that cannot be parsed, or that
// carries no host, is returned unchanged: the input provider accepts things
// that are not URLs at all, and guessing at those would lose targets.
func URL(raw string) string {
trimmed := strings.TrimSpace(raw)
if trimmed == "" {
return raw
}
// The scheme is case insensitive per RFC 3986, but the parser rejects one
// that is not lowercase, so fold it before handing the value over.
if idx := strings.Index(trimmed, "://"); idx > 0 {
trimmed = strings.ToLower(trimmed[:idx]) + trimmed[idx:]
}
parsed, err := urlutil.Parse(trimmed)
if err != nil || parsed == nil || parsed.Host == "" {
return raw
}
parsed.Scheme = strings.ToLower(parsed.Scheme)
parsed.Host = strings.ToLower(parsed.Host)
parsed.Fragment = ""
if host, port, ok := splitHostPort(parsed.Host); ok && isDefaultPort(parsed.Scheme, port) {
parsed.Host = host
}
stripTrackingParams(parsed)
// A trailing slash addresses the same resource as its absence for every
// path but the root, where "/" is the path.
if path := parsed.Path; len(path) > 1 && strings.HasSuffix(path, "/") {
parsed.Path = strings.TrimRight(path, "/")
}
return parsed.String()
}
// stripTrackingParams removes known tracking keys, leaving every other
// parameter and its value untouched.
func stripTrackingParams(parsed *urlutil.URL) {
if parsed.Params == nil || parsed.Params.IsEmpty() {
return
}
// collected first: deleting while iterating would mutate the order under us
var drop []string
parsed.Params.Iterate(func(key string, _ []string) bool {
if IsTrackingParam(key) {
drop = append(drop, key)
}
return true
})
for _, key := range drop {
parsed.Params.Del(key)
}
parsed.Update()
}
// splitHostPort separates a host and port, tolerating bracketed IPv6 literals.
func splitHostPort(host string) (string, string, bool) {
if strings.HasPrefix(host, "[") {
end := strings.LastIndex(host, "]")
if end < 0 {
return host, "", false
}
rest := host[end+1:]
if !strings.HasPrefix(rest, ":") {
return host, "", false
}
return host[:end+1], rest[1:], true
}
idx := strings.LastIndex(host, ":")
if idx < 0 {
return host, "", false
}
return host[:idx], host[idx+1:], true
}
func isDefaultPort(scheme, port string) bool {
return (scheme == "http" && port == "80") || (scheme == "https" && port == "443")
}
// Origin returns the scheme and host a URL addresses, which is what a template
// built only from {{RootURL}} resolves against. It returns "" when the value
// carries no host, so callers can fall back to treating the target as unique.
func Origin(raw string) string {
trimmed := strings.TrimSpace(raw)
if trimmed == "" {
return ""
}
if idx := strings.Index(trimmed, "://"); idx > 0 {
trimmed = strings.ToLower(trimmed[:idx]) + trimmed[idx:]
}
parsed, err := urlutil.Parse(trimmed)
if err != nil || parsed == nil || parsed.Host == "" {
return ""
}
scheme := strings.ToLower(parsed.Scheme)
host := strings.ToLower(parsed.Host)
if h, port, ok := splitHostPort(host); ok && isDefaultPort(scheme, port) {
host = h
}
if scheme == "" {
return host
}
return scheme + "://" + host
}
+92
View File
@@ -0,0 +1,92 @@
package normalize
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestURLCollapsesEquivalentForms(t *testing.T) {
same := "http://acme.test/page"
for _, raw := range []string{
"http://acme.test/page",
"http://ACME.test/page",
"HTTP://acme.test/page",
"http://acme.test:80/page",
"http://acme.test/page/",
"http://acme.test/page#section",
"http://acme.test/page?utm_source=news&utm_campaign=q4",
} {
require.Equal(t, same, URL(raw), "%q addresses the same resource", raw)
}
}
func TestURLKeepsDifferencesThatMatter(t *testing.T) {
// path case is significant on most servers, so it is left alone
require.NotEqual(t, URL("http://acme.test/page"), URL("http://acme.test/PAGE"))
// a non-default port is part of the target
require.NotEqual(t, URL("http://acme.test/page"), URL("http://acme.test:8080/page"))
// so is the scheme
require.NotEqual(t, URL("http://acme.test/page"), URL("https://acme.test/page"))
// and a functional query parameter
require.NotEqual(t, URL("http://acme.test/page"), URL("http://acme.test/page?id=1"))
}
func TestURLKeepsFunctionalParamsWhenStrippingTracking(t *testing.T) {
got := URL("http://acme.test/search?q=admin&utm_source=news&page=2")
require.Contains(t, got, "q=admin")
require.Contains(t, got, "page=2")
require.NotContains(t, got, "utm_source")
}
func TestURLKeepsDefaultPortForOtherSchemes(t *testing.T) {
// 443 is only the default for https, so on http it is a real port
require.Contains(t, URL("http://acme.test:443/page"), ":443")
require.NotContains(t, URL("https://acme.test:443/page"), ":443")
}
func TestURLKeepsRootPath(t *testing.T) {
// "/" is the path at the root, not a trailing slash to trim
require.Equal(t, URL("http://acme.test/"), URL("http://acme.test/"))
require.NotEmpty(t, URL("http://acme.test/"))
}
func TestURLHandlesIPv6(t *testing.T) {
require.Equal(t, URL("http://[::1]:8080/page"), URL("http://[::1]:8080/page/"))
require.Contains(t, URL("http://[::1]:8080/page"), "[::1]:8080")
// the default port still goes
require.NotContains(t, URL("http://[::1]:80/page"), ":80")
}
// The provider accepts bare hosts, ip:port and other non-URL values. Guessing
// at those would lose targets, so they come back untouched.
func TestURLLeavesUnparsableInputAlone(t *testing.T) {
for _, raw := range []string{"", " ", "not a url", "acme.test", "10.0.0.1:8080"} {
require.Equal(t, raw, URL(raw))
}
}
func TestOriginIgnoresPathQueryAndDefaultPort(t *testing.T) {
same := "https://example.com"
for _, raw := range []string{
"https://example.com/a",
"https://EXAMPLE.com/b?q=1#frag",
"https://example.com:443/c",
} {
require.Equal(t, same, Origin(raw), "%q is the same origin", raw)
}
require.NotEqual(t, Origin("https://example.com/a"), Origin("http://example.com/a"))
require.NotEqual(t, Origin("https://example.com/a"), Origin("https://other.test/a"))
require.NotEqual(t, Origin("https://example.com/a"), Origin("https://example.com:8443/a"))
require.Equal(t, "example.com", Origin("example.com"))
require.Empty(t, Origin(""))
require.Empty(t, Origin("not a url"))
}
func TestIsTrackingParam(t *testing.T) {
require.True(t, IsTrackingParam("utm_source"))
require.True(t, IsTrackingParam("UTM_SOURCE"))
require.True(t, IsTrackingParam("fbclid"))
require.False(t, IsTrackingParam("id"))
require.False(t, IsTrackingParam("ref"), "a generic name can be functional")
}
+136 -45
View File
@@ -20,6 +20,10 @@ import (
"github.com/projectdiscovery/hmap/filekv"
"github.com/projectdiscovery/hmap/store/hybrid"
"github.com/projectdiscovery/mapcidr/asn"
"github.com/projectdiscovery/nuclei/v3/internal/configuration"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/input/normalize"
"github.com/projectdiscovery/nuclei/v3/pkg/input/targetprofile"
providerTypes "github.com/projectdiscovery/nuclei/v3/pkg/input/types"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
@@ -39,15 +43,23 @@ const DefaultMaxDedupeItemsCount = 10000
// ListInputProvider is a hmap/filekv backed nuclei ListInputProvider provider
// it supports list type of input ex: urls,file,stdin,uncover,etc. (i.e just url not complete request/response)
type ListInputProvider struct {
ipOptions *ipOptions
inputCount int64
excludedCount int64
dupeCount int64
skippedCount int64
hostMap *hybrid.HybridMap
excludedHosts map[string]struct{}
hostMapStream *filekv.FileDB
hostMapStreamOnce sync.Once
ipOptions *ipOptions
inputCount int64
excludedCount int64
dupeCount int64
skippedCount int64
patternSkippedCount int64
hostMap *hybrid.HybridMap
excludedHosts map[string]struct{}
hostMapStream *filekv.FileDB
hostMapStreamOnce sync.Once
profiles *targetprofile.Registry
// normalizeURLs collapses targets that address the same resource onto one
// form before they are deduplicated.
normalizeURLs bool
// patterns caps how many targets of one structural shape are kept. Nil or
// disabled keeps every target.
patterns *normalize.Fingerprinter
sync.Once
}
@@ -71,13 +83,16 @@ func New(opts *Options) (*ListInputProvider, error) {
}
input := &ListInputProvider{
hostMap: hm,
hostMap: hm,
normalizeURLs: !options.DisableURLNormalization,
patterns: normalize.NewFingerprinter(options.MaxURLsPerPattern),
ipOptions: &ipOptions{
ScanAllIPs: options.ScanAllIPs,
IPV4: sliceutil.Contains(options.IPVersion, "4"),
IPV6: sliceutil.Contains(options.IPVersion, "6"),
},
excludedHosts: make(map[string]struct{}),
profiles: targetprofile.NewRegistry(configuration.TemplatesDirectory(options, config.DefaultConfig)),
}
if options.Stream {
fkvOptions := filekv.DefaultOptions
@@ -105,6 +120,9 @@ func New(opts *Options) (*ListInputProvider, error) {
if input.skippedCount > 0 {
gologger.Info().Msgf("Number of hosts skipped from input due to exclusion: %d", input.skippedCount)
}
if input.patternSkippedCount > 0 {
gologger.Info().Msgf("Supplied input was limited by max urls per pattern (%d removed).", input.patternSkippedCount)
}
return input, nil
}
@@ -122,9 +140,14 @@ func (i *ListInputProvider) Iterate(callback func(value *contextargs.MetaInput)
}
})
}
callbackFunc := func(k, _ []byte) error {
callbackFunc := func(k, v []byte) error {
metaInput := contextargs.NewMetaInput()
if err := metaInput.Unmarshal(string(k)); err != nil {
// v holds the target as supplied when the key is a collapsed form
raw := string(k)
if len(v) > 0 {
raw = string(v)
}
if err := metaInput.Unmarshal(raw); err != nil {
return err
}
if !callback(metaInput) {
@@ -139,8 +162,18 @@ func (i *ListInputProvider) Iterate(callback func(value *contextargs.MetaInput)
}
}
// TargetProfiles returns the per-target template selections read from input.
func (i *ListInputProvider) TargetProfiles() *targetprofile.Registry {
return i.profiles
}
// Set normalizes and stores passed input values
func (i *ListInputProvider) Set(executionId string, value string) {
i.set(executionId, value, nil)
}
// set stores value bound to selection; a nil selection runs every template.
func (i *ListInputProvider) set(executionId string, value string, selection *targetprofile.Selection) {
URL := strings.TrimSpace(value)
if URL == "" {
return
@@ -156,7 +189,7 @@ func (i *ListInputProvider) Set(executionId string, value string) {
})
metaInput := contextargs.NewMetaInput()
metaInput.Input = URL
i.setItem(metaInput)
i.setItem(metaInput, selection)
return
}
@@ -164,7 +197,7 @@ func (i *ListInputProvider) Set(executionId string, value string) {
if iputil.IsIP(urlx.Hostname()) {
metaInput := contextargs.NewMetaInput()
metaInput.Input = URL
i.setItem(metaInput)
i.setItem(metaInput, selection)
return
}
@@ -192,7 +225,7 @@ func (i *ListInputProvider) Set(executionId string, value string) {
metaInput := contextargs.NewMetaInput()
metaInput.Input = URL
metaInput.CustomIP = ip
i.setItem(metaInput)
i.setItem(metaInput, selection)
}
return
} else {
@@ -230,10 +263,10 @@ func (i *ListInputProvider) Set(executionId string, value string) {
if ip != "" {
metaInput.Input = URL
metaInput.CustomIP = ip
i.setItem(metaInput)
i.setItem(metaInput, selection)
} else {
metaInput.Input = URL
i.setItem(metaInput)
i.setItem(metaInput, selection)
}
}
}
@@ -281,23 +314,18 @@ func (i *ListInputProvider) initializeInputSources(opts *Options) error {
// Handle targets flags
for _, target := range options.Targets {
switch {
case iputil.IsCIDR(target):
ips := expand.CIDR(target)
i.addTargets(options.ExecutionId, ips)
case asn.IsASN(target):
ips := expand.ASN(target)
i.addTargets(options.ExecutionId, ips)
default:
i.Set(options.ExecutionId, target)
if err := i.addInput(options.ExecutionId, target); err != nil {
return err
}
}
// Handle stdin
if options.Stdin {
i.scanInputFromReader(
if err := i.scanInputFromReader(
options.ExecutionId,
readerutil.TimeoutReader{Reader: os.Stdin, Timeout: time.Duration(options.InputReadTimeout)})
readerutil.TimeoutReader{Reader: os.Stdin, Timeout: time.Duration(options.InputReadTimeout)}); err != nil {
return err
}
}
// Handle target file
@@ -310,8 +338,11 @@ func (i *ListInputProvider) initializeInputSources(opts *Options) error {
}
}
if input != nil {
i.scanInputFromReader(options.ExecutionId, input)
err := i.scanInputFromReader(options.ExecutionId, input)
_ = input.Close()
if err != nil {
return err
}
}
}
if options.Uncover && options.UncoverQuery != nil {
@@ -356,21 +387,38 @@ func (i *ListInputProvider) initializeInputSources(opts *Options) error {
}
// scanInputFromReader scans a line of input from reader and passes it for storage
func (i *ListInputProvider) scanInputFromReader(executionId string, reader io.Reader) {
func (i *ListInputProvider) scanInputFromReader(executionId string, reader io.Reader) error {
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
item := scanner.Text()
switch {
case iputil.IsCIDR(item):
ips := expand.CIDR(item)
i.addTargets(executionId, ips)
case asn.IsASN(item):
ips := expand.ASN(item)
i.addTargets(executionId, ips)
default:
i.Set(executionId, item)
if err := i.addInput(executionId, scanner.Text()); err != nil {
return err
}
}
return nil
}
// addInput stores a plain target or a target line, expanding CIDR and ASN
// targets. An invalid target line fails the run: skipping it would drop the
// target, and ignoring its selection would scan it with every template.
func (i *ListInputProvider) addInput(executionId string, item string) error {
var selection *targetprofile.Selection
if targetprofile.IsLine(item) {
target, parsed, err := i.profiles.ParseLine(item)
if err != nil {
return errors.Wrap(err, "could not parse target line")
}
item, selection = target, parsed
}
switch {
case iputil.IsCIDR(item):
i.addTargets(executionId, expand.CIDR(item), selection)
case asn.IsASN(item):
i.addTargets(executionId, expand.ASN(item), selection)
default:
i.set(executionId, item, selection)
}
return nil
}
// isExcluded checks if a URL is in the exclusion list
@@ -486,24 +534,67 @@ func (i *ListInputProvider) Del(executionId string, value string) {
}
// setItem in the kv store
func (i *ListInputProvider) setItem(metaInput *contextargs.MetaInput) {
func (i *ListInputProvider) setItem(metaInput *contextargs.MetaInput, selection *targetprofile.Selection) {
key, err := metaInput.MarshalString()
if err != nil {
gologger.Warning().Msgf("%s\n", err)
return
}
if _, ok := i.hostMap.Get(key); ok {
// A crawled list routinely holds the same resource several times over,
// differing only by host case, a fragment, a default port, a trailing slash
// or a tracking parameter. Those collapse onto one entry, but the target
// that is scanned stays exactly as supplied: stripping a parameter from the
// request would shrink the attack surface, not just the duplicate count.
// Duplicates are removed before the pattern cap, so a repeated URL does not
// use up a slot that a later distinct URL still needs.
dedupKey := key
if i.normalizeURLs {
if normalized := i.normalizedKey(metaInput); normalized != "" {
dedupKey = normalized
}
}
if _, ok := i.hostMap.Get(dedupKey); ok {
i.dupeCount++
if i.profiles != nil {
i.profiles.Merge(metaInput.Input, selection)
}
return
}
// Past the per-pattern cap the shape is already covered by the targets kept
// for it, so scanning another of the same shape buys nothing.
if i.patterns != nil && !i.patterns.Accept(metaInput.Input) {
i.patternSkippedCount++
return
}
if i.profiles != nil {
i.profiles.Bind(metaInput.Input, selection)
}
i.inputCount++ // tracks target count
_ = i.hostMap.Set(key, nil)
// the entry is keyed by the collapsed form and carries the original, which
// is what Iterate hands back to the engine
_ = i.hostMap.Set(dedupKey, []byte(key))
if i.hostMapStream != nil {
i.setHostMapStream(key)
}
}
// normalizedKey returns the deduplication key for a target, or "" when the
// value is not a URL and has no collapsed form.
func (i *ListInputProvider) normalizedKey(metaInput *contextargs.MetaInput) string {
collapsed := normalize.URL(metaInput.Input)
if collapsed == metaInput.Input {
return ""
}
clone := *metaInput
clone.Input = collapsed
key, err := clone.MarshalString()
if err != nil {
return ""
}
return key
}
const removeTargetsChunkSize = 5000
// delItem removes all hostMap entries matching any of the given targets in a single scan.
@@ -565,9 +656,9 @@ func (i *ListInputProvider) setHostMapStream(data string) {
}
}
func (i *ListInputProvider) addTargets(executionId string, targets []string) {
func (i *ListInputProvider) addTargets(executionId string, targets []string, selection *targetprofile.Selection) {
for _, target := range targets {
i.Set(executionId, target)
i.set(executionId, target, selection)
}
}
+73 -2
View File
@@ -9,6 +9,7 @@ import (
"github.com/miekg/dns"
"github.com/projectdiscovery/hmap/store/hybrid"
"github.com/projectdiscovery/nuclei/v3/pkg/input/normalize"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
@@ -17,6 +18,76 @@ import (
"github.com/stretchr/testify/require"
)
func TestSetItemDedupDoesNotConsumePatternCap(t *testing.T) {
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
require.NoError(t, err)
input := &ListInputProvider{
hostMap: hm,
normalizeURLs: true,
patterns: normalize.NewFingerprinter(1),
}
t.Cleanup(input.Close)
add := func(raw string) {
meta := contextargs.NewMetaInput()
meta.Input = raw
input.setItem(meta, nil)
}
add("http://acme.test/user/1")
add("http://ACME.test/user/1")
add("http://acme.test/user/2")
add("http://acme.test/admin")
require.Equal(t, int64(1), input.dupeCount)
require.Equal(t, int64(1), input.patternSkippedCount)
require.Equal(t, int64(0), input.skippedCount)
require.Equal(t, int64(2), input.inputCount)
}
func TestSetItemKeepsTheSuppliedTarget(t *testing.T) {
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
require.NoError(t, err)
input := &ListInputProvider{
hostMap: hm,
normalizeURLs: true,
patterns: normalize.NewFingerprinter(0),
}
t.Cleanup(input.Close)
add := func(raw string) {
meta := contextargs.NewMetaInput()
meta.Input = raw
input.setItem(meta, nil)
}
add("http://ACME.test/page/")
add("http://acme.test/page?utm_source=news")
require.Equal(t, int64(1), input.dupeCount)
require.Equal(t, int64(1), input.inputCount)
var got []string
input.Iterate(func(value *contextargs.MetaInput) bool {
got = append(got, value.Input)
return true
})
require.Equal(t, []string{"http://ACME.test/page/"}, got)
}
func TestSetItemWithoutNormalizationKeepsCaseVariants(t *testing.T) {
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
require.NoError(t, err)
input := &ListInputProvider{
hostMap: hm,
normalizeURLs: false,
patterns: normalize.NewFingerprinter(0),
}
t.Cleanup(input.Close)
for _, raw := range []string{"http://ACME.test/page", "http://acme.test/page"} {
meta := contextargs.NewMetaInput()
meta.Input = raw
input.setItem(meta, nil)
}
require.Equal(t, int64(0), input.dupeCount)
require.Equal(t, int64(2), input.inputCount)
}
func Test_expandCIDR(t *testing.T) {
tests := []struct {
cidr string
@@ -36,7 +107,7 @@ func Test_expandCIDR(t *testing.T) {
input := &ListInputProvider{hostMap: hm}
ips := expand.CIDR(tt.cidr)
input.addTargets("", ips)
input.addTargets("", ips, nil)
// scan
got := []string{}
input.hostMap.Scan(func(k, _ []byte) error {
@@ -200,7 +271,7 @@ func Test_expandASNInputValue(t *testing.T) {
input := &ListInputProvider{hostMap: hm}
// get the IP addresses for ASN number
ips := expand.ASN(tt.asn)
input.addTargets("", ips)
input.addTargets("", ips, nil)
// scan the hmap
got := []string{}
input.hostMap.Scan(func(k, v []byte) error {
@@ -0,0 +1,36 @@
package list
import (
"strings"
"testing"
"github.com/projectdiscovery/hmap/store/hybrid"
"github.com/projectdiscovery/nuclei/v3/pkg/input/targetprofile"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/stretchr/testify/require"
)
func TestTargetLines(t *testing.T) {
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
require.NoError(t, err)
input := &ListInputProvider{hostMap: hm, ipOptions: &ipOptions{IPV4: true}, profiles: targetprofile.NewRegistry(t.TempDir())}
defer input.Close()
lines := strings.Join([]string{
`{"target": "192.0.2.0/30", "tags": ["tomcat"]}`,
`{"target": "192.0.2.10", "tags": ["thinkphp"]}`,
`{"target": "192.0.2.10", "tags": ["tomcat"]}`,
`192.0.2.20`,
}, "\n")
require.NoError(t, input.scanInputFromReader("", strings.NewReader(lines)))
require.EqualValues(t, 6, input.Count(), "4 CIDR hosts, one merged duplicate, one plain target")
for _, ip := range []string{"192.0.2.0", "192.0.2.1", "192.0.2.2", "192.0.2.3", "192.0.2.10"} {
require.NotNil(t, input.profiles.For(&contextargs.MetaInput{Input: ip}), ip)
}
require.Nil(t, input.profiles.For(&contextargs.MetaInput{Input: "192.0.2.20"}))
require.Nil(t, input.profiles.LoadFilter(), "a plain target needs every template")
err = input.scanInputFromReader("", strings.NewReader(`{"target": "192.0.2.30", "tagz": ["x"]}`))
require.ErrorContains(t, err, "unknown keys tagz")
}
+252
View File
@@ -0,0 +1,252 @@
package targetprofile
import (
"fmt"
"slices"
"strings"
"sync"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/index"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
)
// Registry maps targets to their template selection.
//
// Targets are bound while inputs are read, then Prepare is called with the
// loaded templates before the scan starts. Selections are interned, so targets
// sharing a selector share one memoized decision per template.
type Registry struct {
templatesDir string
mu sync.RWMutex
targets map[string]*Selection
selections map[string]*Selection
rules map[string]*rule
profiles map[string]selector
unrestricted bool
metadata map[string]*index.Metadata
}
var _ protocols.TargetScope = (*Registry)(nil)
// NewRegistry creates a registry resolving profiles and relative template
// paths against templatesDir.
func NewRegistry(templatesDir string) *Registry {
return &Registry{
templatesDir: templatesDir,
targets: make(map[string]*Selection),
selections: make(map[string]*Selection),
rules: make(map[string]*rule),
profiles: make(map[string]selector),
}
}
// ParseLine parses a target line into its target and selection.
func (r *Registry) ParseLine(line string) (string, *Selection, error) {
target, profile, sel, err := parseLine(line)
if err != nil {
return "", nil, err
}
if profile != "" {
base, err := r.profile(profile)
if err != nil {
return "", nil, fmt.Errorf("target %q: %w", target, err)
}
sel = base.merge(sel)
}
r.mu.Lock()
defer r.mu.Unlock()
compiled, err := r.internRule(sel)
if err != nil {
return "", nil, fmt.Errorf("target %q: %w", target, err)
}
return target, r.internSelection([]*rule{compiled}), nil
}
// Bind records the selection of a newly stored input; nil means unrestricted.
func (r *Registry) Bind(input string, selection *Selection) {
r.mu.Lock()
defer r.mu.Unlock()
if selection == nil {
r.unrestricted = true
if _, ok := r.targets[input]; ok {
r.targets[input] = nil
}
return
}
current, ok := r.targets[input]
switch {
case !ok:
r.targets[input] = selection
case current != nil:
r.targets[input] = r.union(current, selection)
}
}
// Merge records the selection of an input that was already stored, so a
// target listed more than once runs the union of its selections and is
// scanned once.
func (r *Registry) Merge(input string, selection *Selection) {
r.mu.Lock()
defer r.mu.Unlock()
current, ok := r.targets[input]
switch {
case !ok || current == nil:
// already unrestricted
case selection == nil:
r.unrestricted = true
r.targets[input] = nil
default:
r.targets[input] = r.union(current, selection)
}
}
// Scoped reports whether any target has a selection.
func (r *Registry) Scoped() bool {
r.mu.RLock()
defer r.mu.RUnlock()
return len(r.selections) > 0
}
// LoadFilter returns the templates worth loading: those selected for at least
// one target. It returns nil when some target is unrestricted, since that
// target needs every template.
func (r *Registry) LoadFilter() index.FilterFunc {
r.mu.RLock()
defer r.mu.RUnlock()
if r.unrestricted || len(r.rules) == 0 {
return nil
}
rules := make([]*rule, 0, len(r.rules))
for _, compiled := range r.rules {
rules = append(rules, compiled)
}
return func(m *index.Metadata) bool { return anyMatches(rules, m) }
}
// Prepare indexes the metadata of the templates about to run.
func (r *Registry) Prepare(tpls []*templates.Template) {
metadata := make(map[string]*index.Metadata, len(tpls))
for _, tpl := range tpls {
metadata[tpl.Path] = index.NewMetadataFromTemplate(tpl.Path, tpl)
}
r.mu.Lock()
defer r.mu.Unlock()
r.metadata = metadata
for _, selection := range r.selections {
selection.decisions.Clear()
}
}
// For returns the selection of input, or nil when every template applies.
func (r *Registry) For(input *contextargs.MetaInput) protocols.TemplateSelection {
r.mu.RLock()
selection := r.targets[input.Input]
r.mu.RUnlock()
// a nil *Selection must not become a non-nil interface
if selection == nil {
return nil
}
return selection
}
func (r *Registry) profile(name string) (selector, error) {
r.mu.RLock()
sel, ok := r.profiles[name]
r.mu.RUnlock()
if ok {
return sel, nil
}
sel, ignored, err := loadProfile(name, r.templatesDir)
if err != nil {
return nil, err
}
if len(ignored) > 0 {
gologger.Warning().Msgf("Profile %q keys %s apply to the whole scan and are ignored per target", name, strings.Join(ignored, ", "))
}
r.mu.Lock()
r.profiles[name] = sel
r.mu.Unlock()
return sel, nil
}
func (r *Registry) internRule(sel selector) (*rule, error) {
key, err := canonicalKey(sel)
if err != nil {
return nil, err
}
if compiled, ok := r.rules[key]; ok {
return compiled, nil
}
compiled, err := compileRule(sel, r.templatesDir)
if err != nil {
return nil, err
}
r.rules[key] = compiled
return compiled, nil
}
func (r *Registry) internSelection(rules []*rule) *Selection {
keys := make([]string, len(rules))
for i, compiled := range rules {
keys[i] = compiled.key
}
key := strings.Join(keys, "\n")
if selection, ok := r.selections[key]; ok {
return selection
}
selection := &Selection{registry: r, rules: rules}
r.selections[key] = selection
return selection
}
func (r *Registry) union(a, b *Selection) *Selection {
if a == b {
return a
}
rules := slices.Concat(a.rules, b.rules)
slices.SortFunc(rules, func(x, y *rule) int { return strings.Compare(x.key, y.key) })
rules = slices.CompactFunc(rules, func(x, y *rule) bool { return x.key == y.key })
return r.internSelection(rules)
}
// Selection is the template selection of one or more targets: a template is
// selected when any of its rules selects it.
type Selection struct {
registry *Registry
rules []*rule
decisions sync.Map
}
var _ protocols.TemplateSelection = (*Selection)(nil)
// Allows reports whether the template loaded from templatePath is selected.
// Templates unknown to Prepare are allowed rather than silently dropped.
func (s *Selection) Allows(templatePath string) bool {
if allowed, ok := s.decisions.Load(templatePath); ok {
return allowed.(bool)
}
s.registry.mu.RLock()
m := s.registry.metadata[templatePath]
s.registry.mu.RUnlock()
allowed := m == nil || anyMatches(s.rules, m)
s.decisions.Store(templatePath, allowed)
return allowed
}
+210
View File
@@ -0,0 +1,210 @@
package targetprofile
import (
"encoding/json"
"os"
"path/filepath"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/stretchr/testify/require"
)
const templatesDir = "/templates"
func newTemplate(path, id string, sev severity.Severity, product string, tags ...string) *templates.Template {
info := model.Info{
Tags: stringslice.StringSlice{Value: tags},
SeverityHolder: severity.Holder{Severity: sev},
}
if product != "" {
info.Metadata = map[string]any{"product": product}
}
return &templates.Template{ID: id, Path: filepath.Join(templatesDir, path), Info: info}
}
var (
thinkphpRCE = newTemplate("http/cves/thinkphp-rce.yaml", "thinkphp-rce", severity.Critical, "thinkphp", "cve", "thinkphp", "rce")
tomcatPanel = newTemplate("http/exposed-panels/tomcat-panel.yaml", "tomcat-panel", severity.Info, "tomcat", "panel", "tomcat")
ghostcat = newTemplate("network/cves/CVE-2020-1938.yaml", "CVE-2020-1938", severity.Critical, "geode", "cve", "apache", "tomcat")
gitConfig = newTemplate("http/exposures/git-config.yaml", "git-config", severity.Medium, "", "exposure", "git")
allTemplates = []*templates.Template{thinkphpRCE, tomcatPanel, ghostcat, gitConfig}
)
func selected(t *testing.T, r *Registry, input string) []string {
t.Helper()
selection := r.For(&contextargs.MetaInput{Input: input})
var ids []string
for _, tpl := range allTemplates {
if selection == nil || selection.Allows(tpl.Path) {
ids = append(ids, tpl.ID)
}
}
return ids
}
func bindLine(t *testing.T, r *Registry, line string) string {
t.Helper()
target, selection, err := r.ParseLine(line)
require.NoError(t, err)
r.Bind(target, selection)
return target
}
func TestParseLineRejectsInvalidLines(t *testing.T) {
r := NewRegistry(templatesDir)
for name, line := range map[string]string{
"malformed json": `{"target": "a.com"`,
"missing target": `{"tags": ["cve"]}`,
"unknown key": `{"target": "a.com", "tag": ["cve"]}`,
"invalid severity": `{"target": "a.com", "severity": ["urgent"]}`,
"invalid type": `{"target": "a.com", "type": ["smtp"]}`,
"non string value": `{"target": "a.com", "tags": [1]}`,
"missing profile": `{"target": "a.com", "profile": "does-not-exist"}`,
} {
t.Run(name, func(t *testing.T) {
_, _, err := r.ParseLine(line)
require.Error(t, err)
})
}
}
func TestProfileMustBeAnID(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(dir, "profiles"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "profiles", "php.yml"), []byte("tags: [thinkphp]\n"), 0o600))
outside := filepath.Join(dir, "outside.yml")
require.NoError(t, os.WriteFile(outside, []byte("tags: [tomcat]\n"), 0o600))
r := NewRegistry(dir)
for _, profile := range []string{outside, "../outside.yml", "../outside", "profiles/php.yml", "php.yml", `profiles\php`, ".."} {
t.Run(profile, func(t *testing.T) {
line, err := json.Marshal(map[string]string{"target": "a.com", "profile": profile})
require.NoError(t, err)
_, _, err = r.ParseLine(string(line))
require.ErrorContains(t, err, "profile ID")
})
}
_, selection, err := r.ParseLine(`{"target": "a.com", "profile": "php"}`)
require.NoError(t, err)
require.Equal(t, []string{"thinkphp"}, selection.rules[0].filter.Tags)
}
func TestSelectionFilters(t *testing.T) {
r := NewRegistry(templatesDir)
tags := bindLine(t, r, `{"target": "https://a.com", "tags": ["thinkphp"]}`)
commaTags := bindLine(t, r, `{"target": "https://b.com", "tags": "tomcat, exposure"}`)
sev := bindLine(t, r, `{"target": "https://c.com", "severity": ["critical"], "exclude-id": ["CVE-2020-1938"]}`)
paths := bindLine(t, r, `{"target": "https://d.com", "templates": ["http/"]}`)
r.Prepare(allTemplates)
require.Equal(t, []string{"thinkphp-rce"}, selected(t, r, tags))
require.Equal(t, []string{"tomcat-panel", "CVE-2020-1938", "git-config"}, selected(t, r, commaTags))
require.Equal(t, []string{"thinkphp-rce"}, selected(t, r, sev))
require.Equal(t, []string{"thinkphp-rce", "tomcat-panel", "git-config"}, selected(t, r, paths))
require.Equal(t, []string{"thinkphp-rce", "tomcat-panel", "CVE-2020-1938", "git-config"}, selected(t, r, "https://unlisted.com"))
}
func TestSelectionTech(t *testing.T) {
r := NewRegistry(templatesDir)
tomcat := bindLine(t, r, `{"target": "https://a.com", "tech": ["Apache Tomcat"]}`)
php := bindLine(t, r, `{"target": "https://b.com", "tech": ["ThinkPHP"], "severity": ["critical", "medium"]}`)
r.Prepare(allTemplates)
// product-bound templates follow the tech list, generic ones always run;
// Ghostcat declares product "geode" but is kept by its tomcat tag
require.Equal(t, []string{"tomcat-panel", "CVE-2020-1938", "git-config"}, selected(t, r, tomcat))
require.Equal(t, []string{"thinkphp-rce", "git-config"}, selected(t, r, php))
}
func TestProfileWithInlineOverride(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(dir, "profiles"), 0o755))
profile := "code: true\ntags:\n - thinkphp\nseverity:\n - info\n"
require.NoError(t, os.WriteFile(filepath.Join(dir, "profiles", "php.yml"), []byte(profile), 0o600))
r := NewRegistry(dir)
target, selection, err := r.ParseLine(`{"target": "https://a.com", "profile": "php", "severity": "critical"}`)
require.NoError(t, err)
require.Len(t, selection.rules, 1)
require.Equal(t, []string{"thinkphp"}, selection.rules[0].filter.Tags)
require.Equal(t, []severity.Severity{severity.Critical}, selection.rules[0].filter.Severities)
require.Equal(t, "https://a.com", target)
}
func TestDuplicateTargetsMerge(t *testing.T) {
r := NewRegistry(templatesDir)
_, php, err := r.ParseLine(`{"target": "https://a.com", "tags": ["thinkphp"]}`)
require.NoError(t, err)
_, tomcat, err := r.ParseLine(`{"target": "https://a.com", "tags": ["tomcat"]}`)
require.NoError(t, err)
r.Bind("https://a.com", php)
r.Merge("https://a.com", tomcat)
r.Merge("https://a.com", php)
r.Bind("https://b.com", php)
r.Merge("https://b.com", nil)
r.Bind("https://c.com", nil)
r.Merge("https://c.com", php)
r.Prepare(allTemplates)
require.Equal(t, []string{"thinkphp-rce", "tomcat-panel", "CVE-2020-1938"}, selected(t, r, "https://a.com"))
require.Nil(t, r.For(&contextargs.MetaInput{Input: "https://b.com"}), "a plain duplicate lifts the restriction")
require.Nil(t, r.For(&contextargs.MetaInput{Input: "https://c.com"}), "a plain target stays unrestricted")
_, again, err := r.ParseLine(`{"target": "https://z.com", "tags": ["thinkphp"]}`)
require.NoError(t, err)
require.Same(t, php, again, "identical selectors are interned")
}
func TestLoadFilter(t *testing.T) {
t.Run("union of selections", func(t *testing.T) {
r := NewRegistry(templatesDir)
bindLine(t, r, `{"target": "https://a.com", "tags": ["thinkphp"]}`)
bindLine(t, r, `{"target": "https://b.com", "tech": ["tomcat"], "severity": ["info"]}`)
filter := r.LoadFilter()
require.NotNil(t, filter)
r.Prepare(allTemplates)
var loaded []string
for _, tpl := range allTemplates {
if filter(r.metadata[tpl.Path]) {
loaded = append(loaded, tpl.ID)
}
}
require.Equal(t, []string{"thinkphp-rce", "tomcat-panel"}, loaded)
})
t.Run("unrestricted target loads everything", func(t *testing.T) {
r := NewRegistry(templatesDir)
bindLine(t, r, `{"target": "https://a.com", "tags": ["thinkphp"]}`)
r.Bind("https://b.com", nil)
require.Nil(t, r.LoadFilter())
})
t.Run("plain duplicate of a target line loads everything", func(t *testing.T) {
r := NewRegistry(templatesDir)
target := bindLine(t, r, `{"target": "https://a.com", "tags": ["thinkphp"]}`)
r.Merge(target, nil)
require.Nil(t, r.LoadFilter())
})
}
func TestTechTokens(t *testing.T) {
for tech, want := range map[string][]string{
"nginx": {"nginx"},
"ThinkPHP": {"thinkphp"},
"apache_tomcat": {"apache-tomcat"},
"Apache Tomcat": {"apache-tomcat", "apache", "tomcat"},
"apache:tomcat": {"apache-tomcat", "apache", "tomcat"},
"Microsoft IIS": {"microsoft-iis", "microsoft", "iis"},
" ": nil,
} {
require.Equal(t, want, techTokens(tech), tech)
}
}
+178
View File
@@ -0,0 +1,178 @@
package targetprofile
import (
"fmt"
"path/filepath"
"slices"
"strings"
"unicode"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/index"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
)
// rule is a compiled selector.
type rule struct {
key string
filter *index.Filter
paths []string
tech map[string]struct{}
}
func compileRule(sel selector, templatesDir string) (*rule, error) {
var severities, excludeSeverities severity.Severities
if err := setAll(&severities, sel[keySeverity]); err != nil {
return nil, fmt.Errorf("key %q: %w", keySeverity, err)
}
if err := setAll(&excludeSeverities, sel[keyExcludeSeverity]); err != nil {
return nil, fmt.Errorf("key %q: %w", keyExcludeSeverity, err)
}
var protocols, excludeProtocols types.ProtocolTypes
if err := setAll(&protocols, sel[keyType]); err != nil {
return nil, fmt.Errorf("key %q: %w", keyType, err)
}
if err := setAll(&excludeProtocols, sel[keyExcludeType]); err != nil {
return nil, fmt.Errorf("key %q: %w", keyExcludeType, err)
}
r := &rule{
filter: &index.Filter{
Authors: lowered(sel[keyAuthor]),
Tags: lowered(sel[keyTags]),
ExcludeTags: lowered(sel[keyExcludeTags]),
IncludeTags: lowered(sel[keyIncludeTags]),
IDs: sel[keyTemplateID],
ExcludeIDs: sel[keyExcludeID],
Severities: severities,
ExcludeSeverities: excludeSeverities,
ProtocolTypes: protocols,
ExcludeProtocolTypes: excludeProtocols,
},
}
for _, path := range sel[keyTemplates] {
if !filepath.IsAbs(path) {
path = filepath.Join(templatesDir, path)
}
r.paths = append(r.paths, filepath.Clean(path))
}
if len(sel[keyTech]) > 0 {
r.tech = make(map[string]struct{}, len(sel[keyTech]))
for _, tech := range sel[keyTech] {
for _, token := range techTokens(tech) {
r.tech[token] = struct{}{}
}
}
}
key, err := canonicalKey(sel)
if err != nil {
return nil, err
}
r.key = key
return r, nil
}
// matches reports whether the template described by m is selected.
func (r *rule) matches(m *index.Metadata) bool {
if !r.filter.Matches(m) {
return false
}
if len(r.paths) > 0 && !slices.ContainsFunc(r.paths, func(path string) bool { return underPath(m.FilePath, path) }) {
return false
}
return r.matchesTech(m)
}
// matchesTech implements tech-aware selection: templates bound to a product run
// only when the target runs that product, every other template always runs.
// A template is bound when it declares metadata.product; its tags are matched
// too because product metadata is sometimes imprecise (CVE-2020-1938 targets
// Tomcat but declares product "geode"). Erring on running a template keeps
// coverage when metadata and the target's tech list disagree.
func (r *rule) matchesTech(m *index.Metadata) bool {
if len(r.tech) == 0 || m.Product == "" {
return true
}
if _, ok := r.tech[normalizeTech(m.Product)]; ok {
return true
}
for _, tag := range m.Tags {
if _, ok := r.tech[normalizeTech(tag)]; ok {
return true
}
}
return false
}
func anyMatches(rules []*rule, m *index.Metadata) bool {
return slices.ContainsFunc(rules, func(r *rule) bool { return r.matches(m) })
}
// techTokens expands a tech name reported by a fingerprinting tool into the
// tokens template metadata uses: "Apache Tomcat" and "apache:tomcat" yield
// apache-tomcat, apache and tomcat. Extra tokens can only select more
// templates, never fewer.
func techTokens(tech string) []string {
words := strings.FieldsFunc(strings.ToLower(tech), func(r rune) bool { return r == ':' || unicode.IsSpace(r) })
if len(words) == 0 {
return nil
}
tokens := []string{normalizeTech(strings.Join(words, " "))}
if len(words) > 1 {
for _, word := range words {
tokens = append(tokens, normalizeTech(word))
}
}
return tokens
}
// normalizeTech maps a product or tag to its token form ("Apache_Tomcat" and
// "apache tomcat" both become apache-tomcat).
func normalizeTech(tech string) string {
tech = strings.ReplaceAll(strings.ToLower(tech), "_", "-")
return strings.Join(strings.Fields(tech), "-")
}
func underPath(templatePath, path string) bool {
templatePath = filepath.Clean(templatePath)
if templatePath == path || strings.HasPrefix(templatePath, path+string(filepath.Separator)) {
return true
}
matched, _ := filepath.Match(path, templatePath)
return matched
}
// canonicalKey identifies equivalent selectors so they share one compiled rule.
func canonicalKey(sel selector) (string, error) {
normalized := make(map[string][]string, len(sel))
for key, values := range sel {
values = slices.Clone(values)
slices.Sort(values)
normalized[key] = slices.Compact(values)
}
data, err := json.Marshal(normalized)
if err != nil {
return "", err
}
return string(data), nil
}
func setAll(value interface{ Set(string) error }, values []string) error {
if len(values) == 0 {
return nil
}
return value.Set(strings.Join(values, ","))
}
func lowered(values []string) []string {
if len(values) == 0 {
return nil
}
out := make([]string, len(values))
for i, value := range values {
out[i] = strings.ToLower(value)
}
return out
}
+190
View File
@@ -0,0 +1,190 @@
// Package targetprofile binds template selections to individual targets, so a
// single scan can run a different profile on each target.
//
// A target line is a JSON object whose keys mirror template profiles and CLI
// flags:
//
// {"target": "https://a.example", "tags": ["thinkphp"], "tech": ["php"]}
// {"target": "https://b.example", "profile": "wordpress", "severity": ["high"]}
package targetprofile
import (
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"github.com/projectdiscovery/nuclei/v3/internal/configuration"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
)
const (
keyTarget = "target"
keyProfile = "profile"
keyTemplates = "templates"
keyTags = "tags"
keyExcludeTags = "exclude-tags"
keyIncludeTags = "include-tags"
keyAuthor = "author"
keyTemplateID = "template-id"
keyExcludeID = "exclude-id"
keySeverity = "severity"
keyExcludeSeverity = "exclude-severity"
keyType = "type"
keyExcludeType = "exclude-type"
keyTech = "tech"
)
// selectorKeys are the template selection keys a target line or profile may set.
var selectorKeys = []string{
keyTemplates, keyTags, keyExcludeTags, keyIncludeTags, keyAuthor, keyTemplateID,
keyExcludeID, keySeverity, keyExcludeSeverity, keyType, keyExcludeType, keyTech,
}
// selector is a raw template selection keyed by selector key.
type selector map[string][]string
// IsLine reports whether an input line is a target line rather than a plain target.
func IsLine(line string) bool {
return strings.HasPrefix(strings.TrimSpace(line), "{")
}
// parseLine decodes a target line. Unknown keys are rejected so that a typo
// cannot silently widen the scan to every template.
func parseLine(line string) (target, profile string, sel selector, err error) {
var raw map[string]any
if err := json.Unmarshal([]byte(line), &raw); err != nil {
return "", "", nil, fmt.Errorf("invalid target line %q: %w", line, err)
}
target, err = stringValue(raw, keyTarget)
if err != nil {
return "", "", nil, err
}
target = strings.TrimSpace(target)
if target == "" {
return "", "", nil, fmt.Errorf("target line %q has no %q", line, keyTarget)
}
profile, err = stringValue(raw, keyProfile)
if err != nil {
return "", "", nil, err
}
delete(raw, keyTarget)
delete(raw, keyProfile)
sel, ignored, err := toSelector(raw)
if err != nil {
return "", "", nil, fmt.Errorf("target %q: %w", target, err)
}
if len(ignored) > 0 {
return "", "", nil, fmt.Errorf("target %q: unknown keys %s (supported: target, profile, %s)", target, strings.Join(ignored, ", "), strings.Join(selectorKeys, ", "))
}
return target, profile, sel, nil
}
// loadProfile reads the selection keys of a template profile. Keys that
// configure the whole scan (code, var, rate limits...) cannot apply to a single
// target and are returned as ignored.
func loadProfile(profile, templatesDir string) (selector, []string, error) {
// Target lines often come from other tools, so a profile is only looked up
// by ID inside the profiles directory and never read from a given path.
if profile == "" || strings.ContainsAny(profile, `/\`) || filepath.Ext(profile) != "" {
return nil, nil, fmt.Errorf("profile %q must be a profile ID from %s", profile, filepath.Join(templatesDir, "profiles"))
}
path, err := configuration.ResolveProfilePath(profile, templatesDir)
if err != nil {
return nil, nil, err
}
data, err := os.ReadFile(path)
if err != nil {
return nil, nil, fmt.Errorf("read profile %q: %w", path, err)
}
var raw map[string]any
if err := yaml.Unmarshal(data, &raw); err != nil {
return nil, nil, fmt.Errorf("parse profile %q: %w", path, err)
}
sel, ignored, err := toSelector(raw)
if err != nil {
return nil, nil, fmt.Errorf("profile %q: %w", path, err)
}
return sel, ignored, nil
}
// toSelector extracts selector keys from raw, returning the other keys as ignored.
func toSelector(raw map[string]any) (selector, []string, error) {
sel := selector{}
var ignored []string
for key, value := range raw {
if !slices.Contains(selectorKeys, key) {
ignored = append(ignored, key)
continue
}
values, err := stringList(value)
if err != nil {
return nil, nil, fmt.Errorf("key %q: %w", key, err)
}
if len(values) > 0 {
sel[key] = values
}
}
slices.Sort(ignored)
return sel, ignored, nil
}
// merge returns base overridden by the keys set in override, the same way
// CLI flags override a profile.
func (base selector) merge(override selector) selector {
merged := make(selector, len(base)+len(override))
for key, values := range base {
merged[key] = values
}
for key, values := range override {
merged[key] = values
}
return merged
}
func stringValue(raw map[string]any, key string) (string, error) {
value, ok := raw[key]
if !ok || value == nil {
return "", nil
}
str, ok := value.(string)
if !ok {
return "", fmt.Errorf("key %q must be a string", key)
}
return str, nil
}
// stringList accepts a comma-separated string or a list of strings, matching
// how the equivalent CLI flags and profile keys are written.
func stringList(value any) ([]string, error) {
var items []string
switch v := value.(type) {
case nil:
case string:
items = strings.Split(v, ",")
case []any:
for _, item := range v {
str, ok := item.(string)
if !ok {
return nil, fmt.Errorf("expected strings, got %T", item)
}
items = append(items, str)
}
default:
return nil, fmt.Errorf("expected a string or a list of strings, got %T", value)
}
values := make([]string, 0, len(items))
for _, item := range items {
if item = strings.TrimSpace(item); item != "" {
values = append(values, item)
}
}
return values, nil
}
+32
View File
@@ -0,0 +1,32 @@
package installer
import (
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
updateutils "github.com/projectdiscovery/utils/update"
)
// templateRelease is a published release of the official nuclei-templates.
type templateRelease interface {
Version() string
Changelog() string
DownloadSource(showProgressBar bool, callback updateutils.AssetFileCallback) error
}
type githubTemplateRelease struct {
*updateutils.GHReleaseDownloader
}
func (r githubTemplateRelease) Version() string { return r.Latest.GetTagName() }
func (r githubTemplateRelease) Changelog() string { return r.Latest.GetBody() }
func (r githubTemplateRelease) DownloadSource(showProgressBar bool, callback updateutils.AssetFileCallback) error {
return r.DownloadSourceWithCallback(showProgressBar, callback)
}
func latestGitHubTemplateRelease() (templateRelease, error) {
downloader, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTemplatesRepoName)
if err != nil {
return nil, err
}
return githubTemplateRelease{downloader}, nil
}
+48
View File
@@ -0,0 +1,48 @@
package installer
import (
"io/fs"
"maps"
"path/filepath"
"slices"
"strings"
"time"
updateutils "github.com/projectdiscovery/utils/update"
)
// fakeTemplateRelease is an in-memory nuclei-templates release, mapping paths
// to contents.
type fakeTemplateRelease struct {
version string
files map[string]string
}
func (r fakeTemplateRelease) fetch() (templateRelease, error) { return r, nil }
func (r fakeTemplateRelease) Version() string { return r.version }
func (r fakeTemplateRelease) Changelog() string { return "" }
func (r fakeTemplateRelease) DownloadSource(_ bool, callback updateutils.AssetFileCallback) error {
for _, name := range slices.Sorted(maps.Keys(r.files)) {
contents := r.files[name]
// a github zipball nests every entry under one root directory
uri := "projectdiscovery-nuclei-templates-test/" + name
if err := callback(uri, fakeFileInfo{name: name, size: int64(len(contents))}, strings.NewReader(contents)); err != nil {
return err
}
}
return nil
}
type fakeFileInfo struct {
name string
size int64
}
func (f fakeFileInfo) Name() string { return filepath.Base(f.name) }
func (f fakeFileInfo) Size() int64 { return f.size }
func (f fakeFileInfo) Mode() fs.FileMode { return 0o644 }
func (f fakeFileInfo) ModTime() time.Time { return time.Time{} }
func (f fakeFileInfo) IsDir() bool { return false }
func (f fakeFileInfo) Sys() any { return nil }
+105 -37
View File
@@ -16,7 +16,7 @@ import (
"strings"
"github.com/charmbracelet/glamour"
"github.com/google/go-github/v30/github"
"github.com/google/go-github/v92/github"
"github.com/olekukonko/tablewriter"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
@@ -76,6 +76,17 @@ type TemplateManager struct {
CustomTemplates *customtemplates.CustomTemplatesManager // optional if given tries to download custom templates
DisablePublicTemplates bool // if true,
// public templates are not downloaded from the GitHub nuclei-templates repository
// fetchLatestRelease overrides where the official release comes from; nil
// means GitHub
fetchLatestRelease func() (templateRelease, error)
}
func (t *TemplateManager) latestRelease() (templateRelease, error) {
if t.fetchLatestRelease != nil {
return t.fetchLatestRelease()
}
return latestGitHubTemplateRelease()
}
// FreshInstallIfNotExists installs templates if they are not already installed
@@ -99,7 +110,7 @@ func (t *TemplateManager) FreshInstallIfNotExists() error {
// UpdateIfOutdated updates templates if they are outdated
func (t *TemplateManager) UpdateIfOutdated() error {
return withTemplatesUpdateLock(t.updateIfOutdatedLocked)
return withTemplatesUpdateLock(config.DefaultConfig.TemplatesDirectory, t.updateIfOutdatedLocked)
}
func (t *TemplateManager) updateIfOutdatedLocked() error {
@@ -112,15 +123,21 @@ func (t *TemplateManager) updateIfOutdatedLocked() error {
return errkit.Wrapf(err, "failed to recover template ownership at %s", config.DefaultConfig.TemplatesDirectory)
}
// the version in memory predates the lock; a process that held it before
// this one may have already installed the latest release
if err := config.DefaultConfig.ReloadTemplateVersion(); err != nil {
gologger.Debug().Msgf("Could not reload templates version, using the one loaded at startup: %s", err)
}
needsUpdate := config.DefaultConfig.NeedsTemplateUpdate()
// NOTE(dwisiswant0): if PDTM API data is not available
// (LatestNucleiTemplatesVersion is empty) but we have a current template
// version, so we MUST verify against GitHub directly.
if !needsUpdate && config.DefaultConfig.LatestNucleiTemplatesVersion == "" && config.DefaultConfig.TemplateVersion != "" {
ghrd, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTemplatesRepoName)
release, err := t.latestRelease()
if err == nil {
latestVersion := ghrd.Latest.GetTagName()
latestVersion := release.Version()
if config.IsOutdatedVersion(config.DefaultConfig.TemplateVersion, latestVersion) {
needsUpdate = true
gologger.Debug().Msgf("PDTM API unavailable, verified update needed via GitHub API: %s -> %s", config.DefaultConfig.TemplateVersion, latestVersion)
@@ -151,14 +168,14 @@ func (t *TemplateManager) installTemplatesAt(dir string) error {
return errkit.Wrapf(err, "failed to recover template ownership at %s", dir)
}
ghrd, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTemplatesRepoName)
release, err := t.latestRelease()
if err != nil {
return errkit.Wrapf(err, "failed to install templates at %s", dir)
}
// write templates to disk
writtenOutputs, writeErr := t.writeTemplatesToDisk(ghrd, dir)
if _, finalizeErr := t.finalizeTemplateWrite(config.DefaultConfig, writtenOutputs, ghrd.Latest.GetTagName(), writeErr); finalizeErr != nil {
writtenOutputs, writeErr := t.writeTemplatesToDisk(release, dir)
if _, finalizeErr := t.finalizeTemplateWrite(config.DefaultConfig, writtenOutputs, release.Version(), writeErr); finalizeErr != nil {
return errkit.Wrapf(finalizeErr, "failed to finalize template installation at %s", dir)
}
@@ -268,14 +285,17 @@ func (t *TemplateManager) bootstrapTemplateOwnership(dir, version string, fetchA
func fetchTemplateReleaseArchive(version string) (*bytes.Reader, error) {
ctx := context.Background()
httpClient := &http.Client{Timeout: updateutils.DownloadUpdateTimeout}
client := github.NewClient(httpClient)
client, err := github.NewClient(github.WithHTTPClient(httpClient))
if err != nil {
return nil, fmt.Errorf("create github client for prior template release %q: %w", version, err)
}
archiveURL, _, err := client.Repositories.GetArchiveLink(
ctx,
updateutils.Organization,
config.OfficialNucleiTemplatesRepoName,
github.Zipball,
&github.RepositoryContentGetOptions{Ref: version},
true,
1, // Follow one repository rename, as the previous client did.
)
if err != nil {
return nil, fmt.Errorf("resolve prior template release %q archive: %w", version, err)
@@ -287,7 +307,7 @@ func fetchTemplateReleaseArchive(version string) (*bytes.Reader, error) {
}
var contents bytes.Buffer
response, err := client.Do(ctx, request, &contents)
response, err := client.Do(request, &contents)
if err != nil {
return nil, fmt.Errorf("download prior template release %q: %w", version, err)
}
@@ -372,12 +392,12 @@ func (t *TemplateManager) updateTemplatesAt(dir string) error {
oldchecksums = make(map[string]string)
}
ghrd, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTemplatesRepoName)
release, err := t.latestRelease()
if err != nil {
return errkit.Wrapf(err, "failed to install templates at %s", dir)
}
latestVersion := ghrd.Latest.GetTagName()
latestVersion := release.Version()
currentVersion := config.DefaultConfig.TemplateVersion
if config.IsOutdatedVersion(currentVersion, latestVersion) {
@@ -387,7 +407,7 @@ func (t *TemplateManager) updateTemplatesAt(dir string) error {
}
// write templates to disk
writtenOutputs, writeErr := t.writeTemplatesToDisk(ghrd, dir)
writtenOutputs, writeErr := t.writeTemplatesToDisk(release, dir)
newchecksums, finalizeErr := t.finalizeTemplateWrite(config.DefaultConfig, writtenOutputs, latestVersion, writeErr)
if finalizeErr != nil {
return errkit.Wrapf(finalizeErr, "failed to finalize template update at %s", dir)
@@ -398,14 +418,14 @@ func (t *TemplateManager) updateTemplatesAt(dir string) error {
// print summary
if results.totalCount > 0 {
gologger.Info().Msgf("Successfully updated nuclei-templates (%v) to %s. GoodLuck!", ghrd.Latest.GetTagName(), dir)
gologger.Info().Msgf("Successfully updated nuclei-templates (%v) to %s. GoodLuck!", latestVersion, dir)
if !HideUpdateChangesTable {
// print summary table
gologger.Print().Msgf("\nNuclei Templates %s Changelog\n", ghrd.Latest.GetTagName())
gologger.Print().Msgf("\nNuclei Templates %s Changelog\n", latestVersion)
gologger.Print().Msg(results.String())
}
} else {
gologger.Info().Msgf("Successfully updated nuclei-templates (%v) to %s. GoodLuck!", ghrd.Latest.GetTagName(), dir)
gologger.Info().Msgf("Successfully updated nuclei-templates (%v) to %s. GoodLuck!", latestVersion, dir)
}
return nil
@@ -499,16 +519,24 @@ func isActiveIgnoreFilePath(writePath string) bool {
// writeTemplatesToDisk writes release outputs to disk and returns their digests.
// The returned map includes every successfully written output; ownership
// reconciliation filters it to official template paths.
func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownloader, dir string) (map[string]string, error) {
func (t *TemplateManager) writeTemplatesToDisk(release templateRelease, dir string) (map[string]string, error) {
writtenOutputs := make(map[string]string)
touchedDirectories := make(map[string]struct{})
writer, err := newTemplateOutputWriter(dir)
if err != nil {
return writtenOutputs, err
}
defer func() { _ = writer.Close() }()
callbackFunc := func(uri string, f fs.FileInfo, r io.Reader) error {
if f.IsDir() {
return nil
}
rootDir, writePath := t.getTemplateOutputLocation(dir, uri, f)
// every output is under dir except the ignore file, which the writer
// routes to the config directory itself
_, writePath := t.getTemplateOutputLocation(dir, uri, f)
if writePath == "" {
// skip writing file
return nil
@@ -520,7 +548,7 @@ func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownlo
return errkit.Wrapf(err, "failed to read file %s", uri)
}
outputResult, outputErr := writeTemplateOutput(rootDir, writePath, bin, f.Mode())
outputResult, outputErr := writer.write(writePath, bin, f.Mode())
for _, directory := range outputResult.touchedDirectories {
touchedDirectories[directory] = struct{}{}
}
@@ -536,7 +564,7 @@ func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownlo
var writeErr error
if err := ghrd.DownloadSourceWithCallback(!HideProgressBar, callbackFunc); err != nil {
if err := release.DownloadSource(!HideProgressBar, callbackFunc); err != nil {
writeErr = errkit.Wrap(err, "failed to download templates")
}
@@ -549,7 +577,7 @@ func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownlo
}
if !HideReleaseNotes {
output := ghrd.Latest.GetBody()
output := release.Changelog()
// adjust colors for both dark / light terminal themes
r, err := glamour.NewTermRenderer(glamour.WithAutoStyle())
if err != nil {
@@ -597,15 +625,56 @@ type templateOutputWriteResult struct {
touchedDirectories []string
}
func writeTemplateOutput(rootDir, writePath string, contents []byte, mode fs.FileMode) (templateOutputWriteResult, error) {
var result templateOutputWriteResult
if isActiveIgnoreFilePath(writePath) {
if err := config.DefaultConfig.WriteActiveIgnoreFile(contents); err != nil {
return result, err
}
result.touchedDirectories = append(result.touchedDirectories, filepath.Dir(writePath))
return result, nil
// templateOutputWriter writes the outputs of one release under rootDir. A
// release has thousands of files in a few hundred directories, and on windows
// every operation inside an os.Root reopens each path component, so the root
// is opened once and each directory is created once rather than per file.
type templateOutputWriter struct {
rootDir string
root *os.Root
createdDirectories map[string]struct{}
}
func newTemplateOutputWriter(rootDir string) (*templateOutputWriter, error) {
root, err := os.OpenRoot(rootDir)
if err != nil {
return nil, fmt.Errorf("open output root %q: %w", rootDir, err)
}
return &templateOutputWriter{rootDir: rootDir, root: root, createdDirectories: make(map[string]struct{})}, nil
}
func (w *templateOutputWriter) Close() error {
return w.root.Close()
}
// writeTemplateOutput writes a single output; installs write through one
// templateOutputWriter instead.
func writeTemplateOutput(rootDir, writePath string, contents []byte, mode fs.FileMode) (templateOutputWriteResult, error) {
if isActiveIgnoreFilePath(writePath) {
return writeActiveIgnoreFile(writePath, contents)
}
writer, err := newTemplateOutputWriter(rootDir)
if err != nil {
return templateOutputWriteResult{}, err
}
defer func() { _ = writer.Close() }()
return writer.write(writePath, contents, mode)
}
func writeActiveIgnoreFile(writePath string, contents []byte) (templateOutputWriteResult, error) {
if err := config.DefaultConfig.WriteActiveIgnoreFile(contents); err != nil {
return templateOutputWriteResult{}, err
}
return templateOutputWriteResult{touchedDirectories: []string{filepath.Dir(writePath)}}, nil
}
func (w *templateOutputWriter) write(writePath string, contents []byte, mode fs.FileMode) (templateOutputWriteResult, error) {
if isActiveIgnoreFilePath(writePath) {
return writeActiveIgnoreFile(writePath, contents)
}
var result templateOutputWriteResult
rootDir, root := w.rootDir, w.root
relativePath, err := filepath.Rel(rootDir, writePath)
if err != nil {
@@ -616,14 +685,8 @@ func writeTemplateOutput(rootDir, writePath string, contents []byte, mode fs.Fil
return result, fmt.Errorf("output path %q escapes root %q", writePath, rootDir)
}
root, err := os.OpenRoot(rootDir)
if err != nil {
return result, fmt.Errorf("open output root %q: %w", rootDir, err)
}
defer func() { _ = root.Close() }()
parent := filepath.Dir(relativePath)
if parent != "." {
if _, created := w.createdDirectories[parent]; !created && parent != "." {
parentsToSync, err := createTemplateDirectories(root, parent)
for _, parentToSync := range parentsToSync {
result.touchedDirectories = append(result.touchedDirectories, filepath.Clean(filepath.Join(rootDir, parentToSync)))
@@ -631,6 +694,7 @@ func writeTemplateOutput(rootDir, writePath string, contents []byte, mode fs.Fil
if err != nil {
return result, fmt.Errorf("create output parent %q: %w", parent, err)
}
w.createdDirectories[parent] = struct{}{}
}
result.touchedDirectories = append(result.touchedDirectories, filepath.Dir(writePath))
@@ -656,9 +720,12 @@ func writeTemplateOutput(rootDir, writePath string, contents []byte, mode fs.Fil
return result, fmt.Errorf("create temporary output %q: %w", temporaryPath, err)
}
renamed := false
defer func() {
_ = temporary.Close()
_ = root.Remove(temporaryPath)
if !renamed {
_ = root.Remove(temporaryPath)
}
}()
if _, err := temporary.Write(contents); err != nil {
@@ -682,6 +749,7 @@ func writeTemplateOutput(rootDir, writePath string, contents []byte, mode fs.Fil
if err := root.Rename(temporaryPath, relativePath); err != nil {
return result, fmt.Errorf("replace output %q: %w", relativePath, err)
}
renamed = true
return result, nil
}
+58 -36
View File
@@ -19,15 +19,24 @@ func TestTemplateInstallation(t *testing.T) {
// along with necessary changes that are made
HideProgressBar = true
tm := &TemplateManager{}
dir, err := os.MkdirTemp("", "nuclei-templates-*")
require.Nil(t, err)
cfgdir, err := os.MkdirTemp("", "nuclei-config-*")
require.Nil(t, err)
defer func() {
_ = os.RemoveAll(dir)
_ = os.RemoveAll(cfgdir)
}()
templates := []string{
"http/cves/2024/CVE-2024-0001.yaml",
"http/cves/2024/CVE-2024-0002.yaml",
"http/exposures/configs/git-config.yaml",
"dns/dns-saas-service-detection.yaml",
"network/detection/rdp-detect.yaml",
}
release := fakeTemplateRelease{version: "v9.9.9", files: map[string]string{
config.NucleiIgnoreFileName: "tags:\n - fuzz\n",
"README.md": "# nuclei-templates",
}}
for _, path := range templates {
release.files[path] = "id: " + filepath.Base(path)
}
tm := &TemplateManager{fetchLatestRelease: release.fetch}
dir := t.TempDir()
cfgdir := t.TempDir()
// set the config directory to a temporary directory
config.DefaultConfig.SetConfigDir(cfgdir)
@@ -35,36 +44,43 @@ func TestTemplateInstallation(t *testing.T) {
templatesTempDir := filepath.Join(dir, "templates")
config.DefaultConfig.SetTemplatesDir(templatesTempDir)
err = tm.FreshInstallIfNotExists()
if err != nil {
if strings.Contains(err.Error(), "rate limit") {
t.Skip("Skipping test due to github rate limit")
}
require.Nil(t, err)
require.NoError(t, tm.FreshInstallIfNotExists())
for _, path := range templates {
require.FileExists(t, filepath.Join(templatesTempDir, filepath.FromSlash(path)))
}
// we should switch to more fine granular tests for template
// integrity, but for now, we just check that the templates are installed
counter := 0
err = filepath.Walk(templatesTempDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
counter++
}
return nil
})
require.Nil(t, err)
// we should have at least 1000 templates
require.Greater(t, counter, 1000)
require.NoFileExists(t, filepath.Join(templatesTempDir, "README.md"), "meta files are not installed as templates")
// every time we install templates, it should override the ignore file with latest one
require.FileExists(t, config.DefaultConfig.GetActiveIgnoreFilePath())
ownership, err := loadTemplateOwnership(templatesTempDir)
require.NoError(t, err)
require.NotEmpty(t, ownership.Files, "fresh installation should record official template ownership")
t.Logf("Installed %d templates", counter)
require.Len(t, ownership.Files, len(templates), "fresh installation should record ownership of every installed template")
}
func TestUpdateIfOutdatedSkipsReleaseInstalledWhileWaiting(t *testing.T) {
templatesDir := t.TempDir()
stateDir := t.TempDir()
// another process held the update lock and installed the latest release
installer := &config.Config{TemplateVersion: "v2.0.0"}
installer.SetStateDir(stateDir)
installer.SetTemplatesDir(templatesDir)
require.NoError(t, installer.WriteTemplatesConfig())
// this process loaded its state before that install finished
cfg := &config.Config{LatestNucleiTemplatesVersion: "v2.0.0", Logger: gologger.DefaultLogger}
cfg.SetStateDir(stateDir)
cfg.SetTemplatesDir(templatesDir)
previousConfig := config.DefaultConfig
config.DefaultConfig = cfg
t.Cleanup(func() { config.DefaultConfig = previousConfig })
tm := &TemplateManager{fetchLatestRelease: func() (templateRelease, error) {
t.Fatal("downloaded a release another process already installed")
return nil, nil
}}
require.NoError(t, tm.UpdateIfOutdated())
require.Equal(t, "v2.0.0", cfg.TemplateVersion)
}
func TestIsOutdatedVersion(t *testing.T) {
@@ -230,12 +246,18 @@ func benchmarkWriteTemplateOutputs(b *testing.B, syncEachOutput bool) {
rootDir := b.TempDir()
contents := []byte("id: benchmark\ninfo:\n name: Benchmark\n author: test\n severity: info\n")
touchedDirectories := make(map[string]struct{})
writer, err := newTemplateOutputWriter(rootDir)
if err != nil {
b.Fatal(err)
}
defer func() { _ = writer.Close() }()
b.ReportAllocs()
b.ResetTimer()
for index := 0; index < b.N; index++ {
writePath := filepath.Join(rootDir, fmt.Sprintf("group-%d", index%16), fmt.Sprintf("template-%d.yaml", index))
outputResult, err := writeTemplateOutput(rootDir, writePath, contents, 0o644)
// releases nest templates a few directories deep, e.g. http/cves/2024
writePath := filepath.Join(rootDir, fmt.Sprintf("protocol-%d", index%4), fmt.Sprintf("category-%d", index%16), fmt.Sprintf("group-%d", index%64), fmt.Sprintf("template-%d.yaml", index))
outputResult, err := writer.write(writePath, contents, 0o644)
if err != nil {
b.Fatal(err)
}
+17 -8
View File
@@ -1,22 +1,22 @@
package installer
import (
"crypto/sha256"
"fmt"
"os"
"path/filepath"
"time"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/utils/errkit"
)
const templatesUpdateLockName = "nuclei-templates-update.lock"
// withTemplatesUpdateLock serializes template install/update across processes.
// Parallel `go test` packages each have their own sync.Once, so without a
// cross-process lock they race on the shared templates directory (ENOENT,
// partial trees, checksum mismatches).
func withTemplatesUpdateLock(fn func() error) error {
lockPath := filepath.Join(os.TempDir(), templatesUpdateLockName)
// withTemplatesUpdateLock serializes template install/update of templatesDir
// across processes. Parallel `go test` packages each have their own sync.Once,
// so without a cross-process lock they race on a shared templates directory
// (ENOENT, partial trees, checksum mismatches).
func withTemplatesUpdateLock(templatesDir string, fn func() error) error {
lockPath := templatesUpdateLockPath(templatesDir)
deadline := time.Now().Add(10 * time.Minute)
for {
@@ -40,3 +40,12 @@ func withTemplatesUpdateLock(fn func() error) error {
time.Sleep(250 * time.Millisecond)
}
}
// templatesUpdateLockPath derives one lock per templates directory, so updates
// of different directories do not wait on each other. Every alias of a
// directory maps to the same lock, including a symlink whose target does not
// exist yet.
func templatesUpdateLockPath(templatesDir string) string {
sum := sha256.Sum256([]byte(config.CanonicalTemplatesPath(templatesDir)))
return filepath.Join(os.TempDir(), fmt.Sprintf("nuclei-templates-update-%x.lock", sum[:8]))
}
+60
View File
@@ -0,0 +1,60 @@
package installer
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
)
func TestTemplatesUpdateLockPath(t *testing.T) {
t.Run("same directory, same lock", func(t *testing.T) {
dir := t.TempDir()
require.Equal(t, templatesUpdateLockPath(dir), templatesUpdateLockPath(dir+string(filepath.Separator)))
})
t.Run("different directories, different locks", func(t *testing.T) {
require.NotEqual(t, templatesUpdateLockPath(t.TempDir()), templatesUpdateLockPath(t.TempDir()))
})
t.Run("stable across a fresh install creating the directory", func(t *testing.T) {
dir := filepath.Join(t.TempDir(), "missing", "nuclei-templates")
before := templatesUpdateLockPath(dir)
require.NoError(t, os.MkdirAll(dir, 0o755))
require.Equal(t, before, templatesUpdateLockPath(dir))
})
t.Run("symlink to a directory, same lock", func(t *testing.T) {
dir := t.TempDir()
link := filepath.Join(t.TempDir(), "templates-link")
if err := os.Symlink(dir, link); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
require.Equal(t, templatesUpdateLockPath(dir), templatesUpdateLockPath(link))
})
t.Run("dangling symlink, same lock before and after creation", func(t *testing.T) {
parent := t.TempDir()
target := filepath.Join(parent, "nuclei-templates")
link := filepath.Join(parent, "templates-link")
if err := os.Symlink(target, link); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
require.Equal(t, templatesUpdateLockPath(target), templatesUpdateLockPath(link))
require.Equal(t, templatesUpdateLockPath(filepath.Join(target, "nested")), templatesUpdateLockPath(filepath.Join(link, "nested")))
require.NoError(t, os.MkdirAll(target, 0o755))
require.Equal(t, templatesUpdateLockPath(target), templatesUpdateLockPath(link))
})
t.Run("relative dangling symlink, same lock", func(t *testing.T) {
parent := t.TempDir()
target := filepath.Join(parent, "nuclei-templates")
link := filepath.Join(parent, "templates-link")
if err := os.Symlink("nuclei-templates", link); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
require.Equal(t, templatesUpdateLockPath(target), templatesUpdateLockPath(link))
})
}
+1
View File
@@ -44,6 +44,7 @@ import (
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libtelnet"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libtsch"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libvnc"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libwebsocket"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libwmi"
"github.com/projectdiscovery/nuclei/v3/pkg/js/global"
"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
+4
View File
@@ -153,6 +153,9 @@ func (s *session) prepareCommon() {
}
s.config.runtime.SetContextValue("executionId", s.config.opts.ExecutionId)
s.config.runtime.SetContextValue("ctx", executionCtx)
// Go calls derive their context from this, so the execution timeout also
// cancels a library call in flight instead of leaving it running
s.config.runtime.SetContext(executionCtx)
enableRequire(s.config.runtime)
}
@@ -221,6 +224,7 @@ func (s *session) cleanupCommon() {
}
s.config.runtime.RemoveContextValue("executionId")
s.config.runtime.RemoveContextValue("ctx")
s.config.runtime.SetContext(context.TODO())
s.config.runtime.RemoveContextValue("timeoutVariants")
s.config.runtime.RemoveContextValue("proxyURL")
s.config.runtime.RemoveContextValue("customHeaders")
@@ -5,7 +5,7 @@ package {{.PackageName}}
import (
{{$pkgName}} "{{.PackagePath}}"
"github.com/Mzack9999/goja"
"github.com/projectdiscovery/goja"
"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
)
@@ -0,0 +1,31 @@
package websocket
import (
lib_websocket "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/websocket"
"github.com/projectdiscovery/goja"
"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
)
var (
module = gojs.NewGojaModule("nuclei/websocket")
)
func init() {
module.Set(
gojs.Objects{
// Functions
"NewClient": lib_websocket.NewClient,
// Var and consts
// Objects / Classes
"Client": lib_websocket.NewClient,
"Options": gojs.GetClassConstructor[lib_websocket.Options](&lib_websocket.Options{}),
},
).Register()
}
func Enable(runtime *goja.Runtime) {
module.Enable(runtime)
}
+1 -1
View File
@@ -15,7 +15,7 @@ export function NewExecDialer(execID: string): Dialer | null {
/**
* Client is a stateful DCE/RPC + SMB client backed by Mzack9999/goimpacket.
* Client is a stateful DCE/RPC + SMB client backed by projectdiscovery/goimpacket.
* One Client wraps one authenticated SMB session against the target host;
* individual RPC interfaces (SAMR, LSARPC, EPMAPPER, ...) are bound on
* demand by the corresponding helper methods.
+1
View File
@@ -28,4 +28,5 @@ export * as structs from './structs';
export * as telnet from './telnet';
export * as tsch from './tsch';
export * as vnc from './vnc';
export * as websocket from './websocket';
export * as wmi from './wmi';
+9 -4
View File
@@ -3,8 +3,10 @@
/**
* CreateGoldenTicket forges a TGT for the supplied user against the given
* realm using the krbtgt NT hash (or AES key). It returns the ASN.1-encoded
* ticket and the session key. If req.OutputFile is empty no file is written;
* pass an absolute path to also persist a ccache.
* ticket and the session key. goimpacket writes <username>.ccache in the current
* directory, replacing slashes in the username with dots. The destination must
* be inside the template directory unless -allow-local-file-access is enabled.
* req.OutputFile must be empty; custom paths and in-memory generation are unsupported.
* @example
* ```javascript
* const krb = require('nuclei/krbforge');
@@ -27,6 +29,8 @@ export function CreateGoldenTicket(req: TicketRequest): Ticket | null {
* CreateSilverTicket forges a service ticket (TGS) for the supplied SPN. The
* hash supplied must belong to the service account that owns the SPN (e.g.
* the machine account NT hash for cifs/host SPNs).
* File output follows CreateGoldenTicket. req.OutputFile and outputFile must
* be empty; custom paths and in-memory generation are unsupported.
* @example
* ```javascript
* const krb = require('nuclei/krbforge');
@@ -36,11 +40,11 @@ export function CreateGoldenTicket(req: TicketRequest): Ticket | null {
* domain_sid: 'S-1-5-21-1004336348-1177238915-682003330',
* nthash: '31d6cfe0d16ae931b73c59d7e0c089c0',
* spn: 'cifs/server01.acme.local',
* }, '/tmp/silver.ccache');
* });
* log(t.output_file);
* ```
*/
export function CreateSilverTicket(req: TicketRequest, outputFile: string): Ticket | null {
export function CreateSilverTicket(req: TicketRequest, outputFile?: string): Ticket | null {
return null;
}
@@ -89,6 +93,7 @@ export interface TicketRequest {
KVNO?: number,
/** Retained for compatibility; must be empty. */
OutputFile?: string,
}
+30
View File
@@ -51,6 +51,25 @@ export class SMBClient {
}
/**
* Authenticate performs SMB session setup with the supplied credentials without
* accessing or enumerating shares. user may be "DOMAIN\\user" or "user@domain";
* an empty user and password request a null session. Guest and null status come
* from the server's completed session setup, not from the supplied username.
* Each call opens and closes a new session. Rejected authentication, connection
* failures, and policy denials return an error and no result (throw in JavaScript).
* @example
* ```javascript
* const smb = require('nuclei/smb');
* const client = new smb.SMBClient();
* const result = client.Authenticate('acme.com', 445, 'username', 'password');
* const validCredentials = result.Success && !result.IsGuest && !result.IsNullSession;
* ```
*/
public Authenticate(host: string, port: number, user: string, password: string): AuthenticationResult | null {
return null;
}
/**
* ListShares tries to connect to provided host and port
* and list shares by using given credentials.
@@ -147,6 +166,17 @@ export class SMBClient {
}
/**
* AuthenticationResult describes a completed SMB session setup.
* Success also includes guest and null sessions. To validate credentials,
* require Success && !IsGuest && !IsNullSession.
*/
export interface AuthenticationResult {
Success?: boolean,
IsGuest?: boolean,
IsNullSession?: boolean,
}
/**
* ShareEntry is a single file or directory on an SMB share.
*/
+145
View File
@@ -0,0 +1,145 @@
/**
* Client is a WebSocket client for nuclei JS templates.
* @example
* ```javascript
* const websocket = require('nuclei/websocket');
* const client = new websocket.Client('ws://acme.com/socket');
* client.Send('hello');
* const reply = client.Receive();
* client.Close();
* ```
*/
export class Client {
public URL?: string;
public Protocol?: string;
/**
* ResponseHeaders are the handshake response headers, set after connecting.
* Upgrade, Connection and Sec-WebSocket-* headers are validated by the
* handshake and not included; Protocol holds the negotiated subprotocol.
*/
public ResponseHeaders?: Record<string, string>;
// Constructor of Client
constructor(url: string, options?: Options) {}
/**
* Connect performs the opening handshake. It is optional; Send and Receive
* connect on demand.
* @example
* ```javascript
* const websocket = require('nuclei/websocket');
* const client = new websocket.Client('ws://acme.com/socket');
* client.Connect();
* log(client.Protocol);
* ```
*/
public Connect(): void {
return;
}
/**
* Send sends a text message.
* @example
* ```javascript
* const websocket = require('nuclei/websocket');
* const client = new websocket.Client('ws://acme.com/socket');
* client.Send('hello');
* ```
*/
public Send(data: string): void {
return;
}
/**
* SendHex sends a binary message given as hex.
* @example
* ```javascript
* const websocket = require('nuclei/websocket');
* const client = new websocket.Client('ws://acme.com/socket');
* client.SendHex('68656c6c6f');
* ```
*/
public SendHex(data: string): void {
return;
}
/**
* Receive returns the next text or binary message as a string. Pings are
* answered automatically.
* @example
* ```javascript
* const websocket = require('nuclei/websocket');
* const client = new websocket.Client('ws://acme.com/socket');
* const message = client.Receive();
* ```
*/
public Receive(): string | null {
return null;
}
/**
* ReceiveHex returns the next text or binary message as hex.
* @example
* ```javascript
* const websocket = require('nuclei/websocket');
* const client = new websocket.Client('ws://acme.com/socket');
* const message = client.ReceiveHex();
* ```
*/
public ReceiveHex(): string | null {
return null;
}
/**
* Close sends a close frame and closes the connection.
* @example
* ```javascript
* const websocket = require('nuclei/websocket');
* const client = new websocket.Client('ws://acme.com/socket');
* client.Close();
* ```
*/
public Close(): void {
return;
}
}
/**
* Options configures a WebSocket Client. All fields are optional.
* @example
* ```javascript
* const websocket = require('nuclei/websocket');
* const opts = new websocket.Options();
* opts.Headers = {'Origin': 'https://acme.com'};
* opts.Protocols = ['chat'];
* const client = new websocket.Client('wss://acme.com/socket', opts);
* ```
*/
export interface Options {
Headers?: Record<string, string>,
Protocols?: string[],
TimeoutSeconds?: number,
ServerName?: string,
}
+12 -12
View File
@@ -1,4 +1,4 @@
// Package dcerpc exposes a small subset of the Mzack9999/goimpacket DCE/RPC
// Package dcerpc exposes a small subset of the projectdiscovery/goimpacket DCE/RPC
// stack to nuclei javascript templates. It is the entry point for AD attack
// templates that need to talk EPMAPPER / SAMR / LSARPC / SVCCTL / TSCH / WINREG
// / SRVSVC to a domain controller or member server.
@@ -33,16 +33,16 @@ import (
"strconv"
"time"
gpatexec "github.com/Mzack9999/goimpacket/pkg/atexec"
gprpc "github.com/Mzack9999/goimpacket/pkg/dcerpc"
gpepm "github.com/Mzack9999/goimpacket/pkg/dcerpc/epmapper"
gplsa "github.com/Mzack9999/goimpacket/pkg/dcerpc/lsarpc"
gpsamr "github.com/Mzack9999/goimpacket/pkg/dcerpc/samr"
gpsvcctl "github.com/Mzack9999/goimpacket/pkg/dcerpc/svcctl"
gptsch "github.com/Mzack9999/goimpacket/pkg/dcerpc/tsch"
gpsession "github.com/Mzack9999/goimpacket/pkg/session"
gpsmb "github.com/Mzack9999/goimpacket/pkg/smb"
gpsmbexec "github.com/Mzack9999/goimpacket/pkg/smbexec"
gpatexec "github.com/projectdiscovery/goimpacket/pkg/atexec"
gprpc "github.com/projectdiscovery/goimpacket/pkg/dcerpc"
gpepm "github.com/projectdiscovery/goimpacket/pkg/dcerpc/epmapper"
gplsa "github.com/projectdiscovery/goimpacket/pkg/dcerpc/lsarpc"
gpsamr "github.com/projectdiscovery/goimpacket/pkg/dcerpc/samr"
gpsvcctl "github.com/projectdiscovery/goimpacket/pkg/dcerpc/svcctl"
gptsch "github.com/projectdiscovery/goimpacket/pkg/dcerpc/tsch"
gpsession "github.com/projectdiscovery/goimpacket/pkg/session"
gpsmb "github.com/projectdiscovery/goimpacket/pkg/smb"
gpsmbexec "github.com/projectdiscovery/goimpacket/pkg/smbexec"
"github.com/projectdiscovery/goja"
"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/smbsession"
@@ -60,7 +60,7 @@ type DomainUser = gpsamr.DomainUser
type LookupResult = gplsa.LookupResult
type (
// Client is a stateful DCE/RPC + SMB client backed by Mzack9999/goimpacket.
// Client is a stateful DCE/RPC + SMB client backed by projectdiscovery/goimpacket.
// One Client wraps one authenticated SMB session against the target host;
// individual RPC interfaces (SAMR, LSARPC, EPMAPPER, ...) are bound on
// demand by the corresponding helper methods.
+4 -4
View File
@@ -3,10 +3,10 @@ package dcerpc
import (
"fmt"
gpsrvsvc "github.com/Mzack9999/goimpacket/pkg/dcerpc/srvsvc"
gpsvcctl "github.com/Mzack9999/goimpacket/pkg/dcerpc/svcctl"
winstation "github.com/Mzack9999/goimpacket/pkg/dcerpc/tsts"
gpwkssvc "github.com/Mzack9999/goimpacket/pkg/dcerpc/wkssvc"
gpsrvsvc "github.com/projectdiscovery/goimpacket/pkg/dcerpc/srvsvc"
gpsvcctl "github.com/projectdiscovery/goimpacket/pkg/dcerpc/svcctl"
winstation "github.com/projectdiscovery/goimpacket/pkg/dcerpc/tsts"
gpwkssvc "github.com/projectdiscovery/goimpacket/pkg/dcerpc/wkssvc"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
)
+4 -4
View File
@@ -3,10 +3,10 @@ package dcerpc
import (
"testing"
gpsrvsvc "github.com/Mzack9999/goimpacket/pkg/dcerpc/srvsvc"
gpsvcctl "github.com/Mzack9999/goimpacket/pkg/dcerpc/svcctl"
winstation "github.com/Mzack9999/goimpacket/pkg/dcerpc/tsts"
gpwkssvc "github.com/Mzack9999/goimpacket/pkg/dcerpc/wkssvc"
gpsrvsvc "github.com/projectdiscovery/goimpacket/pkg/dcerpc/srvsvc"
gpsvcctl "github.com/projectdiscovery/goimpacket/pkg/dcerpc/svcctl"
winstation "github.com/projectdiscovery/goimpacket/pkg/dcerpc/tsts"
gpwkssvc "github.com/projectdiscovery/goimpacket/pkg/dcerpc/wkssvc"
"github.com/stretchr/testify/require"
)
+1 -1
View File
@@ -1,7 +1,7 @@
package dcerpc
import (
gptr "github.com/Mzack9999/goimpacket/pkg/transport"
gptr "github.com/projectdiscovery/goimpacket/pkg/transport"
"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/gptransport"
)
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"net"
"time"
gpwmiexec "github.com/Mzack9999/goimpacket/pkg/wmiexec"
gpwmiexec "github.com/projectdiscovery/goimpacket/pkg/wmiexec"
"github.com/oiweiwei/go-msrpc/dcerpc"
"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/gptransport"
+2 -2
View File
@@ -1,4 +1,4 @@
// Package gptransport binds Mzack9999/goimpacket TCP dials to nuclei's
// Package gptransport binds projectdiscovery/goimpacket TCP dials to nuclei's
// per-execution fastdialer and network policy.
//
// Import this package (directly or via smbsession / dcerpc) so init() installs
@@ -11,7 +11,7 @@ import (
"fmt"
"net"
gptr "github.com/Mzack9999/goimpacket/pkg/transport"
gptr "github.com/projectdiscovery/goimpacket/pkg/transport"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
)
+23 -62
View File
@@ -9,16 +9,13 @@ package krbforge
import (
"encoding/hex"
"fmt"
"os"
"path/filepath"
"strings"
gpkrb "github.com/Mzack9999/goimpacket/pkg/kerberos"
gpkrb "github.com/projectdiscovery/goimpacket/pkg/kerberos"
"github.com/projectdiscovery/goja"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
filepathutil "github.com/projectdiscovery/nuclei/v3/pkg/utils/filepath"
)
// TicketRequest mirrors gopacket's TicketConfig with json-friendly tags.
@@ -35,7 +32,10 @@ type TicketRequest struct {
ExtraSIDs []string `json:"extra_sids,omitempty"`
DurationHours int `json:"duration_hours,omitempty"`
KVNO int `json:"kvno,omitempty"`
OutputFile string `json:"output_file,omitempty"`
// OutputFile is retained for compatibility and must be empty.
// goimpacket always writes its default ccache file.
OutputFile string `json:"output_file,omitempty"`
}
// Ticket is the forged ticket plus metadata.
@@ -48,9 +48,10 @@ type Ticket struct {
// CreateGoldenTicket forges a TGT for the supplied user against the given
// realm using the krbtgt NT hash (or AES key). It returns the ASN.1-encoded
// ticket and the session key. If req.OutputFile is empty no file is written;
// pass an output path allowed by the nuclei file sandbox to also persist a
// ccache. Use -allow-local-file-access to allow writing outside the sandbox.
// ticket and the session key. goimpacket writes <username>.ccache in the current
// directory, replacing slashes in the username with dots. The destination must
// be inside the template directory unless -allow-local-file-access is enabled.
// req.OutputFile must be empty; custom paths and in-memory generation are unsupported.
//
// @example
// ```javascript
@@ -90,16 +91,14 @@ func createGoldenTicket(executionID string, req TicketRequest) (*Ticket, error)
return nil, err
}
if cfg.OutputFile == "" {
cfg.OutputFile = "-"
}
return createTicket(cfg)
}
// CreateSilverTicket forges a service ticket (TGS) for the supplied SPN. The
// hash supplied must belong to the service account that owns the SPN (e.g.
// the machine account NT hash for cifs/host SPNs).
// File output follows CreateGoldenTicket. req.OutputFile and outputFile must
// be empty; custom paths and in-memory generation are unsupported.
//
// @example
// ```javascript
@@ -111,7 +110,7 @@ func createGoldenTicket(executionID string, req TicketRequest) (*Ticket, error)
// domain_sid: 'S-1-5-21-1004336348-1177238915-682003330',
// nthash: '31d6cfe0d16ae931b73c59d7e0c089c0',
// spn: 'cifs/server01.acme.local',
// }, '/tmp/silver.ccache');
// });
//
// log(t.output_file);
// ```
@@ -150,10 +149,6 @@ func createSilverTicket(executionID string, req TicketRequest, outputFile string
return nil, err
}
if cfg.OutputFile == "" {
cfg.OutputFile = "-"
}
return createTicket(cfg)
}
@@ -167,18 +162,23 @@ func createTicket(cfg *gpkrb.TicketConfig) (*Ticket, error) {
HexTicket: hex.EncodeToString(res.Ticket),
HexKey: hex.EncodeToString(res.SessionKey),
EncType: res.EncType,
OutputFile: cfg.OutputFile,
OutputFile: res.Filename,
}, nil
}
func buildConfig(executionID string, req TicketRequest, outputFile string) (*gpkrb.TicketConfig, error) {
if outputFile == "" {
outputFile = req.OutputFile
if outputFile != "" || req.OutputFile != "" {
return nil, fmt.Errorf("goimpacket writes <username>.ccache in the current directory: custom output paths and in-memory ticket generation are not supported; leave outputFile empty")
}
normalizedOutputFile, err := normalizeOutputFile(executionID, outputFile)
// Match goimpacket's destination before it performs the file write.
filename := strings.ReplaceAll(req.Username, "/", ".") + ".ccache"
destination, err := filepath.Abs(filename)
if err != nil {
return nil, err
return nil, fmt.Errorf("resolve ccache destination %q: %w", filename, err)
}
if _, err := protocolstate.NormalizePathWithExecutionId(executionID, destination); err != nil {
return nil, fmt.Errorf("validate ccache destination: %w", err)
}
return &gpkrb.TicketConfig{
@@ -194,48 +194,9 @@ func buildConfig(executionID string, req TicketRequest, outputFile string) (*gpk
ExtraSIDs: req.ExtraSIDs,
Duration: req.DurationHours,
KVNO: req.KVNO,
OutputFile: normalizedOutputFile,
}, nil
}
func normalizeOutputFile(executionID string, outputFile string) (string, error) {
if outputFile == "" || outputFile == "-" {
return outputFile, nil
}
if protocolstate.IsLfaAllowed(&types.Options{ExecutionId: executionID}) {
// Preserve the existing relative-path behavior when
// -allow-local-file-access is enabled: avoid implicit CWD writes by
// placing relative ccache paths in temp.
if !filepath.IsAbs(outputFile) {
outputFile = filepath.Join(os.TempDir(), outputFile)
}
normalized, err := filepath.Abs(outputFile)
if err != nil {
return "", fmt.Errorf("normalize output file %q: %w", outputFile, err)
}
return normalized, nil
}
normalized := outputFile
if !filepath.IsAbs(normalized) {
normalized = filepath.Join(config.DefaultConfig.GetTemplateDir(), normalized)
}
normalized, err := filepath.Abs(normalized)
if err != nil {
return "", fmt.Errorf("normalize output file %q: %w", outputFile, err)
}
if filepathutil.IsPathWithinDirectory(normalized, config.DefaultConfig.GetTemplateDir()) {
return normalized, nil
}
return "", fmt.Errorf("path %v is outside nuclei-template directory and -allow-local-file-access is not enabled", outputFile)
}
func exportTicketRequest(vm *goja.Runtime, value goja.Value) (TicketRequest, error) {
var req TicketRequest
if err := vm.ExportTo(value, &req); err != nil {
+135 -70
View File
@@ -13,103 +13,168 @@ import (
"github.com/stretchr/testify/require"
)
func TestBuildConfigRejectsOutputFileOutsideSandbox(t *testing.T) {
setTemplateDir(t)
executionID := "deny-" + t.Name()
setLocalFileAccess(executionID, false)
outsidePath := filepath.Join(t.TempDir(), "ticket.ccache")
func TestCreateTicketDefaultOutput(t *testing.T) {
for _, golden := range []bool{true, false} {
for _, location := range []string{"sandbox", "outside denied", "outside allowed"} {
t.Run(fmt.Sprintf("golden=%v/%s", golden, location), func(t *testing.T) {
templatesDir := setTemplateDir(t)
cwd := templatesDir
if location != "sandbox" {
cwd = t.TempDir()
}
t.Chdir(cwd)
executionID := t.Name()
setLocalFileAccess(executionID, location == "outside allowed")
t.Run("request output_file", func(t *testing.T) {
req := validTicketRequest()
req.OutputFile = outsidePath
req := validTicketRequest()
// goimpacket replaces slashes in SPN-style usernames with dots.
req.Username = "service/http"
var ticket *Ticket
var err error
if golden {
req.SPN = ""
ticket, err = createGoldenTicket(executionID, req)
} else {
ticket, err = createSilverTicket(executionID, req, "")
}
filename := "service.http.ccache"
if location == "outside denied" {
require.ErrorContains(t, err, "outside nuclei-template directory")
require.Nil(t, ticket)
_, err = os.Stat(filepath.Join(cwd, filename))
require.ErrorIs(t, err, os.ErrNotExist)
return
}
_, err := buildConfig(executionID, req, "")
require.Error(t, err)
require.Contains(t, err.Error(), "-allow-local-file-access is not enabled")
})
t.Run("silver outputFile argument", func(t *testing.T) {
_, err := buildConfig(executionID, validTicketRequest(), outsidePath)
require.Error(t, err)
require.Contains(t, err.Error(), "-allow-local-file-access is not enabled")
})
require.NoError(t, err)
require.Equal(t, filename, ticket.OutputFile)
require.NotEmpty(t, ticket.HexTicket)
require.NotEmpty(t, ticket.HexKey)
body, err := os.ReadFile(filepath.Join(cwd, ticket.OutputFile))
require.NoError(t, err)
require.NotEmpty(t, body)
})
}
}
}
func TestBuildConfigAllowsOutputFileInsideSandbox(t *testing.T) {
templatesDir := setTemplateDir(t)
executionID := "sandbox-" + t.Name()
setLocalFileAccess(executionID, false)
outputPath := filepath.Join(templatesDir, "generated", "ticket.ccache")
req := validTicketRequest()
req.OutputFile = outputPath
cfg, err := buildConfig(executionID, req, "")
require.NoError(t, err)
require.Equal(t, outputPath, cfg.OutputFile)
}
func TestBuildConfigNormalizesRelativeOutputFileInsideSandbox(t *testing.T) {
templatesDir := setTemplateDir(t)
executionID := "relative-" + t.Name()
setLocalFileAccess(executionID, false)
cfg, err := buildConfig(executionID, validTicketRequest(), filepath.Join("generated", "ticket.ccache"))
require.NoError(t, err)
require.Equal(t, filepath.Join(templatesDir, "generated", "ticket.ccache"), cfg.OutputFile)
}
func TestBuildConfigAllowsOutputFileWhenLocalFileAccessEnabled(t *testing.T) {
setTemplateDir(t)
executionID := "allow-" + t.Name()
func TestCreateTicketRejectsUnsupportedOutput(t *testing.T) {
cwd := setTemplateDir(t)
t.Chdir(cwd)
executionID := t.Name()
setLocalFileAccess(executionID, true)
outsidePath := filepath.Join(t.TempDir(), "ticket.ccache")
cfg, err := buildConfig(executionID, validTicketRequest(), outsidePath)
require.NoError(t, err)
require.Equal(t, outsidePath, cfg.OutputFile)
for _, output := range []string{"-", "custom.ccache", filepath.Join(cwd, "custom.ccache")} {
for _, method := range []string{"golden request", "silver request", "silver argument"} {
t.Run(method+"/"+output, func(t *testing.T) {
req := validTicketRequest()
var ticket *Ticket
var err error
switch method {
case "golden request":
req.OutputFile = output
ticket, err = createGoldenTicket(executionID, req)
case "silver request":
req.OutputFile = output
ticket, err = createSilverTicket(executionID, req, "")
case "silver argument":
ticket, err = createSilverTicket(executionID, req, output)
}
require.ErrorContains(t, err, "custom output paths and in-memory ticket generation are not supported")
require.Nil(t, ticket)
files, err := os.ReadDir(cwd)
require.NoError(t, err)
require.Empty(t, files)
})
}
}
}
func TestCreateGoldenTicketRejectsOutputFileOutsideSandboxFromRuntime(t *testing.T) {
func TestCreateTicketReportsWriteFailure(t *testing.T) {
cwd := setTemplateDir(t)
t.Chdir(cwd)
executionID := t.Name()
setLocalFileAccess(executionID, false)
require.NoError(t, os.Mkdir(filepath.Join(cwd, "Administrator.ccache"), 0700))
ticket, err := createGoldenTicket(executionID, validTicketRequest())
require.ErrorContains(t, err, "failed to save ccache")
require.Nil(t, ticket)
}
func TestCreateTicketRejectsDefaultOutputSymlink(t *testing.T) {
cwd := setTemplateDir(t)
t.Chdir(cwd)
executionID := t.Name()
setLocalFileAccess(executionID, false)
outside := filepath.Join(t.TempDir(), "ticket.ccache")
require.NoError(t, os.WriteFile(outside, []byte("unchanged"), 0600))
if err := os.Symlink(outside, filepath.Join(cwd, "Administrator.ccache")); err != nil {
t.Skipf("symlink unavailable: %v", err)
}
ticket, err := createGoldenTicket(executionID, validTicketRequest())
require.ErrorContains(t, err, "outside nuclei-template directory")
require.Nil(t, ticket)
body, err := os.ReadFile(outside)
require.NoError(t, err)
require.Equal(t, "unchanged", string(body))
}
func TestCreateTicketRejectsDefaultOutputHardLink(t *testing.T) {
cwd := setTemplateDir(t)
t.Chdir(cwd)
executionID := t.Name()
setLocalFileAccess(executionID, false)
outside := filepath.Join(t.TempDir(), "ticket.ccache")
require.NoError(t, os.WriteFile(outside, []byte("unchanged"), 0600))
if err := os.Link(outside, filepath.Join(cwd, "Administrator.ccache")); err != nil {
t.Skipf("hard link unavailable: %v", err)
}
ticket, err := createGoldenTicket(executionID, validTicketRequest())
require.ErrorContains(t, err, "hard link")
require.Nil(t, ticket)
body, err := os.ReadFile(outside)
require.NoError(t, err)
require.Equal(t, "unchanged", string(body))
}
func TestCreateGoldenTicketDeniesDefaultOutputFromRuntime(t *testing.T) {
setTemplateDir(t)
executionID := "runtime-" + t.Name()
cwd := t.TempDir()
t.Chdir(cwd)
executionID := t.Name()
setLocalFileAccess(executionID, false)
runtime := goja.New()
runtime.SetContextValue("executionId", executionID)
req := validTicketRequest()
req.OutputFile = filepath.Join(t.TempDir(), "ticket.ccache")
var panicValue any
func() {
defer func() {
panicValue = recover()
}()
defer func() { panicValue = recover() }()
CreateGoldenTicket(goja.FunctionCall{
Arguments: []goja.Value{runtime.ToValue(req)},
}, runtime)
}()
require.NotNil(t, panicValue)
require.Contains(t, fmt.Sprint(panicValue), "-allow-local-file-access is not enabled")
require.Contains(t, fmt.Sprint(panicValue), "outside nuclei-template directory")
_, err := os.Stat(filepath.Join(cwd, req.Username+".ccache"))
require.ErrorIs(t, err, os.ErrNotExist)
}
func TestCreateSilverTicketDoesNotWriteDefaultCCache(t *testing.T) {
setTemplateDir(t)
executionID := "silver-" + t.Name()
func TestBuildConfigPreservesIdentity(t *testing.T) {
t.Chdir(setTemplateDir(t))
executionID := t.Name()
setLocalFileAccess(executionID, false)
cwd := t.TempDir()
t.Chdir(cwd)
ticket, err := createSilverTicket(executionID, validTicketRequest(), "")
req := validTicketRequest()
req.Username = "service/http"
cfg, err := buildConfig(executionID, req, "")
require.NoError(t, err)
require.Equal(t, "-", ticket.OutputFile)
_, err = os.Stat(filepath.Join(cwd, "Administrator.ccache"))
require.ErrorIs(t, err, os.ErrNotExist)
require.Equal(t, req.Username, cfg.Username)
require.Equal(t, req.Domain, cfg.Domain)
}
func setTemplateDir(t *testing.T) string {
+1 -1
View File
@@ -14,7 +14,7 @@ package krbroast
import (
"fmt"
gpkrb "github.com/Mzack9999/goimpacket/pkg/kerberos"
gpkrb "github.com/projectdiscovery/goimpacket/pkg/kerberos"
"github.com/projectdiscovery/goja"
"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/dcerpc"
+5 -5
View File
@@ -1,4 +1,4 @@
// Package secretsdump exposes Mzack9999/goimpacket's DCSync (DRSUAPI
// Package secretsdump exposes projectdiscovery/goimpacket's DCSync (DRSUAPI
// IDL_DRSGetNCChanges) primitive to nuclei javascript templates.
//
// DCSync requires Replicating Directory Changes / Replicating Directory
@@ -14,10 +14,10 @@ import (
"encoding/hex"
"fmt"
gprpc "github.com/Mzack9999/goimpacket/pkg/dcerpc"
gpdrs "github.com/Mzack9999/goimpacket/pkg/dcerpc/drsuapi"
gpsession "github.com/Mzack9999/goimpacket/pkg/session"
gpsmb "github.com/Mzack9999/goimpacket/pkg/smb"
gprpc "github.com/projectdiscovery/goimpacket/pkg/dcerpc"
gpdrs "github.com/projectdiscovery/goimpacket/pkg/dcerpc/drsuapi"
gpsession "github.com/projectdiscovery/goimpacket/pkg/session"
gpsmb "github.com/projectdiscovery/goimpacket/pkg/smb"
"github.com/projectdiscovery/goja"
"github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
+41
View File
@@ -33,6 +33,7 @@ import (
"time"
"github.com/praetorian-inc/fingerprintx/pkg/plugins"
"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/smbsession"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/zmap/zgrab2/lib/smb/smb"
)
@@ -48,6 +49,15 @@ type (
// ```
SMBClient struct{}
// AuthenticationResult describes a completed SMB session setup.
// Success also includes guest and null sessions. To validate credentials,
// require Success && !IsGuest && !IsNullSession.
AuthenticationResult struct {
Success bool `json:"success"`
IsGuest bool `json:"is_guest"`
IsNullSession bool `json:"is_null_session"`
}
// ProtocolInfo summarises dialects and capabilities discovered during
// SMB negotiation (nmap smb-protocols style).
// @example
@@ -146,6 +156,37 @@ func (c *SMBClient) ListSMBv2Metadata(ctx context.Context, host string, port int
return memoizedcollectSMBv2Metadata(ctx, executionId, host, port, 5*time.Second)
}
// Authenticate performs SMB session setup with the supplied credentials without
// accessing or enumerating shares. user may be "DOMAIN\\user" or "user@domain";
// an empty user and password request a null session. Guest and null status come
// from the server's completed session setup, not from the supplied username.
// Each call opens and closes a new session. Rejected authentication, connection
// failures, and policy denials return an error and no result (throw in JavaScript).
// @example
// ```javascript
// const smb = require('nuclei/smb');
// const client = new smb.SMBClient();
// const result = client.Authenticate('acme.com', 445, 'username', 'password');
// const validCredentials = result.Success && !result.IsGuest && !result.IsNullSession;
// ```
func (c *SMBClient) Authenticate(ctx context.Context, host string, port int, user, password string) (*AuthenticationResult, error) {
executionID, ok := ctx.Value("executionId").(string)
if !ok || executionID == "" {
return nil, fmt.Errorf("smb: missing executionId in context")
}
sess, err := smbsession.Dial(ctx, executionID, host, port, smbsession.Creds{User: user, Password: password})
if err != nil {
return nil, err
}
defer sess.Close()
setup := sess.Native().Session
return &AuthenticationResult{
Success: true,
IsGuest: setup.IsGuest(),
IsNullSession: setup.IsAnonymous(),
}, nil
}
// ListShares tries to connect to provided host and port
// and list shares by using given credentials.
// Credentials cannot be blank. guest or anonymous credentials
+246
View File
@@ -0,0 +1,246 @@
package smb_test
import (
"context"
"encoding/asn1"
"encoding/binary"
"errors"
"fmt"
"io"
"net"
"testing"
"time"
"github.com/projectdiscovery/goja"
"github.com/projectdiscovery/goja_nodejs/require"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libsmb"
"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/smb"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
assert "github.com/stretchr/testify/require"
)
func TestAuthenticateSessionStatus(t *testing.T) {
for _, tc := range []struct {
name string
user string
flags uint16
status uint32
want smb.AuthenticationResult
}{
{"regular", `DOMAIN\user`, 0, 0, smb.AuthenticationResult{Success: true}},
{"guest username without guest flag", "guest", 0, 0, smb.AuthenticationResult{Success: true}},
{"guest fallback", "unknown-user", 1, 0, smb.AuthenticationResult{Success: true, IsGuest: true}},
{"null session", "", 2, 0, smb.AuthenticationResult{Success: true, IsNullSession: true}},
{"both flags", "user", 3, 0, smb.AuthenticationResult{Success: true, IsGuest: true, IsNullSession: true}},
{"unrelated flag", "user", 0x8000, 0, smb.AuthenticationResult{Success: true}},
{"rejected credentials", "user", 0, 0xc000006d, smb.AuthenticationResult{}},
} {
for _, javascript := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/javascript=%v", tc.name, javascript), func(t *testing.T) {
password := "password"
if tc.user == "" {
password = ""
}
executionID := t.Name()
assert.NoError(t, protocolstate.Init(&types.Options{ExecutionId: executionID}))
t.Cleanup(func() { protocolstate.Close(executionID) })
port, done := serveAuthentication(t, tc.flags, tc.status)
// production runs carry the js execution timeout in their context;
// without one a stalled exchange holds the package until go
// test's 1h timeout instead of failing this test
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if javascript {
vm := goja.New()
vm.SetContext(ctx)
new(require.Registry).Enable(vm)
vm.SetContextValue("executionId", executionID)
assert.NoError(t, vm.Set("port", port))
assert.NoError(t, vm.Set("user", tc.user))
assert.NoError(t, vm.Set("password", password))
value, err := vm.RunString(`
const client = new (require('nuclei/smb').SMBClient)();
const result = client.Authenticate('127.0.0.1', port, user, password);
[result.Success, result.IsGuest, result.IsNullSession];
`)
if tc.status != 0 {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.Equal(t, []interface{}{tc.want.Success, tc.want.IsGuest, tc.want.IsNullSession}, value.Export())
}
} else {
ctx := context.WithValue(ctx, "executionId", executionID) //nolint:staticcheck
result, err := (&smb.SMBClient{}).Authenticate(ctx, "127.0.0.1", port, tc.user, password)
if tc.status != 0 {
assert.Error(t, err)
assert.Nil(t, result)
} else {
assert.NoError(t, err)
assert.Equal(t, tc.want, *result)
}
}
select {
case err := <-done:
assert.NoError(t, err)
case <-time.After(6 * time.Second):
t.Fatal("SMB authentication exchange did not finish")
}
})
}
}
}
func TestAuthenticateMissingExecutionID(t *testing.T) {
for _, tc := range []struct {
name string
executionID any
}{
{"missing", nil},
{"wrong type", 42},
{"empty", ""},
} {
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
if tc.executionID != nil {
ctx = context.WithValue(ctx, "executionId", tc.executionID) //nolint:staticcheck
}
result, err := (&smb.SMBClient{}).Authenticate(ctx, "127.0.0.1", 445, "user", "password")
assert.EqualError(t, err, "smb: missing executionId in context")
assert.Nil(t, result)
})
}
}
func TestAuthenticateDeniedHost(t *testing.T) {
const executionID = "smb-auth-denied"
assert.NoError(t, protocolstate.Init(&types.Options{ExecutionId: executionID, ExcludeTargets: []string{"127.0.0.1"}}))
t.Cleanup(func() { protocolstate.Close(executionID) })
ctx := context.WithValue(context.Background(), "executionId", executionID) //nolint:staticcheck
result, err := (&smb.SMBClient{}).Authenticate(ctx, "127.0.0.1", 445, "user", "password")
assert.EqualError(t, err, protocolstate.ErrHostDenied.Msgf("127.0.0.1").Error())
assert.Nil(t, result)
}
func TestAuthenticateCanceledContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
ctx = context.WithValue(ctx, "executionId", "smb-auth-canceled") //nolint:staticcheck
result, err := (&smb.SMBClient{}).Authenticate(ctx, "127.0.0.1", 445, "user", "password")
assert.ErrorIs(t, err, context.Canceled)
assert.Nil(t, result)
}
// serveAuthentication completes the NTLM exchange and rejects any command other
// than negotiate, session setup, and logoff. This detects share enumeration even
// when it would fail or its result would be ignored by Authenticate.
func serveAuthentication(t *testing.T, flags uint16, status uint32) (int, <-chan error) {
t.Helper()
listener, err := net.Listen("tcp", "127.0.0.1:0")
assert.NoError(t, err)
t.Cleanup(func() { _ = listener.Close() })
done := make(chan error, 1)
go func() {
conn, err := listener.Accept()
if err != nil {
done <- err
return
}
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
err = authenticationExchange(conn, flags, status)
done <- errors.Join(err, conn.Close())
}()
return listener.Addr().(*net.TCPAddr).Port, done
}
func authenticationExchange(conn net.Conn, flags uint16, status uint32) error {
// A minimal NTLM challenge with target-info terminator and a fixed nonce.
challenge := make([]byte, 52)
copy(challenge, "NTLMSSP\x00")
binary.LittleEndian.PutUint32(challenge[8:], 2)
binary.LittleEndian.PutUint32(challenge[20:], 0x00880205)
copy(challenge[24:], "12345678")
binary.LittleEndian.PutUint16(challenge[40:], 4)
binary.LittleEndian.PutUint16(challenge[42:], 4)
binary.LittleEndian.PutUint32(challenge[44:], 48)
token, err := asn1.MarshalWithParams(struct {
State asn1.Enumerated `asn1:"explicit,tag:0"`
Token []byte `asn1:"explicit,tag:2"`
}{1, challenge}, "explicit,tag:1")
if err != nil {
return err
}
for step, command := range []uint16{0, 1, 1, 2} {
request, err := readSMBPacket(conn)
if err != nil {
return err
}
if got := binary.LittleEndian.Uint16(request[12:]); got != command {
return fmt.Errorf("step %d: unexpected SMB command %d, want %d (share access is forbidden)", step, got, command)
}
response := make([]byte, 64)
copy(response, "\xfeSMB")
binary.LittleEndian.PutUint16(response[4:], 64)
binary.LittleEndian.PutUint16(response[12:], command)
binary.LittleEndian.PutUint16(response[14:], 1)
binary.LittleEndian.PutUint32(response[16:], 1) // server-to-client
copy(response[24:32], request[24:32]) // message ID
binary.LittleEndian.PutUint64(response[40:], 1) // session ID
switch step {
case 0:
body := make([]byte, 65)
binary.LittleEndian.PutUint16(body, 65)
binary.LittleEndian.PutUint16(body[2:], 1) // signing enabled
binary.LittleEndian.PutUint16(body[4:], 0x0210)
for _, offset := range []int{28, 32, 36} {
binary.LittleEndian.PutUint32(body[offset:], 65536)
}
response = append(response, body...)
case 1, 2:
body := make([]byte, 8)
binary.LittleEndian.PutUint16(body, 9)
binary.LittleEndian.PutUint16(body[4:], 72)
if step == 1 {
binary.LittleEndian.PutUint32(response[8:], 0xc0000016) // more processing required
binary.LittleEndian.PutUint16(body[6:], uint16(len(token)))
body = append(body, token...)
} else {
binary.LittleEndian.PutUint32(response[8:], status)
binary.LittleEndian.PutUint16(body[2:], flags)
}
response = append(response, body...)
case 3:
// Closing the transport after logoff avoids signing a response; the
// test server does not validate credentials or derive signing keys.
return nil
}
if err := binary.Write(conn, binary.BigEndian, uint32(len(response))); err != nil {
return err
}
if _, err := conn.Write(response); err != nil {
return err
}
if step == 2 && status != 0 {
_, err := readSMBPacket(conn)
if err != io.EOF {
return fmt.Errorf("expected connection close after rejected authentication, got %v", err)
}
return nil
}
}
return nil
}
func readSMBPacket(conn net.Conn) ([]byte, error) {
var size uint32
if err := binary.Read(conn, binary.BigEndian, &size); err != nil {
return nil, err
}
if size < 64 || size > 65536 {
return nil, fmt.Errorf("unexpected SMB packet size %d", size)
}
packet := make([]byte, size)
_, err := io.ReadFull(conn, packet)
return packet, err
}
+7 -17
View File
@@ -9,8 +9,8 @@ import (
"strings"
"time"
gpsession "github.com/Mzack9999/goimpacket/pkg/session"
gpsmb "github.com/Mzack9999/goimpacket/pkg/smb"
gpsession "github.com/projectdiscovery/goimpacket/pkg/session"
gpsmb "github.com/projectdiscovery/goimpacket/pkg/smb"
"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/gptransport"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
@@ -92,23 +92,13 @@ func Dial(ctx context.Context, executionID, host string, port int, creds Creds)
target := gpsession.Target{Host: host, Port: port}
client := gpsmb.NewClientWithDialer(target, gpCreds, gptransport.NewExecDialer(executionID))
errCh := make(chan error, 1)
go func() {
errCh <- client.Connect()
}()
select {
case <-ctx.Done():
client.Close()
// Drain connect result so the goroutine can exit; ignore the outcome
// because cancellation already won the race.
<-errCh
return nil, ctx.Err()
case err := <-errCh:
if err != nil {
return nil, err
if err := client.ConnectContext(ctx); err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return nil, ctxErr
}
return &Session{client: client}, nil
return nil, err
}
return &Session{client: client}, nil
}
// FromClient wraps an already-connected goimpacket SMB client (e.g. dcerpc's).
+280
View File
@@ -0,0 +1,280 @@
package websocket
import (
"context"
"crypto/tls"
"encoding/hex"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"time"
"github.com/gobwas/ws"
"github.com/gobwas/ws/wsutil"
"github.com/projectdiscovery/goja"
jsnet "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/net"
"github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
)
const (
defaultTimeout = 5 * time.Second
// maxMessageSize caps a received message, matching the net module's read limit.
maxMessageSize = 8 * 1024 * 1024
)
type (
// Options configures a WebSocket Client. All fields are optional.
// @example
// ```javascript
// const websocket = require('nuclei/websocket');
// const opts = new websocket.Options();
// opts.Headers = {'Origin': 'https://acme.com'};
// opts.Protocols = ['chat'];
// const client = new websocket.Client('wss://acme.com/socket', opts);
// ```
Options struct {
// Headers are extra HTTP headers sent with the opening handshake.
Headers map[string]string
// Protocols are the subprotocols offered in the handshake, in preference order.
Protocols []string
// TimeoutSeconds bounds the handshake and each send or receive.
// 0 uses the scan's TCP read timeout.
TimeoutSeconds int
// ServerName overrides the TLS server name for wss:// URLs.
ServerName string
}
)
type (
// Client is a WebSocket client for nuclei JS templates.
// @example
// ```javascript
// const websocket = require('nuclei/websocket');
// const client = new websocket.Client('ws://acme.com/socket');
// client.Send('hello');
// const reply = client.Receive();
// client.Close();
// ```
Client struct {
// URL is the ws:// or wss:// endpoint.
URL string
// Protocol is the subprotocol the server accepted, set after connecting.
Protocol string
// ResponseHeaders are the handshake response headers, set after connecting.
// Upgrade, Connection and Sec-WebSocket-* headers are validated by the
// handshake and not included; Protocol holds the negotiated subprotocol.
ResponseHeaders map[string]string
nj *utils.NucleiJS
executionID string
opts Options
timeout time.Duration
conn net.Conn
reader io.Reader
}
)
// NewClient creates a new WebSocket client for the given ws:// or wss:// URL.
//
// Constructor: constructor(public url: string, public options?: Options)
func NewClient(call goja.ConstructorCall, runtime *goja.Runtime) *goja.Object {
c := &Client{nj: utils.NewNucleiJS(runtime)}
c.nj.ObjectSig = "Client(url, {Options})"
rawURL, _ := c.nj.GetArg(call.Arguments, 0).(string)
c.nj.Require(rawURL != "", "websocket url cannot be empty")
if len(call.Arguments) > 1 && !goja.IsUndefined(call.Arguments[1]) && !goja.IsNull(call.Arguments[1]) {
if err := runtime.ExportTo(call.Arguments[1], &c.opts); err != nil {
c.nj.HandleError(err, "invalid websocket options")
}
}
parsed, err := url.Parse(rawURL)
c.nj.HandleError(err, "invalid websocket url")
c.nj.Require(parsed.Scheme == "ws" || parsed.Scheme == "wss", "websocket url must use ws:// or wss://")
c.nj.Require(parsed.Hostname() != "", "websocket url host cannot be empty")
c.URL = rawURL
c.executionID = c.nj.ExecutionId()
c.timeout = timeoutFor(c.nj.Context(), c.opts.TimeoutSeconds)
// defense in depth: the dialer re-checks the network policy on every dial
c.nj.Require(protocolstate.IsHostAllowed(c.executionID, parsed.Hostname()), protocolstate.ErrHostDenied.Msgf(parsed.Hostname()).Error())
return utils.LinkConstructor(call, runtime, c)
}
// Connect performs the opening handshake. It is optional; Send and Receive
// connect on demand.
// @example
// ```javascript
// const websocket = require('nuclei/websocket');
// const client = new websocket.Client('ws://acme.com/socket');
// client.Connect();
// log(client.Protocol);
// ```
func (c *Client) Connect() {
c.nj.Require(c.executionID != "", "websocket: missing executionId in runtime")
if c.conn != nil {
return
}
header := http.Header{}
for key, value := range c.opts.Headers {
header.Set(key, value)
}
responseHeaders := map[string]string{}
dialer := ws.Dialer{
Header: ws.HandshakeHeaderHTTP(header),
Protocols: c.opts.Protocols,
Timeout: c.timeout,
TLSConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10, ServerName: c.opts.ServerName},
// route through the net module so network policy and proxies apply
NetDial: jsnet.Dial,
OnHeader: func(key, value []byte) error {
responseHeaders[string(key)] = string(value)
return nil
},
}
ctx := context.WithValue(c.nj.Context(), "executionId", c.executionID) //nolint:staticcheck
conn, buffered, handshake, err := dialer.Dial(ctx, c.URL)
c.nj.HandleError(err, "websocket handshake failed")
c.conn = conn
c.reader = conn
if buffered != nil {
// frames the server sent right after the handshake are already buffered
c.reader = io.MultiReader(buffered, conn)
}
c.Protocol = handshake.Protocol
c.ResponseHeaders = responseHeaders
}
// Send sends a text message.
// @example
// ```javascript
// const websocket = require('nuclei/websocket');
// const client = new websocket.Client('ws://acme.com/socket');
// client.Send('hello');
// ```
func (c *Client) Send(data string) {
c.write(ws.OpText, []byte(data))
}
// SendHex sends a binary message given as hex.
// @example
// ```javascript
// const websocket = require('nuclei/websocket');
// const client = new websocket.Client('ws://acme.com/socket');
// client.SendHex('68656c6c6f');
// ```
func (c *Client) SendHex(data string) {
payload, err := hex.DecodeString(data)
c.nj.HandleError(err, "invalid hex data")
c.write(ws.OpBinary, payload)
}
// Receive returns the next text or binary message as a string. Pings are
// answered automatically.
// @example
// ```javascript
// const websocket = require('nuclei/websocket');
// const client = new websocket.Client('ws://acme.com/socket');
// const message = client.Receive();
// ```
func (c *Client) Receive() string {
return string(c.read())
}
// ReceiveHex returns the next text or binary message as hex.
// @example
// ```javascript
// const websocket = require('nuclei/websocket');
// const client = new websocket.Client('ws://acme.com/socket');
// const message = client.ReceiveHex();
// ```
func (c *Client) ReceiveHex() string {
return hex.EncodeToString(c.read())
}
// Close sends a close frame and closes the connection.
// @example
// ```javascript
// const websocket = require('nuclei/websocket');
// const client = new websocket.Client('ws://acme.com/socket');
// client.Close();
// ```
func (c *Client) Close() {
if c.conn == nil {
return
}
_ = c.conn.SetWriteDeadline(time.Now().Add(c.timeout))
_ = ws.WriteFrame(c.conn, ws.MaskFrame(ws.NewCloseFrame(ws.NewCloseFrameBody(ws.StatusNormalClosure, ""))))
c.release()
}
func (c *Client) release() {
_ = c.conn.Close()
c.conn = nil
c.reader = nil
}
func (c *Client) write(op ws.OpCode, payload []byte) {
c.Connect()
c.nj.HandleError(c.conn.SetWriteDeadline(time.Now().Add(c.timeout)), "could not set write deadline")
c.nj.HandleError(wsutil.WriteClientMessage(c.conn, op, payload), "websocket send failed")
}
func (c *Client) read() []byte {
c.Connect()
c.nj.HandleError(c.conn.SetReadDeadline(time.Now().Add(c.timeout)), "could not set read deadline")
control := wsutil.ControlFrameHandler(c.conn, ws.StateClientSide)
reader := wsutil.Reader{
Source: c.reader,
State: ws.StateClientSide,
MaxFrameSize: maxMessageSize,
// control frames can also arrive between the fragments of a message
OnIntermediate: control,
}
for {
header, err := reader.NextFrame()
c.nj.HandleError(err, "websocket receive failed")
if header.OpCode.IsControl() {
if err := control(header, &reader); err != nil {
var closed wsutil.ClosedError
if errors.As(err, &closed) {
// the control handler already answered the close frame
c.release()
c.nj.Throw("websocket closed by server: %d %s", closed.Code, closed.Reason)
}
c.nj.HandleError(err, "websocket receive failed")
}
continue
}
payload, err := io.ReadAll(io.LimitReader(&reader, maxMessageSize+1))
c.nj.HandleError(err, "websocket receive failed")
c.nj.Require(len(payload) <= maxMessageSize, fmt.Sprintf("websocket message exceeds %d bytes", maxMessageSize))
return payload
}
}
// timeoutFor returns the configured timeout, falling back to the scan's TCP
// read timeout like the net module.
func timeoutFor(ctx context.Context, seconds int) time.Duration {
if seconds > 0 {
return time.Duration(seconds) * time.Second
}
if timeouts, ok := ctx.Value("timeoutVariants").(*types.Timeouts); ok && timeouts != nil && timeouts.TcpReadTimeout > 0 {
return timeouts.TcpReadTimeout
}
return defaultTimeout
}
+260
View File
@@ -0,0 +1,260 @@
package websocket
import (
"context"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/gobwas/ws"
"github.com/gobwas/ws/wsutil"
"github.com/projectdiscovery/goja"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
)
// echoHandler upgrades the connection, greets, then echoes every message
// back with its original opcode until the client closes.
func echoHandler() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
upgrader := ws.HTTPUpgrader{
Protocol: func(protocol string) bool { return protocol == "chat" },
Header: http.Header{"X-Server": []string{"echo"}},
}
conn, _, _, err := upgrader.Upgrade(r, w)
if err != nil {
return
}
defer func() { _ = conn.Close() }()
if err := wsutil.WriteServerText(conn, []byte("welcome")); err != nil {
return
}
for {
payload, op, err := wsutil.ReadClientData(conn)
if err != nil {
return
}
if string(payload) == "close-me" {
_ = ws.WriteFrame(conn, ws.NewCloseFrame(ws.NewCloseFrameBody(ws.StatusGoingAway, "bye")))
return
}
if err := wsutil.WriteServerMessage(conn, op, payload); err != nil {
return
}
}
}
}
func initExec(t *testing.T, options *types.Options) string {
t.Helper()
executionID := "websocket-" + strings.NewReplacer("/", "-", " ", "-").Replace(t.Name())
options.ExecutionId = executionID
require.NoError(t, protocolstate.Init(options))
t.Cleanup(func() { protocolstate.Close(executionID) })
return executionID
}
// newRuntimeClient constructs a *Client through the goja runtime exactly as a
// nuclei template would.
func newRuntimeClient(t *testing.T, executionID, rawURL string, opts Options) (*Client, error) {
t.Helper()
runtime := goja.New()
runtime.SetContextValue("executionId", executionID)
runtime.SetContextValue("ctx", context.Background())
obj, err := runtime.New(runtime.ToValue(NewClient), runtime.ToValue(rawURL), runtime.ToValue(opts))
if err != nil {
return nil, err
}
client, ok := obj.Export().(*Client)
require.True(t, ok, "expected *Client export, got %T", obj.Export())
t.Cleanup(client.Close)
return client, nil
}
// thrown returns the message a client method throws into the JS runtime.
func thrown(fn func()) (message string) {
defer func() {
if r := recover(); r != nil {
message = fmt.Sprint(r)
}
}()
fn()
return ""
}
func wsURL(server *httptest.Server) string {
return "ws" + strings.TrimPrefix(server.URL, "http")
}
func TestTextAndBinaryRoundTrip(t *testing.T) {
server := httptest.NewServer(echoHandler())
defer server.Close()
executionID := initExec(t, &types.Options{})
client, err := newRuntimeClient(t, executionID, wsURL(server)+"/socket", Options{
Protocols: []string{"chat"},
Headers: map[string]string{"Origin": "https://acme.test"},
})
require.NoError(t, err)
require.Equal(t, "welcome", client.Receive(), "a message sent right after the handshake is not lost")
require.Equal(t, "chat", client.Protocol)
require.Equal(t, "echo", client.ResponseHeaders["X-Server"])
client.Send("hello")
require.Equal(t, "hello", client.Receive())
client.SendHex("00ff10")
require.Equal(t, "00ff10", client.ReceiveHex())
}
func TestServerCloseIsReported(t *testing.T) {
server := httptest.NewServer(echoHandler())
defer server.Close()
executionID := initExec(t, &types.Options{})
client, err := newRuntimeClient(t, executionID, wsURL(server), Options{})
require.NoError(t, err)
require.Equal(t, "welcome", client.Receive())
client.Send("close-me")
require.Contains(t, thrown(func() { client.Receive() }), "closed by server: 1001 bye")
}
func TestRejectedHandshake(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "forbidden", http.StatusForbidden)
}))
defer server.Close()
executionID := initExec(t, &types.Options{})
client, err := newRuntimeClient(t, executionID, wsURL(server), Options{})
require.NoError(t, err)
message := thrown(client.Connect)
require.Contains(t, message, "websocket handshake failed")
require.Contains(t, message, "403")
}
func TestSecureWebSocket(t *testing.T) {
server := httptest.NewTLSServer(echoHandler())
defer server.Close()
executionID := initExec(t, &types.Options{})
client, err := newRuntimeClient(t, executionID, "wss"+strings.TrimPrefix(server.URL, "https"), Options{})
require.NoError(t, err)
require.Equal(t, "welcome", client.Receive())
client.Send("over tls")
require.Equal(t, "over tls", client.Receive())
}
func TestConstructorValidation(t *testing.T) {
executionID := initExec(t, &types.Options{})
for rawURL, want := range map[string]string{
"": "cannot be empty",
"http://acme.test/": "ws:// or wss://",
"ws:///path-only": "host cannot be empty",
"ws://%zz.test/sock": "invalid websocket url",
} {
t.Run(rawURL, func(t *testing.T) {
_, err := newRuntimeClient(t, executionID, rawURL, Options{})
require.ErrorContains(t, err, want)
})
}
}
func TestNetworkPolicyDeniesHost(t *testing.T) {
executionID := initExec(t, &types.Options{ExcludeTargets: []string{"127.0.0.1"}})
_, err := newRuntimeClient(t, executionID, "ws://127.0.0.1:9/socket", Options{})
require.ErrorContains(t, err, "network policy")
}
func TestHTTPProxy(t *testing.T) {
server := httptest.NewServer(echoHandler())
defer server.Close()
target := strings.TrimPrefix(server.URL, "http://")
proxyLn, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer func() { _ = proxyLn.Close() }()
tunneled := make(chan string, 1)
go func() {
conn, err := proxyLn.Accept()
if err != nil {
return
}
defer func() { _ = conn.Close() }()
buf := make([]byte, 1024)
n, err := conn.Read(buf)
if err != nil {
return
}
tunneled <- strings.SplitN(string(buf[:n]), "\r\n", 2)[0]
upstream, err := net.Dial("tcp", target)
if err != nil {
return
}
defer func() { _ = upstream.Close() }()
_, _ = conn.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n"))
go func() { _, _ = io.Copy(upstream, conn) }()
_, _ = io.Copy(conn, upstream)
}()
executionID := initExec(t, &types.Options{})
runtime := goja.New()
runtime.SetContextValue("executionId", executionID)
runtime.SetContextValue("ctx", context.WithValue(context.Background(), "proxyURL", "http://"+proxyLn.Addr().String())) //nolint:staticcheck
obj, err := runtime.New(runtime.ToValue(NewClient), runtime.ToValue("ws://"+target+"/socket"))
require.NoError(t, err)
client := obj.Export().(*Client)
defer client.Close()
require.Equal(t, "welcome", client.Receive())
require.Equal(t, "CONNECT "+target+" HTTP/1.1", <-tunneled)
}
func TestPingInsideFragmentedMessageIsAnswered(t *testing.T) {
pong := make(chan bool, 1)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
conn, _, _, err := ws.UpgradeHTTP(r, w)
if err != nil {
return
}
defer func() { _ = conn.Close() }()
_ = ws.WriteFrame(conn, ws.NewFrame(ws.OpText, false, []byte("frag")))
_ = ws.WriteFrame(conn, ws.NewPingFrame([]byte("mid")))
_ = ws.WriteFrame(conn, ws.NewFrame(ws.OpContinuation, true, []byte("mented")))
_ = conn.SetReadDeadline(time.Now().Add(2 * time.Second))
frame, err := ws.ReadFrame(conn)
pong <- err == nil && frame.Header.OpCode == ws.OpPong
}))
defer server.Close()
executionID := initExec(t, &types.Options{})
client, err := newRuntimeClient(t, executionID, wsURL(server), Options{})
require.NoError(t, err)
require.Equal(t, "fragmented", client.Receive())
require.True(t, <-pong, "a ping between fragments must be answered")
}
func TestServerCloseReleasesConnection(t *testing.T) {
server := httptest.NewServer(echoHandler())
defer server.Close()
executionID := initExec(t, &types.Options{})
client, err := newRuntimeClient(t, executionID, wsURL(server), Options{})
require.NoError(t, err)
require.Equal(t, "welcome", client.Receive())
client.Send("close-me")
require.Contains(t, thrown(func() { client.Receive() }), "closed by server")
require.Nil(t, client.conn, "the closed connection must be released")
}
+48 -8
View File
@@ -1,6 +1,10 @@
package workflow
import (
"maps"
"slices"
"sync"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter"
@@ -13,6 +17,14 @@ type workflowLoader struct {
pathFilter *filter.PathFilter
tagFilter *templates.TagFilter
options *protocols.ExecutorOptions
// every workflow step that selects templates by tag searches the whole
// templates directory, so it is walked once per loader, and each template
// is parsed once: going through the parser per step re-stats every cached
// template, which is millions of stats when loading all workflows
templatesDirectoryPaths func() []string
parsedMu sync.Mutex
parsed map[string]*templates.Template
}
// NewLoader returns a new workflow loader structure
@@ -38,22 +50,50 @@ func NewLoader(options *protocols.ExecutorOptions) (model.WorkflowLoader, error)
ExcludedTemplates: options.Options.ExcludedTemplates,
}, options.Catalog)
return &workflowLoader{pathFilter: pathFilter, tagFilter: tagFilter, options: options}, nil
loader := &workflowLoader{pathFilter: pathFilter, tagFilter: tagFilter, options: options, parsed: make(map[string]*templates.Template)}
loader.templatesDirectoryPaths = sync.OnceValue(loader.listTemplatesDirectory)
return loader, nil
}
func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string {
func (w *workflowLoader) listTemplatesDirectory() []string {
includedTemplates, errs := w.options.Catalog.GetTemplatesPath([]string{config.DefaultConfig.TemplatesDirectory})
for template, err := range errs {
gologger.Error().Msgf("Could not find template '%s': %s", template, err)
}
return slices.Collect(maps.Keys(w.pathFilter.Match(includedTemplates)))
}
templatePathMap := w.pathFilter.Match(includedTemplates)
// parseTemplate returns the template at path, or nil if it does not parse.
// The parser is called outside the lock: parsing a workflow compiles it, which
// looks templates up by tag again on the same goroutine.
func (w *workflowLoader) parseTemplate(path string) *templates.Template {
w.parsedMu.Lock()
template, ok := w.parsed[path]
w.parsedMu.Unlock()
if ok {
return template
}
loadedTemplates := make([]string, 0, len(templatePathMap))
for templatePath := range templatePathMap {
loaded, _ := w.options.Parser.LoadTemplate(templatePath, w.tagFilter, templateTags, w.options.Catalog)
if loaded {
loadedTemplates = append(loadedTemplates, templatePath)
value, err := w.options.Parser.ParseTemplate(path, w.options.Catalog)
template, _ = value.(*templates.Template)
if err != nil {
template = nil
}
w.parsedMu.Lock()
w.parsed[path] = template
w.parsedMu.Unlock()
return template
}
func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string {
var loadedTemplates []string
for _, path := range w.templatesDirectoryPaths() {
template := w.parseTemplate(path)
if template == nil {
continue
}
if loaded, _ := templates.MatchLoadFilters(template, path, w.tagFilter, templateTags); loaded {
loadedTemplates = append(loadedTemplates, path)
}
}
return loadedTemplates
@@ -0,0 +1,71 @@
package workflow
import (
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
type countingCatalog struct {
catalog.Catalog
paths []string
walks int
}
func (c *countingCatalog) GetTemplatesPath([]string) ([]string, map[string]error) {
c.walks++
return c.paths, nil
}
type countingParser struct {
templates map[string]*templates.Template
parses map[string]int
}
func (p *countingParser) ParseTemplate(path string, _ catalog.Catalog) (any, error) {
p.parses[path]++
return p.templates[path], nil
}
func (p *countingParser) LoadTemplate(string, any, []string, catalog.Catalog) (bool, error) {
panic("tag lookups must not reparse through LoadTemplate")
}
func (p *countingParser) LoadWorkflow(string, catalog.Catalog) (bool, error) {
return false, nil
}
func TestGetTemplatePathsByTagsReadsTemplatesDirectoryOnce(t *testing.T) {
template := func(id, tags string) *templates.Template {
return &templates.Template{ID: id, Info: model.Info{
Name: id,
Authors: stringslice.StringSlice{Value: "author"},
Tags: stringslice.StringSlice{Value: tags},
SeverityHolder: severity.Holder{Severity: severity.Info},
}}
}
parser := &countingParser{
templates: map[string]*templates.Template{
"wordpress.yaml": template("wordpress", "wordpress"),
"jira.yaml": template("jira", "jira"),
},
parses: make(map[string]int),
}
cat := &countingCatalog{paths: []string{"wordpress.yaml", "jira.yaml"}}
loader, err := NewLoader(&protocols.ExecutorOptions{Options: types.DefaultOptions(), Catalog: cat, Parser: parser})
require.NoError(t, err)
walksBeforeLookups := cat.walks
require.Equal(t, []string{"wordpress.yaml"}, loader.GetTemplatePathsByTags([]string{"wordpress"}))
require.Equal(t, []string{"jira.yaml"}, loader.GetTemplatePathsByTags([]string{"jira"}))
require.Equal(t, 1, cat.walks-walksBeforeLookups, "templates directory walked per lookup")
require.Equal(t, map[string]int{"wordpress.yaml": 1, "jira.yaml": 1}, parser.parses, "templates parsed per lookup")
}
Loaded 100 of 219 files, more files were not shown because too many files have changed in this diff. Show more